From d5fba303e7abb745e5b453f22beac1083eab6e8e Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Sun, 26 Nov 2023 18:07:01 +0000 Subject: [PATCH 01/70] added rustc_llvm to compiler workspace --- Cargo.lock | 8 + Cargo.toml | 1 + compiler/rustc_llvm/Cargo.toml | 14 + compiler/rustc_llvm/build.rs | 399 ++++ .../rustc_llvm/llvm-wrapper/.editorconfig | 6 + .../llvm-wrapper/ArchiveWrapper.cpp | 217 ++ .../llvm-wrapper/CoverageMappingWrapper.cpp | 206 ++ .../rustc_llvm/llvm-wrapper/LLVMWrapper.h | 123 + compiler/rustc_llvm/llvm-wrapper/Linker.cpp | 48 + .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 1628 +++++++++++++ compiler/rustc_llvm/llvm-wrapper/README | 16 + .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 2069 +++++++++++++++++ .../rustc_llvm/llvm-wrapper/SymbolWrapper.cpp | 96 + compiler/rustc_llvm/src/lib.rs | 209 ++ 14 files changed, 5040 insertions(+) create mode 100644 compiler/rustc_llvm/Cargo.toml create mode 100644 compiler/rustc_llvm/build.rs create mode 100644 compiler/rustc_llvm/llvm-wrapper/.editorconfig create mode 100644 compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp create mode 100644 compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp create mode 100644 compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h create mode 100644 compiler/rustc_llvm/llvm-wrapper/Linker.cpp create mode 100644 compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp create mode 100644 compiler/rustc_llvm/llvm-wrapper/README create mode 100644 compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp create mode 100644 compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp create mode 100644 compiler/rustc_llvm/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 5e5cbeda91f..fd79975371c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2241,6 +2241,14 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc_llvm" +version = "0.0.0" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "rustix" version = "0.37.25" diff --git a/Cargo.toml b/Cargo.toml index ebaef5a49eb..9e3163e9a4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ members = [ "compiler/plc_util", "compiler/plc_xml", "compiler/plc_derive", + "compiler/rustc_llvm", ] default-members = [".", "compiler/plc_driver", "compiler/plc_xml"] diff --git a/compiler/rustc_llvm/Cargo.toml b/compiler/rustc_llvm/Cargo.toml new file mode 100644 index 00000000000..58e219e5a46 --- /dev/null +++ b/compiler/rustc_llvm/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "rustc_llvm" +version = "0.0.0" +edition = "2021" + +[dependencies] +# tidy-alphabetical-start +libc = "0.2.73" +# tidy-alphabetical-end + +[build-dependencies] +# tidy-alphabetical-start +cc = "1.0.69" +# tidy-alphabetical-end diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs new file mode 100644 index 00000000000..ed1e8771323 --- /dev/null +++ b/compiler/rustc_llvm/build.rs @@ -0,0 +1,399 @@ +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fmt::Display; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +const OPTIONAL_COMPONENTS: &[&str] = &[ + "x86", + "arm", + "aarch64", + "amdgpu", + "avr", + "loongarch", + "m68k", + "csky", + "mips", + "powerpc", + "systemz", + "jsbackend", + "webassembly", + "msp430", + "sparc", + "nvptx", + "hexagon", + "riscv", + "bpf", +]; + +const REQUIRED_COMPONENTS: &[&str] = + &["ipo", "bitreader", "bitwriter", "linker", "asmparser", "lto", "coverage", "instrumentation"]; + +fn detect_llvm_link() -> (&'static str, &'static str) { + // Force the link mode we want, preferring static by default, but + // possibly overridden by `configure --enable-llvm-link-shared`. + if tracked_env_var_os("LLVM_LINK_SHARED").is_some() { + ("dylib", "--link-shared") + } else { + ("static", "--link-static") + } +} + +// Because Cargo adds the compiler's dylib path to our library search path, llvm-config may +// break: the dylib path for the compiler, as of this writing, contains a copy of the LLVM +// shared library, which means that when our freshly built llvm-config goes to load it's +// associated LLVM, it actually loads the compiler's LLVM. In particular when building the first +// compiler (i.e., in stage 0) that's a problem, as the compiler's LLVM is likely different from +// the one we want to use. As such, we restore the environment to what bootstrap saw. This isn't +// perfect -- we might actually want to see something from Cargo's added library paths -- but +// for now it works. +fn restore_library_path() { + let key = tracked_env_var_os("REAL_LIBRARY_PATH_VAR").expect("REAL_LIBRARY_PATH_VAR"); + if let Some(env) = tracked_env_var_os("REAL_LIBRARY_PATH") { + env::set_var(&key, &env); + } else { + env::remove_var(&key); + } +} + +/// Reads an environment variable and adds it to dependencies. +/// Supposed to be used for all variables except those set for build scripts by cargo +/// +fn tracked_env_var_os + Display>(key: K) -> Option { + println!("cargo:rerun-if-env-changed={key}"); + env::var_os(key) +} + +fn rerun_if_changed_anything_in_dir(dir: &Path) { + let mut stack = dir + .read_dir() + .unwrap() + .map(|e| e.unwrap()) + .filter(|e| &*e.file_name() != ".git") + .collect::>(); + while let Some(entry) = stack.pop() { + let path = entry.path(); + if entry.file_type().unwrap().is_dir() { + stack.extend(path.read_dir().unwrap().map(|e| e.unwrap())); + } else { + println!("cargo:rerun-if-changed={}", path.display()); + } + } +} + +#[track_caller] +fn output(cmd: &mut Command) -> String { + let output = match cmd.stderr(Stdio::inherit()).output() { + Ok(status) => status, + Err(e) => { + println!("\n\nfailed to execute command: {cmd:?}\nerror: {e}\n\n"); + std::process::exit(1); + } + }; + if !output.status.success() { + panic!( + "command did not execute successfully: {:?}\n\ + expected success, got: {}", + cmd, output.status + ); + } + String::from_utf8(output.stdout).unwrap() +} + +fn main() { + for component in REQUIRED_COMPONENTS.iter().chain(OPTIONAL_COMPONENTS.iter()) { + println!("cargo:rustc-check-cfg=values(llvm_component,\"{component}\")"); + } + + if tracked_env_var_os("RUST_CHECK").is_some() { + // If we're just running `check`, there's no need for LLVM to be built. + return; + } + + restore_library_path(); + + let target = env::var("TARGET").expect("TARGET was not set"); + let llvm_config = + tracked_env_var_os("LLVM_CONFIG").map(|x| Some(PathBuf::from(x))).unwrap_or_else(|| { + if let Some(dir) = tracked_env_var_os("CARGO_TARGET_DIR").map(PathBuf::from) { + let to_test = dir + .parent() + .unwrap() + .parent() + .unwrap() + .join(&target) + .join("llvm/bin/llvm-config"); + if Command::new(&to_test).output().is_ok() { + return Some(to_test); + } + } + None + }); + + if let Some(llvm_config) = &llvm_config { + println!("cargo:rerun-if-changed={}", llvm_config.display()); + } + let llvm_config = llvm_config.unwrap_or_else(|| PathBuf::from("llvm-config")); + + // Test whether we're cross-compiling LLVM. This is a pretty rare case + // currently where we're producing an LLVM for a different platform than + // what this build script is currently running on. + // + // In that case, there's no guarantee that we can actually run the target, + // so the build system works around this by giving us the LLVM_CONFIG for + // the host platform. This only really works if the host LLVM and target + // LLVM are compiled the same way, but for us that's typically the case. + // + // We *want* detect this cross compiling situation by asking llvm-config + // what its host-target is. If that's not the TARGET, then we're cross + // compiling. Unfortunately `llvm-config` seems either be buggy, or we're + // misconfiguring it, because the `i686-pc-windows-gnu` build of LLVM will + // report itself with a `--host-target` of `x86_64-pc-windows-gnu`. This + // tricks us into thinking we're doing a cross build when we aren't, so + // havoc ensues. + // + // In any case, if we're cross compiling, this generally just means that we + // can't trust all the output of llvm-config because it might be targeted + // for the host rather than the target. As a result a bunch of blocks below + // are gated on `if !is_crossed` + let target = env::var("TARGET").expect("TARGET was not set"); + let host = env::var("HOST").expect("HOST was not set"); + let is_crossed = target != host; + + let components = output(Command::new(&llvm_config).arg("--components")); + let mut components = components.split_whitespace().collect::>(); + components.retain(|c| OPTIONAL_COMPONENTS.contains(c) || REQUIRED_COMPONENTS.contains(c)); + + for component in REQUIRED_COMPONENTS { + if !components.contains(component) { + panic!("require llvm component {component} but wasn't found"); + } + } + + for component in components.iter() { + println!("cargo:rustc-cfg=llvm_component=\"{component}\""); + } + + // Link in our own LLVM shims, compiled with the same flags as LLVM + let mut cmd = Command::new(&llvm_config); + cmd.arg("--cxxflags"); + let cxxflags = output(&mut cmd); + let mut cfg = cc::Build::new(); + cfg.warnings(false); + for flag in cxxflags.split_whitespace() { + // Ignore flags like `-m64` when we're doing a cross build + if is_crossed && flag.starts_with("-m") { + continue; + } + + if flag.starts_with("-flto") { + continue; + } + + // -Wdate-time is not supported by the netbsd cross compiler + if is_crossed && target.contains("netbsd") && flag.contains("date-time") { + continue; + } + + // Include path contains host directory, replace it with target + if is_crossed && flag.starts_with("-I") { + cfg.flag(&flag.replace(&host, &target)); + continue; + } + + cfg.flag(flag); + } + + for component in &components { + let mut flag = String::from("LLVM_COMPONENT_"); + flag.push_str(&component.to_uppercase()); + cfg.define(&flag, None); + } + + if tracked_env_var_os("LLVM_RUSTLLVM").is_some() { + cfg.define("LLVM_RUSTLLVM", None); + } + + if tracked_env_var_os("LLVM_NDEBUG").is_some() { + cfg.define("NDEBUG", None); + cfg.debug(false); + } + + rerun_if_changed_anything_in_dir(Path::new("llvm-wrapper")); + cfg.file("llvm-wrapper/PassWrapper.cpp") + .file("llvm-wrapper/RustWrapper.cpp") + .file("llvm-wrapper/ArchiveWrapper.cpp") + .file("llvm-wrapper/CoverageMappingWrapper.cpp") + .file("llvm-wrapper/SymbolWrapper.cpp") + .file("llvm-wrapper/Linker.cpp") + .cpp(true) + .cpp_link_stdlib(None) // we handle this below + .compile("llvm-wrapper"); + + let (llvm_kind, llvm_link_arg) = detect_llvm_link(); + + // Link in all LLVM libraries, if we're using the "wrong" llvm-config then + // we don't pick up system libs because unfortunately they're for the host + // of llvm-config, not the target that we're attempting to link. + let mut cmd = Command::new(&llvm_config); + cmd.arg(llvm_link_arg).arg("--libs"); + + if !is_crossed { + cmd.arg("--system-libs"); + } + + // We need libkstat for getHostCPUName on SPARC builds. + // See also: https://github.com/llvm/llvm-project/issues/64186 + if target.starts_with("sparcv9") && target.contains("solaris") { + println!("cargo:rustc-link-lib=kstat"); + } + + if (target.starts_with("arm") && !target.contains("freebsd")) + || target.starts_with("mips-") + || target.starts_with("mipsel-") + || target.starts_with("powerpc-") + { + // 32-bit targets need to link libatomic. + println!("cargo:rustc-link-lib=atomic"); + } else if target.contains("windows-gnu") { + println!("cargo:rustc-link-lib=shell32"); + println!("cargo:rustc-link-lib=uuid"); + } else if target.contains("haiku") + || target.contains("darwin") + || (is_crossed && (target.contains("dragonfly") || target.contains("solaris"))) + { + println!("cargo:rustc-link-lib=z"); + } else if target.contains("netbsd") { + // On NetBSD/i386, gcc and g++ is built for i486 (to maximize backward compat) + // However, LLVM insists on using 64-bit atomics. + // This gives rise to a need to link rust itself with -latomic for these targets + if target.starts_with("i586") || target.starts_with("i686") { + println!("cargo:rustc-link-lib=atomic"); + } + println!("cargo:rustc-link-lib=z"); + println!("cargo:rustc-link-lib=execinfo"); + } + cmd.args(&components); + + for lib in output(&mut cmd).split_whitespace() { + let name = if let Some(stripped) = lib.strip_prefix("-l") { + stripped + } else if let Some(stripped) = lib.strip_prefix('-') { + stripped + } else if Path::new(lib).exists() { + // On MSVC llvm-config will print the full name to libraries, but + // we're only interested in the name part + let name = Path::new(lib).file_name().unwrap().to_str().unwrap(); + name.trim_end_matches(".lib") + } else if lib.ends_with(".lib") { + // Some MSVC libraries just come up with `.lib` tacked on, so chop + // that off + lib.trim_end_matches(".lib") + } else { + continue; + }; + + // Don't need or want this library, but LLVM's CMake build system + // doesn't provide a way to disable it, so filter it here even though we + // may or may not have built it. We don't reference anything from this + // library and it otherwise may just pull in extra dependencies on + // libedit which we don't want + if name == "LLVMLineEditor" { + continue; + } + + let kind = if name.starts_with("LLVM") { llvm_kind } else { "dylib" }; + println!("cargo:rustc-link-lib={kind}={name}"); + } + + // LLVM ldflags + // + // If we're a cross-compile of LLVM then unfortunately we can't trust these + // ldflags (largely where all the LLVM libs are located). Currently just + // hack around this by replacing the host triple with the target and pray + // that those -L directories are the same! + let mut cmd = Command::new(&llvm_config); + cmd.arg(llvm_link_arg).arg("--ldflags"); + for lib in output(&mut cmd).split_whitespace() { + if is_crossed { + if let Some(stripped) = lib.strip_prefix("-LIBPATH:") { + println!("cargo:rustc-link-search=native={}", stripped.replace(&host, &target)); + } else if let Some(stripped) = lib.strip_prefix("-L") { + println!("cargo:rustc-link-search=native={}", stripped.replace(&host, &target)); + } + } else if let Some(stripped) = lib.strip_prefix("-LIBPATH:") { + println!("cargo:rustc-link-search=native={stripped}"); + } else if let Some(stripped) = lib.strip_prefix("-l") { + println!("cargo:rustc-link-lib={stripped}"); + } else if let Some(stripped) = lib.strip_prefix("-L") { + println!("cargo:rustc-link-search=native={stripped}"); + } + } + + // Some LLVM linker flags (-L and -l) may be needed even when linking + // rustc_llvm, for example when using static libc++, we may need to + // manually specify the library search path and -ldl -lpthread as link + // dependencies. + let llvm_linker_flags = tracked_env_var_os("LLVM_LINKER_FLAGS"); + if let Some(s) = llvm_linker_flags { + for lib in s.into_string().unwrap().split_whitespace() { + if let Some(stripped) = lib.strip_prefix("-l") { + println!("cargo:rustc-link-lib={stripped}"); + } else if let Some(stripped) = lib.strip_prefix("-L") { + println!("cargo:rustc-link-search=native={stripped}"); + } + } + } + + let llvm_static_stdcpp = tracked_env_var_os("LLVM_STATIC_STDCPP"); + let llvm_use_libcxx = tracked_env_var_os("LLVM_USE_LIBCXX"); + + let stdcppname = if target.contains("openbsd") { + if target.contains("sparc64") { "estdc++" } else { "c++" } + } else if target.contains("darwin") + || target.contains("freebsd") + || target.contains("windows-gnullvm") + || target.contains("aix") + { + "c++" + } else if target.contains("netbsd") && llvm_static_stdcpp.is_some() { + // NetBSD uses a separate library when relocation is required + "stdc++_p" + } else if llvm_use_libcxx.is_some() { + "c++" + } else { + "stdc++" + }; + + // RISC-V GCC erroneously requires libatomic for sub-word + // atomic operations. Some BSD uses Clang as its system + // compiler and provides no libatomic in its base system so + // does not want this. + if target.starts_with("riscv") && !target.contains("freebsd") && !target.contains("openbsd") { + println!("cargo:rustc-link-lib=atomic"); + } + + // C++ runtime library + if !target.contains("msvc") { + if let Some(s) = llvm_static_stdcpp { + assert!(!cxxflags.contains("stdlib=libc++")); + let path = PathBuf::from(s); + println!("cargo:rustc-link-search=native={}", path.parent().unwrap().display()); + if target.contains("windows") { + println!("cargo:rustc-link-lib=static:-bundle={stdcppname}"); + } else { + println!("cargo:rustc-link-lib=static={stdcppname}"); + } + } else if cxxflags.contains("stdlib=libc++") { + println!("cargo:rustc-link-lib=c++"); + } else { + println!("cargo:rustc-link-lib={stdcppname}"); + } + } + + // Libstdc++ depends on pthread which Rust doesn't link on MinGW + // since nothing else requires it. + if target.ends_with("windows-gnu") { + println!("cargo:rustc-link-lib=static:-bundle=pthread"); + } +} diff --git a/compiler/rustc_llvm/llvm-wrapper/.editorconfig b/compiler/rustc_llvm/llvm-wrapper/.editorconfig new file mode 100644 index 00000000000..865cd45f708 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/.editorconfig @@ -0,0 +1,6 @@ +[*.{h,cpp}] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_style = space +indent_size = 2 diff --git a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp new file mode 100644 index 00000000000..54fdc84c77d --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp @@ -0,0 +1,217 @@ +#include "LLVMWrapper.h" + +#include "llvm/Object/Archive.h" +#include "llvm/Object/ArchiveWriter.h" +#include "llvm/Support/Path.h" + +using namespace llvm; +using namespace llvm::object; + +struct RustArchiveMember { + const char *Filename; + const char *Name; + Archive::Child Child; + + RustArchiveMember() + : Filename(nullptr), Name(nullptr), + Child(nullptr, nullptr, nullptr) + { + } + ~RustArchiveMember() {} +}; + +struct RustArchiveIterator { + bool First; + Archive::child_iterator Cur; + Archive::child_iterator End; + std::unique_ptr Err; + + RustArchiveIterator(Archive::child_iterator Cur, Archive::child_iterator End, + std::unique_ptr Err) + : First(true), + Cur(Cur), + End(End), + Err(std::move(Err)) {} +}; + +enum class LLVMRustArchiveKind { + GNU, + BSD, + DARWIN, + COFF, + AIX_BIG, +}; + +static Archive::Kind fromRust(LLVMRustArchiveKind Kind) { + switch (Kind) { + case LLVMRustArchiveKind::GNU: + return Archive::K_GNU; + case LLVMRustArchiveKind::BSD: + return Archive::K_BSD; + case LLVMRustArchiveKind::DARWIN: + return Archive::K_DARWIN; + case LLVMRustArchiveKind::COFF: + return Archive::K_COFF; + case LLVMRustArchiveKind::AIX_BIG: + return Archive::K_AIXBIG; + default: + report_fatal_error("Bad ArchiveKind."); + } +} + +typedef OwningBinary *LLVMRustArchiveRef; +typedef RustArchiveMember *LLVMRustArchiveMemberRef; +typedef Archive::Child *LLVMRustArchiveChildRef; +typedef Archive::Child const *LLVMRustArchiveChildConstRef; +typedef RustArchiveIterator *LLVMRustArchiveIteratorRef; + +extern "C" LLVMRustArchiveRef LLVMRustOpenArchive(char *Path) { + ErrorOr> BufOr = + MemoryBuffer::getFile(Path, -1, false); + if (!BufOr) { + LLVMRustSetLastError(BufOr.getError().message().c_str()); + return nullptr; + } + + Expected> ArchiveOr = + Archive::create(BufOr.get()->getMemBufferRef()); + + if (!ArchiveOr) { + LLVMRustSetLastError(toString(ArchiveOr.takeError()).c_str()); + return nullptr; + } + + OwningBinary *Ret = new OwningBinary( + std::move(ArchiveOr.get()), std::move(BufOr.get())); + + return Ret; +} + +extern "C" void LLVMRustDestroyArchive(LLVMRustArchiveRef RustArchive) { + delete RustArchive; +} + +extern "C" LLVMRustArchiveIteratorRef +LLVMRustArchiveIteratorNew(LLVMRustArchiveRef RustArchive) { + Archive *Archive = RustArchive->getBinary(); + std::unique_ptr Err = std::make_unique(Error::success()); + auto Cur = Archive->child_begin(*Err); + if (*Err) { + LLVMRustSetLastError(toString(std::move(*Err)).c_str()); + return nullptr; + } + auto End = Archive->child_end(); + return new RustArchiveIterator(Cur, End, std::move(Err)); +} + +extern "C" LLVMRustArchiveChildConstRef +LLVMRustArchiveIteratorNext(LLVMRustArchiveIteratorRef RAI) { + if (RAI->Cur == RAI->End) + return nullptr; + + // Advancing the iterator validates the next child, and this can + // uncover an error. LLVM requires that we check all Errors, + // so we only advance the iterator if we actually need to fetch + // the next child. + // This means we must not advance the iterator in the *first* call, + // but instead advance it *before* fetching the child in all later calls. + if (!RAI->First) { + ++RAI->Cur; + if (*RAI->Err) { + LLVMRustSetLastError(toString(std::move(*RAI->Err)).c_str()); + return nullptr; + } + } else { + RAI->First = false; + } + + if (RAI->Cur == RAI->End) + return nullptr; + + const Archive::Child &Child = *RAI->Cur.operator->(); + Archive::Child *Ret = new Archive::Child(Child); + + return Ret; +} + +extern "C" void LLVMRustArchiveChildFree(LLVMRustArchiveChildRef Child) { + delete Child; +} + +extern "C" void LLVMRustArchiveIteratorFree(LLVMRustArchiveIteratorRef RAI) { + delete RAI; +} + +extern "C" const char * +LLVMRustArchiveChildName(LLVMRustArchiveChildConstRef Child, size_t *Size) { + Expected NameOrErr = Child->getName(); + if (!NameOrErr) { + // rustc_codegen_llvm currently doesn't use this error string, but it might be + // useful in the future, and in the mean time this tells LLVM that the + // error was not ignored and that it shouldn't abort the process. + LLVMRustSetLastError(toString(NameOrErr.takeError()).c_str()); + return nullptr; + } + StringRef Name = NameOrErr.get(); + *Size = Name.size(); + return Name.data(); +} + +extern "C" LLVMRustArchiveMemberRef +LLVMRustArchiveMemberNew(char *Filename, char *Name, + LLVMRustArchiveChildRef Child) { + RustArchiveMember *Member = new RustArchiveMember; + Member->Filename = Filename; + Member->Name = Name; + if (Child) + Member->Child = *Child; + return Member; +} + +extern "C" void LLVMRustArchiveMemberFree(LLVMRustArchiveMemberRef Member) { + delete Member; +} + +extern "C" LLVMRustResult +LLVMRustWriteArchive(char *Dst, size_t NumMembers, + const LLVMRustArchiveMemberRef *NewMembers, + bool WriteSymbtab, LLVMRustArchiveKind RustKind) { + + std::vector Members; + auto Kind = fromRust(RustKind); + + for (size_t I = 0; I < NumMembers; I++) { + auto Member = NewMembers[I]; + assert(Member->Name); + if (Member->Filename) { + Expected MOrErr = + NewArchiveMember::getFile(Member->Filename, true); + if (!MOrErr) { + LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); + return LLVMRustResult::Failure; + } + MOrErr->MemberName = sys::path::filename(MOrErr->MemberName); + Members.push_back(std::move(*MOrErr)); + } else { + Expected MOrErr = + NewArchiveMember::getOldMember(Member->Child, true); + if (!MOrErr) { + LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); + return LLVMRustResult::Failure; + } + Members.push_back(std::move(*MOrErr)); + } + } + +#if LLVM_VERSION_LT(18, 0) + auto Result = writeArchive(Dst, Members, WriteSymbtab, Kind, true, false); +#else + auto SymtabMode = WriteSymbtab ? SymtabWritingMode::NormalSymtab : SymtabWritingMode::NoSymtab; + auto Result = writeArchive(Dst, Members, SymtabMode, Kind, true, false); +#endif + if (!Result) + return LLVMRustResult::Success; + LLVMRustSetLastError(toString(std::move(Result)).c_str()); + + return LLVMRustResult::Failure; +} diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp new file mode 100644 index 00000000000..d61ec0b641c --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp @@ -0,0 +1,206 @@ +#include "LLVMWrapper.h" +#include "llvm/ProfileData/Coverage/CoverageMapping.h" +#include "llvm/ProfileData/Coverage/CoverageMappingWriter.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/ADT/ArrayRef.h" + +#include + +using namespace llvm; + +// FFI equivalent of enum `llvm::coverage::Counter::CounterKind` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L97-L99 +enum class LLVMRustCounterKind { + Zero = 0, + CounterValueReference = 1, + Expression = 2, +}; + +// FFI equivalent of struct `llvm::coverage::Counter` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L94-L149 +struct LLVMRustCounter { + LLVMRustCounterKind CounterKind; + uint32_t ID; +}; + +static coverage::Counter fromRust(LLVMRustCounter Counter) { + switch (Counter.CounterKind) { + case LLVMRustCounterKind::Zero: + return coverage::Counter::getZero(); + case LLVMRustCounterKind::CounterValueReference: + return coverage::Counter::getCounter(Counter.ID); + case LLVMRustCounterKind::Expression: + return coverage::Counter::getExpression(Counter.ID); + } + report_fatal_error("Bad LLVMRustCounterKind!"); +} + +// FFI equivalent of enum `llvm::coverage::CounterMappingRegion::RegionKind` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L213-L234 +enum class LLVMRustCounterMappingRegionKind { + CodeRegion = 0, + ExpansionRegion = 1, + SkippedRegion = 2, + GapRegion = 3, + BranchRegion = 4, +}; + +static coverage::CounterMappingRegion::RegionKind +fromRust(LLVMRustCounterMappingRegionKind Kind) { + switch (Kind) { + case LLVMRustCounterMappingRegionKind::CodeRegion: + return coverage::CounterMappingRegion::CodeRegion; + case LLVMRustCounterMappingRegionKind::ExpansionRegion: + return coverage::CounterMappingRegion::ExpansionRegion; + case LLVMRustCounterMappingRegionKind::SkippedRegion: + return coverage::CounterMappingRegion::SkippedRegion; + case LLVMRustCounterMappingRegionKind::GapRegion: + return coverage::CounterMappingRegion::GapRegion; + case LLVMRustCounterMappingRegionKind::BranchRegion: + return coverage::CounterMappingRegion::BranchRegion; + } + report_fatal_error("Bad LLVMRustCounterMappingRegionKind!"); +} + +// FFI equivalent of struct `llvm::coverage::CounterMappingRegion` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L211-L304 +struct LLVMRustCounterMappingRegion { + LLVMRustCounter Count; + LLVMRustCounter FalseCount; + uint32_t FileID; + uint32_t ExpandedFileID; + uint32_t LineStart; + uint32_t ColumnStart; + uint32_t LineEnd; + uint32_t ColumnEnd; + LLVMRustCounterMappingRegionKind Kind; +}; + +// FFI equivalent of enum `llvm::coverage::CounterExpression::ExprKind` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L154 +enum class LLVMRustCounterExprKind { + Subtract = 0, + Add = 1, +}; + +// FFI equivalent of struct `llvm::coverage::CounterExpression` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L160 +struct LLVMRustCounterExpression { + LLVMRustCounterExprKind Kind; + LLVMRustCounter LHS; + LLVMRustCounter RHS; +}; + +static coverage::CounterExpression::ExprKind +fromRust(LLVMRustCounterExprKind Kind) { + switch (Kind) { + case LLVMRustCounterExprKind::Subtract: + return coverage::CounterExpression::Subtract; + case LLVMRustCounterExprKind::Add: + return coverage::CounterExpression::Add; + } + report_fatal_error("Bad LLVMRustCounterExprKind!"); +} + +extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer( + const char *const Filenames[], + size_t FilenamesLen, + const size_t *const Lengths, + size_t LengthsLen, + RustStringRef BufferOut) { + if (FilenamesLen != LengthsLen) { + report_fatal_error( + "Mismatched lengths in LLVMRustCoverageWriteFilenamesSectionToBuffer"); + } + + SmallVector FilenameRefs; + FilenameRefs.reserve(FilenamesLen); + for (size_t i = 0; i < FilenamesLen; i++) { + FilenameRefs.emplace_back(Filenames[i], Lengths[i]); + } + auto FilenamesWriter = + coverage::CoverageFilenamesSectionWriter(ArrayRef(FilenameRefs)); + RawRustStringOstream OS(BufferOut); + FilenamesWriter.write(OS); +} + +extern "C" void LLVMRustCoverageWriteMappingToBuffer( + const unsigned *VirtualFileMappingIDs, + unsigned NumVirtualFileMappingIDs, + const LLVMRustCounterExpression *RustExpressions, + unsigned NumExpressions, + const LLVMRustCounterMappingRegion *RustMappingRegions, + unsigned NumMappingRegions, + RustStringRef BufferOut) { + // Convert from FFI representation to LLVM representation. + SmallVector MappingRegions; + MappingRegions.reserve(NumMappingRegions); + for (const auto &Region : ArrayRef( + RustMappingRegions, NumMappingRegions)) { + MappingRegions.emplace_back( + fromRust(Region.Count), fromRust(Region.FalseCount), + Region.FileID, Region.ExpandedFileID, + Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd, + fromRust(Region.Kind)); + } + + std::vector Expressions; + Expressions.reserve(NumExpressions); + for (const auto &Expression : + ArrayRef(RustExpressions, NumExpressions)) { + Expressions.emplace_back(fromRust(Expression.Kind), + fromRust(Expression.LHS), + fromRust(Expression.RHS)); + } + + auto CoverageMappingWriter = coverage::CoverageMappingWriter( + ArrayRef(VirtualFileMappingIDs, NumVirtualFileMappingIDs), + Expressions, + MappingRegions); + RawRustStringOstream OS(BufferOut); + CoverageMappingWriter.write(OS); +} + +extern "C" LLVMValueRef LLVMRustCoverageCreatePGOFuncNameVar( + LLVMValueRef F, + const char *FuncName, + size_t FuncNameLen) { + StringRef FuncNameRef(FuncName, FuncNameLen); + return wrap(createPGOFuncNameVar(*cast(unwrap(F)), FuncNameRef)); +} + +extern "C" uint64_t LLVMRustCoverageHashByteArray( + const char *Bytes, + size_t NumBytes) { + StringRef StrRef(Bytes, NumBytes); + return IndexedInstrProf::ComputeHash(StrRef); +} + +static void WriteSectionNameToString(LLVMModuleRef M, + InstrProfSectKind SK, + RustStringRef Str) { + Triple TargetTriple(unwrap(M)->getTargetTriple()); + auto name = getInstrProfSectionName(SK, TargetTriple.getObjectFormat()); + RawRustStringOstream OS(Str); + OS << name; +} + +extern "C" void LLVMRustCoverageWriteMapSectionNameToString(LLVMModuleRef M, + RustStringRef Str) { + WriteSectionNameToString(M, IPSK_covmap, Str); +} + +extern "C" void LLVMRustCoverageWriteFuncSectionNameToString(LLVMModuleRef M, + RustStringRef Str) { + WriteSectionNameToString(M, IPSK_covfun, Str); +} + +extern "C" void LLVMRustCoverageWriteMappingVarNameToString(RustStringRef Str) { + auto name = getCoverageMappingVarName(); + RawRustStringOstream OS(Str); + OS << name; +} + +extern "C" uint32_t LLVMRustCoverageMappingVersion() { + return coverage::CovMapVersion::Version6; +} diff --git a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h new file mode 100644 index 00000000000..142384e6d0c --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h @@ -0,0 +1,123 @@ +#include "llvm-c/BitReader.h" +#include "llvm-c/Core.h" +#include "llvm-c/Object.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/Lint.h" +#include "llvm/Analysis/Passes.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/Memory.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Scalar.h" + +#define LLVM_VERSION_GE(major, minor) \ + (LLVM_VERSION_MAJOR > (major) || \ + LLVM_VERSION_MAJOR == (major) && LLVM_VERSION_MINOR >= (minor)) + +#define LLVM_VERSION_LT(major, minor) (!LLVM_VERSION_GE((major), (minor))) + +#include "llvm/IR/LegacyPassManager.h" + +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" + +#include "llvm/IR/DIBuilder.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/IRPrintingPasses.h" +#include "llvm/Linker/Linker.h" + +#if LLVM_VERSION_GE(16, 0) +#include "llvm/TargetParser/Triple.h" +#else +#include "llvm/ADT/Triple.h" +#endif + +extern "C" void LLVMRustSetLastError(const char *); + +enum class LLVMRustResult { Success, Failure }; + +enum LLVMRustAttribute { + AlwaysInline = 0, + ByVal = 1, + Cold = 2, + InlineHint = 3, + MinSize = 4, + Naked = 5, + NoAlias = 6, + NoCapture = 7, + NoInline = 8, + NonNull = 9, + NoRedZone = 10, + NoReturn = 11, + NoUnwind = 12, + OptimizeForSize = 13, + ReadOnly = 14, + SExt = 15, + StructRet = 16, + UWTable = 17, + ZExt = 18, + InReg = 19, + SanitizeThread = 20, + SanitizeAddress = 21, + SanitizeMemory = 22, + NonLazyBind = 23, + OptimizeNone = 24, + ReturnsTwice = 25, + ReadNone = 26, + SanitizeHWAddress = 28, + WillReturn = 29, + StackProtectReq = 30, + StackProtectStrong = 31, + StackProtect = 32, + NoUndef = 33, + SanitizeMemTag = 34, + NoCfCheck = 35, + ShadowCallStack = 36, + AllocSize = 37, + AllocatedPointer = 38, + AllocAlign = 39, + SanitizeSafeStack = 40, +}; + +typedef struct OpaqueRustString *RustStringRef; +typedef struct LLVMOpaqueTwine *LLVMTwineRef; +typedef struct LLVMOpaqueSMDiagnostic *LLVMSMDiagnosticRef; + +extern "C" void LLVMRustStringWriteImpl(RustStringRef Str, const char *Ptr, + size_t Size); + +class RawRustStringOstream : public llvm::raw_ostream { + RustStringRef Str; + uint64_t Pos; + + void write_impl(const char *Ptr, size_t Size) override { + LLVMRustStringWriteImpl(Str, Ptr, Size); + Pos += Size; + } + + uint64_t current_pos() const override { return Pos; } + +public: + explicit RawRustStringOstream(RustStringRef Str) : Str(Str), Pos(0) {} + + ~RawRustStringOstream() { + // LLVM requires this. + flush(); + } +}; diff --git a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp new file mode 100644 index 00000000000..8766e96f086 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp @@ -0,0 +1,48 @@ +#include "llvm/Linker/Linker.h" + +#include "LLVMWrapper.h" + +using namespace llvm; + +struct RustLinker { + Linker L; + LLVMContext &Ctx; + + RustLinker(Module &M) : + L(M), + Ctx(M.getContext()) + {} +}; + +extern "C" RustLinker* +LLVMRustLinkerNew(LLVMModuleRef DstRef) { + Module *Dst = unwrap(DstRef); + + return new RustLinker(*Dst); +} + +extern "C" void +LLVMRustLinkerFree(RustLinker *L) { + delete L; +} + +extern "C" bool +LLVMRustLinkerAdd(RustLinker *L, char *BC, size_t Len) { + std::unique_ptr Buf = + MemoryBuffer::getMemBufferCopy(StringRef(BC, Len)); + + Expected> SrcOrError = + llvm::getLazyBitcodeModule(Buf->getMemBufferRef(), L->Ctx); + if (!SrcOrError) { + LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); + return false; + } + + auto Src = std::move(*SrcOrError); + + if (L->L.linkInModule(std::move(Src))) { + LLVMRustSetLastError(""); + return false; + } + return true; +} diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp new file mode 100644 index 00000000000..d2cd79b456a --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -0,0 +1,1628 @@ +#include + +#include +#include +#include +#include + +#include "LLVMWrapper.h" + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/IR/AutoUpgrade.h" +#include "llvm/IR/AssemblyAnnotationWriter.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Verifier.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/IRObjectFile.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Passes/PassPlugin.h" +#include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/Support/CBindingWrapping.h" +#include "llvm/Support/FileSystem.h" +#if LLVM_VERSION_GE(17, 0) +#include "llvm/Support/VirtualFileSystem.h" +#endif +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/Transforms/IPO/FunctionImport.h" +#include "llvm/Transforms/IPO/Internalize.h" +#include "llvm/Transforms/IPO/LowerTypeTests.h" +#include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" +#include "llvm/Transforms/Utils/AddDiscriminators.h" +#include "llvm/Transforms/Utils/FunctionImportUtils.h" +#include "llvm/LTO/LTO.h" +#include "llvm/Bitcode/BitcodeWriter.h" + +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" +#include "llvm/Support/TimeProfiler.h" +#include "llvm/Transforms/Instrumentation/GCOVProfiler.h" +#include "llvm/Transforms/Instrumentation/InstrProfiling.h" +#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" +#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" +#include "llvm/Transforms/Utils/CanonicalizeAliases.h" +#include "llvm/Transforms/Utils/NameAnonGlobals.h" +#include "llvm/Transforms/Utils.h" + +using namespace llvm; + +static codegen::RegisterCodeGenFlags CGF; + +typedef struct LLVMOpaquePass *LLVMPassRef; +typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; + +DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) +DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + +extern "C" void LLVMRustTimeTraceProfilerInitialize() { + timeTraceProfilerInitialize( + /* TimeTraceGranularity */ 0, + /* ProcName */ "rustc"); +} + +extern "C" void LLVMRustTimeTraceProfilerFinishThread() { + timeTraceProfilerFinishThread(); +} + +extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) { + StringRef FN(FileName); + std::error_code EC; + raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); + + timeTraceProfilerWrite(OS); + timeTraceProfilerCleanup(); +} + +#ifdef LLVM_COMPONENT_X86 +#define SUBTARGET_X86 SUBTARGET(X86) +#else +#define SUBTARGET_X86 +#endif + +#ifdef LLVM_COMPONENT_ARM +#define SUBTARGET_ARM SUBTARGET(ARM) +#else +#define SUBTARGET_ARM +#endif + +#ifdef LLVM_COMPONENT_AARCH64 +#define SUBTARGET_AARCH64 SUBTARGET(AArch64) +#else +#define SUBTARGET_AARCH64 +#endif + +#ifdef LLVM_COMPONENT_AVR +#define SUBTARGET_AVR SUBTARGET(AVR) +#else +#define SUBTARGET_AVR +#endif + +#ifdef LLVM_COMPONENT_M68k +#define SUBTARGET_M68K SUBTARGET(M68k) +#else +#define SUBTARGET_M68K +#endif + +#ifdef LLVM_COMPONENT_CSKY +#define SUBTARGET_CSKY SUBTARGET(CSKY) +#else +#define SUBTARGET_CSKY +#endif + +#ifdef LLVM_COMPONENT_MIPS +#define SUBTARGET_MIPS SUBTARGET(Mips) +#else +#define SUBTARGET_MIPS +#endif + +#ifdef LLVM_COMPONENT_POWERPC +#define SUBTARGET_PPC SUBTARGET(PPC) +#else +#define SUBTARGET_PPC +#endif + +#ifdef LLVM_COMPONENT_SYSTEMZ +#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) +#else +#define SUBTARGET_SYSTEMZ +#endif + +#ifdef LLVM_COMPONENT_MSP430 +#define SUBTARGET_MSP430 SUBTARGET(MSP430) +#else +#define SUBTARGET_MSP430 +#endif + +#ifdef LLVM_COMPONENT_RISCV +#define SUBTARGET_RISCV SUBTARGET(RISCV) +#else +#define SUBTARGET_RISCV +#endif + +#ifdef LLVM_COMPONENT_SPARC +#define SUBTARGET_SPARC SUBTARGET(Sparc) +#else +#define SUBTARGET_SPARC +#endif + +#ifdef LLVM_COMPONENT_HEXAGON +#define SUBTARGET_HEXAGON SUBTARGET(Hexagon) +#else +#define SUBTARGET_HEXAGON +#endif + +#ifdef LLVM_COMPONENT_LOONGARCH +#define SUBTARGET_LOONGARCH SUBTARGET(LoongArch) +#else +#define SUBTARGET_LOONGARCH +#endif + +#define GEN_SUBTARGETS \ + SUBTARGET_X86 \ + SUBTARGET_ARM \ + SUBTARGET_AARCH64 \ + SUBTARGET_AVR \ + SUBTARGET_M68K \ + SUBTARGET_CSKY \ + SUBTARGET_MIPS \ + SUBTARGET_PPC \ + SUBTARGET_SYSTEMZ \ + SUBTARGET_MSP430 \ + SUBTARGET_SPARC \ + SUBTARGET_HEXAGON \ + SUBTARGET_RISCV \ + SUBTARGET_LOONGARCH \ + +#define SUBTARGET(x) \ + namespace llvm { \ + extern const SubtargetFeatureKV x##FeatureKV[]; \ + extern const SubtargetFeatureKV x##SubTypeKV[]; \ + } + +GEN_SUBTARGETS +#undef SUBTARGET + +extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, + const char *Feature) { + TargetMachine *Target = unwrap(TM); + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + return MCInfo->checkFeatures(std::string("+") + Feature); +} + +enum class LLVMRustCodeModel { + Tiny, + Small, + Kernel, + Medium, + Large, + None, +}; + +#if LLVM_VERSION_LT(16, 0) +static Optional +#else +static std::optional +#endif +fromRust(LLVMRustCodeModel Model) { + switch (Model) { + case LLVMRustCodeModel::Tiny: + return CodeModel::Tiny; + case LLVMRustCodeModel::Small: + return CodeModel::Small; + case LLVMRustCodeModel::Kernel: + return CodeModel::Kernel; + case LLVMRustCodeModel::Medium: + return CodeModel::Medium; + case LLVMRustCodeModel::Large: + return CodeModel::Large; + case LLVMRustCodeModel::None: +#if LLVM_VERSION_LT(16, 0) + return None; +#else + return std::nullopt; +#endif + default: + report_fatal_error("Bad CodeModel."); + } +} + +enum class LLVMRustCodeGenOptLevel { + None, + Less, + Default, + Aggressive, +}; + +#if LLVM_VERSION_GE(18, 0) + using CodeGenOptLevelEnum = llvm::CodeGenOptLevel; +#else + using CodeGenOptLevelEnum = llvm::CodeGenOpt::Level; +#endif + +static CodeGenOptLevelEnum fromRust(LLVMRustCodeGenOptLevel Level) { + switch (Level) { + case LLVMRustCodeGenOptLevel::None: + return CodeGenOptLevelEnum::None; + case LLVMRustCodeGenOptLevel::Less: + return CodeGenOptLevelEnum::Less; + case LLVMRustCodeGenOptLevel::Default: + return CodeGenOptLevelEnum::Default; + case LLVMRustCodeGenOptLevel::Aggressive: + return CodeGenOptLevelEnum::Aggressive; + default: + report_fatal_error("Bad CodeGenOptLevel."); + } +} + +enum class LLVMRustPassBuilderOptLevel { + O0, + O1, + O2, + O3, + Os, + Oz, +}; + +static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { + switch (Level) { + case LLVMRustPassBuilderOptLevel::O0: + return OptimizationLevel::O0; + case LLVMRustPassBuilderOptLevel::O1: + return OptimizationLevel::O1; + case LLVMRustPassBuilderOptLevel::O2: + return OptimizationLevel::O2; + case LLVMRustPassBuilderOptLevel::O3: + return OptimizationLevel::O3; + case LLVMRustPassBuilderOptLevel::Os: + return OptimizationLevel::Os; + case LLVMRustPassBuilderOptLevel::Oz: + return OptimizationLevel::Oz; + default: + report_fatal_error("Bad PassBuilderOptLevel."); + } +} + +enum class LLVMRustRelocModel { + Static, + PIC, + DynamicNoPic, + ROPI, + RWPI, + ROPIRWPI, +}; + +static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { + switch (RustReloc) { + case LLVMRustRelocModel::Static: + return Reloc::Static; + case LLVMRustRelocModel::PIC: + return Reloc::PIC_; + case LLVMRustRelocModel::DynamicNoPic: + return Reloc::DynamicNoPIC; + case LLVMRustRelocModel::ROPI: + return Reloc::ROPI; + case LLVMRustRelocModel::RWPI: + return Reloc::RWPI; + case LLVMRustRelocModel::ROPIRWPI: + return Reloc::ROPI_RWPI; + } + report_fatal_error("Bad RelocModel."); +} + +/// getLongestEntryLength - Return the length of the longest entry in the table. +template +static size_t getLongestEntryLength(ArrayRef Table) { + size_t MaxLen = 0; + for (auto &I : Table) + MaxLen = std::max(MaxLen, std::strlen(I.Key)); + return MaxLen; +} + +using PrintBackendInfo = void(void*, const char* Data, size_t Len); + +extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM, + const char* TargetCPU, + PrintBackendInfo Print, + void* Out) { + const TargetMachine *Target = unwrap(TM); + const Triple::ArchType HostArch = Triple(sys::getDefaultTargetTriple()).getArch(); + const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); + + std::ostringstream Buf; + +#if LLVM_VERSION_GE(17, 0) + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + const ArrayRef CPUTable = MCInfo->getAllProcessorDescriptions(); +#else + Buf << "Full target CPU help is not supported by this LLVM version.\n\n"; + SubtargetSubTypeKV TargetCPUKV = { TargetCPU, {{}}, {{}} }; + const ArrayRef CPUTable = TargetCPUKV; +#endif + unsigned MaxCPULen = getLongestEntryLength(CPUTable); + + Buf << "Available CPUs for this target:\n"; + // Don't print the "native" entry when the user specifies --target with a + // different arch since that could be wrong or misleading. + if (HostArch == TargetArch) { + MaxCPULen = std::max(MaxCPULen, (unsigned) std::strlen("native")); + const StringRef HostCPU = sys::getHostCPUName(); + Buf << " " << std::left << std::setw(MaxCPULen) << "native" + << " - Select the CPU of the current host " + "(currently " << HostCPU.str() << ").\n"; + } + for (auto &CPU : CPUTable) { + // Compare cpu against current target to label the default + if (strcmp(CPU.Key, TargetCPU) == 0) { + Buf << " " << std::left << std::setw(MaxCPULen) << CPU.Key + << " - This is the default target CPU for the current build target " + "(currently " << Target->getTargetTriple().str() << ")."; + } + else { + Buf << " " << CPU.Key; + } + Buf << "\n"; + } + + const auto &BufString = Buf.str(); + Print(Out, BufString.data(), BufString.size()); +} + +extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { +#ifdef LLVM_RUSTLLVM + const TargetMachine *Target = unwrap(TM); + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + const ArrayRef FeatTable = MCInfo->getFeatureTable(); + return FeatTable.size(); +#else + return 0; +#endif +} + +extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, + const char** Feature, const char** Desc) { +#ifdef LLVM_RUSTLLVM + const TargetMachine *Target = unwrap(TM); + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + const ArrayRef FeatTable = MCInfo->getFeatureTable(); + const SubtargetFeatureKV Feat = FeatTable[Index]; + *Feature = Feat.Key; + *Desc = Feat.Desc; +#endif +} + +extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { + StringRef Name = sys::getHostCPUName(); + *len = Name.size(); + return Name.data(); +} + +extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( + const char *TripleStr, const char *CPU, const char *Feature, + const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, + LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, + bool FunctionSections, + bool DataSections, + bool UniqueSectionNames, + bool TrapUnreachable, + bool Singlethread, + bool AsmComments, + bool EmitStackSizeSection, + bool RelaxELFRelocations, + bool UseInitArray, + const char *SplitDwarfFile, + const char *OutputObjFile, + const char *DebugInfoCompression, + bool ForceEmulatedTls, + const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) { + + auto OptLevel = fromRust(RustOptLevel); + auto RM = fromRust(RustReloc); + auto CM = fromRust(RustCM); + + std::string Error; + Triple Trip(Triple::normalize(TripleStr)); + const llvm::Target *TheTarget = + TargetRegistry::lookupTarget(Trip.getTriple(), Error); + if (TheTarget == nullptr) { + LLVMRustSetLastError(Error.c_str()); + return nullptr; + } + + TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Trip); + + Options.FloatABIType = FloatABI::Default; + if (UseSoftFloat) { + Options.FloatABIType = FloatABI::Soft; + } + Options.DataSections = DataSections; + Options.FunctionSections = FunctionSections; + Options.UniqueSectionNames = UniqueSectionNames; + Options.MCOptions.AsmVerbose = AsmComments; + Options.MCOptions.PreserveAsmComments = AsmComments; + Options.MCOptions.ABIName = ABIStr; + if (SplitDwarfFile) { + Options.MCOptions.SplitDwarfFile = SplitDwarfFile; + } + if (OutputObjFile) { + Options.ObjectFilenameForDebug = OutputObjFile; + } +#if LLVM_VERSION_GE(16, 0) + if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) { + Options.CompressDebugSections = DebugCompressionType::Zlib; + } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) { + Options.CompressDebugSections = DebugCompressionType::Zstd; + } else if (!strcmp("none", DebugInfoCompression)) { + Options.CompressDebugSections = DebugCompressionType::None; + } +#endif + + Options.RelaxELFRelocations = RelaxELFRelocations; + Options.UseInitArray = UseInitArray; + +#if LLVM_VERSION_LT(17, 0) + if (ForceEmulatedTls) { + Options.ExplicitEmulatedTLS = true; + Options.EmulatedTLS = true; + } +#else + Options.EmulatedTLS = ForceEmulatedTls || Trip.hasDefaultEmulatedTLS(); +#endif + + if (TrapUnreachable) { + // Tell LLVM to codegen `unreachable` into an explicit trap instruction. + // This limits the extent of possible undefined behavior in some cases, as + // it prevents control flow from "falling through" into whatever code + // happens to be laid out next in memory. + Options.TrapUnreachable = true; + } + + if (Singlethread) { + Options.ThreadModel = ThreadModel::Single; + } + + Options.EmitStackSizeSection = EmitStackSizeSection; + + + if (ArgsCstrBuff != nullptr) + { + int buffer_offset = 0; + assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); + + const size_t arg0_len = std::strlen(ArgsCstrBuff); + char* arg0 = new char[arg0_len + 1]; + memcpy(arg0, ArgsCstrBuff, arg0_len); + arg0[arg0_len] = '\0'; + buffer_offset += arg0_len + 1; + + const int num_cmd_arg_strings = + std::count(&ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0'); + + std::string* cmd_arg_strings = new std::string[num_cmd_arg_strings]; + for (int i = 0; i < num_cmd_arg_strings; ++i) + { + assert(buffer_offset < ArgsCstrBuffLen); + const int len = std::strlen(ArgsCstrBuff + buffer_offset); + cmd_arg_strings[i] = std::string(&ArgsCstrBuff[buffer_offset], len); + buffer_offset += len + 1; + } + + assert(buffer_offset == ArgsCstrBuffLen); + + Options.MCOptions.Argv0 = arg0; + Options.MCOptions.CommandLineArgs = + llvm::ArrayRef(cmd_arg_strings, num_cmd_arg_strings); + } + + TargetMachine *TM = TheTarget->createTargetMachine( + Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); + return wrap(TM); +} + +extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { + + MCTargetOptions& MCOptions = unwrap(TM)->Options.MCOptions; + delete[] MCOptions.Argv0; + delete[] MCOptions.CommandLineArgs.data(); + + delete unwrap(TM); +} + +// Unfortunately, the LLVM C API doesn't provide a way to create the +// TargetLibraryInfo pass, so we use this method to do so. +extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, + bool DisableSimplifyLibCalls) { + Triple TargetTriple(unwrap(M)->getTargetTriple()); + TargetLibraryInfoImpl TLII(TargetTriple); + if (DisableSimplifyLibCalls) + TLII.disableAllFunctions(); + unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); +} + +extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { + // Initializing the command-line options more than once is not allowed. So, + // check if they've already been initialized. (This could happen if we're + // being called from rustpkg, for example). If the arguments change, then + // that's just kinda unfortunate. + static bool Initialized = false; + if (Initialized) + return; + Initialized = true; + cl::ParseCommandLineOptions(Argc, Argv); +} + +enum class LLVMRustFileType { + AssemblyFile, + ObjectFile, +}; + +static CodeGenFileType fromRust(LLVMRustFileType Type) { + switch (Type) { + case LLVMRustFileType::AssemblyFile: +#if LLVM_VERSION_GE(18, 0) + return CodeGenFileType::AssemblyFile; +#else + return CGFT_AssemblyFile; +#endif + case LLVMRustFileType::ObjectFile: +#if LLVM_VERSION_GE(18, 0) + return CodeGenFileType::ObjectFile; +#else + return CGFT_ObjectFile; +#endif + default: + report_fatal_error("Bad FileType."); + } +} + +extern "C" LLVMRustResult +LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, + LLVMModuleRef M, const char *Path, const char *DwoPath, + LLVMRustFileType RustFileType) { + llvm::legacy::PassManager *PM = unwrap(PMR); + auto FileType = fromRust(RustFileType); + + std::string ErrorInfo; + std::error_code EC; + raw_fd_ostream OS(Path, EC, sys::fs::OF_None); + if (EC) + ErrorInfo = EC.message(); + if (ErrorInfo != "") { + LLVMRustSetLastError(ErrorInfo.c_str()); + return LLVMRustResult::Failure; + } + + buffer_ostream BOS(OS); + if (DwoPath) { + raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None); + EC.clear(); + if (EC) + ErrorInfo = EC.message(); + if (ErrorInfo != "") { + LLVMRustSetLastError(ErrorInfo.c_str()); + return LLVMRustResult::Failure; + } + buffer_ostream DBOS(DOS); + unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); + PM->run(*unwrap(M)); + } else { + unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); + PM->run(*unwrap(M)); + } + + // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output + // stream (OS), so the only real safe place to delete this is here? Don't we + // wish this was written in Rust? + LLVMDisposePassManager(PMR); + return LLVMRustResult::Success; +} + +extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler + const char*, // pass name + const char*); // IR name +extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler + +std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { + if (const auto *Cast = any_cast(&WrappedIr)) + return (*Cast)->getName().str(); + if (const auto *Cast = any_cast(&WrappedIr)) + return (*Cast)->getName().str(); + if (const auto *Cast = any_cast(&WrappedIr)) + return (*Cast)->getName().str(); + if (const auto *Cast = any_cast(&WrappedIr)) + return (*Cast)->getName(); + return ""; +} + + +void LLVMSelfProfileInitializeCallbacks( + PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, + LLVMRustSelfProfileBeforePassCallback BeforePassCallback, + LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { + PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( + StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + }); + + PIC.registerAfterPassCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, + const PreservedAnalyses &Preserved) { + AfterPassCallback(LlvmSelfProfiler); + }); + + PIC.registerAfterPassInvalidatedCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { + AfterPassCallback(LlvmSelfProfiler); + }); + + PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( + StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + }); + + PIC.registerAfterAnalysisCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { + AfterPassCallback(LlvmSelfProfiler); + }); +} + +enum class LLVMRustOptStage { + PreLinkNoLTO, + PreLinkThinLTO, + PreLinkFatLTO, + ThinLTO, + FatLTO, +}; + +struct LLVMRustSanitizerOptions { + bool SanitizeAddress; + bool SanitizeAddressRecover; + bool SanitizeCFI; + bool SanitizeKCFI; + bool SanitizeMemory; + bool SanitizeMemoryRecover; + int SanitizeMemoryTrackOrigins; + bool SanitizeThread; + bool SanitizeHWAddress; + bool SanitizeHWAddressRecover; + bool SanitizeKernelAddress; + bool SanitizeKernelAddressRecover; +}; + +extern "C" LLVMRustResult +LLVMRustOptimize( + LLVMModuleRef ModuleRef, + LLVMTargetMachineRef TMRef, + LLVMRustPassBuilderOptLevel OptLevelRust, + LLVMRustOptStage OptStage, + bool IsLinkerPluginLTO, + bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, + bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, + bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, + LLVMRustSanitizerOptions *SanitizerOptions, + const char *PGOGenPath, const char *PGOUsePath, + bool InstrumentCoverage, const char *InstrProfileOutput, + bool InstrumentGCOV, + const char *PGOSampleUsePath, bool DebugInfoForProfiling, + void* LlvmSelfProfiler, + LLVMRustSelfProfileBeforePassCallback BeforePassCallback, + LLVMRustSelfProfileAfterPassCallback AfterPassCallback, + const char *ExtraPasses, size_t ExtraPassesLen, + const char *LLVMPlugins, size_t LLVMPluginsLen) { + Module *TheModule = unwrap(ModuleRef); + TargetMachine *TM = unwrap(TMRef); + OptimizationLevel OptLevel = fromRust(OptLevelRust); + + + PipelineTuningOptions PTO; + PTO.LoopUnrolling = UnrollLoops; + PTO.LoopInterleaving = UnrollLoops; + PTO.LoopVectorization = LoopVectorize; + PTO.SLPVectorization = SLPVectorize; + PTO.MergeFunctions = MergeFunctions; + + // FIXME: We may want to expose this as an option. + bool DebugPassManager = false; + + PassInstrumentationCallbacks PIC; +#if LLVM_VERSION_LT(16, 0) + StandardInstrumentations SI(DebugPassManager); +#else + StandardInstrumentations SI(TheModule->getContext(), DebugPassManager); +#endif + SI.registerCallbacks(PIC); + + if (LlvmSelfProfiler){ + LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); + } + +#if LLVM_VERSION_LT(16, 0) + Optional PGOOpt; +#else + std::optional PGOOpt; +#endif +#if LLVM_VERSION_GE(17, 0) + auto FS = vfs::getRealFileSystem(); +#endif + if (PGOGenPath) { + assert(!PGOUsePath && !PGOSampleUsePath); + PGOOpt = PGOOptions(PGOGenPath, "", "", +#if LLVM_VERSION_GE(17, 0) + "", + FS, +#endif + PGOOptions::IRInstr, PGOOptions::NoCSAction, + DebugInfoForProfiling); + } else if (PGOUsePath) { + assert(!PGOSampleUsePath); + PGOOpt = PGOOptions(PGOUsePath, "", "", +#if LLVM_VERSION_GE(17, 0) + "", + FS, +#endif + PGOOptions::IRUse, PGOOptions::NoCSAction, + DebugInfoForProfiling); + } else if (PGOSampleUsePath) { + PGOOpt = PGOOptions(PGOSampleUsePath, "", "", +#if LLVM_VERSION_GE(17, 0) + "", + FS, +#endif + PGOOptions::SampleUse, PGOOptions::NoCSAction, + DebugInfoForProfiling); + } else if (DebugInfoForProfiling) { + PGOOpt = PGOOptions("", "", "", +#if LLVM_VERSION_GE(17, 0) + "", + FS, +#endif + PGOOptions::NoAction, PGOOptions::NoCSAction, + DebugInfoForProfiling); + } + + PassBuilder PB(TM, PTO, PGOOpt, &PIC); + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + + if (LLVMPluginsLen) { + auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); + SmallVector Plugins; + PluginsStr.split(Plugins, ',', -1, false); + for (auto PluginPath: Plugins) { + auto Plugin = PassPlugin::Load(PluginPath.str()); + if (!Plugin) { + LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); + return LLVMRustResult::Failure; + } + Plugin->registerPassBuilderCallbacks(PB); + } + } + + FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); + + Triple TargetTriple(TheModule->getTargetTriple()); + std::unique_ptr TLII(new TargetLibraryInfoImpl(TargetTriple)); + if (DisableSimplifyLibCalls) + TLII->disableAllFunctions(); + FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); + + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + // We manually collect pipeline callbacks so we can apply them at O0, where the + // PassBuilder does not create a pipeline. + std::vector> + PipelineStartEPCallbacks; + std::vector> + OptimizerLastEPCallbacks; + + if (!IsLinkerPluginLTO + && SanitizerOptions && SanitizerOptions->SanitizeCFI + && !NoPrepopulatePasses) { + PipelineStartEPCallbacks.push_back( + [](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr, + /*ImportSummary=*/nullptr, + /*DropTypeTests=*/false)); + } + ); + } + + if (VerifyIR) { + PipelineStartEPCallbacks.push_back( + [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(VerifierPass()); + } + ); + } + + if (InstrumentGCOV) { + PipelineStartEPCallbacks.push_back( + [](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); + } + ); + } + + if (InstrumentCoverage) { + PipelineStartEPCallbacks.push_back( + [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) { + InstrProfOptions Options; + if (InstrProfileOutput) { + Options.InstrProfileOutput = InstrProfileOutput; + } + // cargo run tests in multhreading mode by default + // so use atomics for coverage counters + Options.Atomic = true; + MPM.addPass(InstrProfiling(Options, false)); + } + ); + } + + if (SanitizerOptions) { + if (SanitizerOptions->SanitizeMemory) { + MemorySanitizerOptions Options( + SanitizerOptions->SanitizeMemoryTrackOrigins, + SanitizerOptions->SanitizeMemoryRecover, + /*CompileKernel=*/false, + /*EagerChecks=*/true); + OptimizerLastEPCallbacks.push_back( + [Options](ModulePassManager &MPM, OptimizationLevel Level) { +#if LLVM_VERSION_LT(16, 0) + MPM.addPass(ModuleMemorySanitizerPass(Options)); + MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); +#else + MPM.addPass(MemorySanitizerPass(Options)); +#endif + } + ); + } + + if (SanitizerOptions->SanitizeThread) { + OptimizerLastEPCallbacks.push_back( + [](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(ModuleThreadSanitizerPass()); + MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); + } + ); + } + + if (SanitizerOptions->SanitizeAddress || SanitizerOptions->SanitizeKernelAddress) { + OptimizerLastEPCallbacks.push_back( + [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { + auto CompileKernel = SanitizerOptions->SanitizeKernelAddress; + AddressSanitizerOptions opts = AddressSanitizerOptions{ + CompileKernel, + SanitizerOptions->SanitizeAddressRecover + || SanitizerOptions->SanitizeKernelAddressRecover, + /*UseAfterScope=*/true, + AsanDetectStackUseAfterReturnMode::Runtime, + }; +#if LLVM_VERSION_LT(16, 0) + MPM.addPass(ModuleAddressSanitizerPass(opts)); +#else + MPM.addPass(AddressSanitizerPass(opts)); +#endif + } + ); + } + if (SanitizerOptions->SanitizeHWAddress) { + OptimizerLastEPCallbacks.push_back( + [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { + HWAddressSanitizerOptions opts( + /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, + /*DisableOptimization=*/false); + MPM.addPass(HWAddressSanitizerPass(opts)); + } + ); + } + } + + ModulePassManager MPM; + bool NeedThinLTOBufferPasses = UseThinLTOBuffers; + if (!NoPrepopulatePasses) { + // The pre-link pipelines don't support O0 and require using buildO0DefaultPipeline() instead. + // At the same time, the LTO pipelines do support O0 and using them is required. + bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; + if (OptLevel == OptimizationLevel::O0 && !IsLTO) { + for (const auto &C : PipelineStartEPCallbacks) + PB.registerPipelineStartEPCallback(C); + for (const auto &C : OptimizerLastEPCallbacks) + PB.registerOptimizerLastEPCallback(C); + + // Pass false as we manually schedule ThinLTOBufferPasses below. + MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); + } else { + for (const auto &C : PipelineStartEPCallbacks) + PB.registerPipelineStartEPCallback(C); + if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { + for (const auto &C : OptimizerLastEPCallbacks) + PB.registerOptimizerLastEPCallback(C); + } + + switch (OptStage) { + case LLVMRustOptStage::PreLinkNoLTO: + MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); + break; + case LLVMRustOptStage::PreLinkThinLTO: + MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); + // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback + // passes may still run afterwards. This means we need to run the buffer passes again. + // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks + // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks. + if (OptimizerLastEPCallbacks.empty()) + NeedThinLTOBufferPasses = false; + for (const auto &C : OptimizerLastEPCallbacks) + C(MPM, OptLevel); + break; + case LLVMRustOptStage::PreLinkFatLTO: + MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); + NeedThinLTOBufferPasses = false; + break; + case LLVMRustOptStage::ThinLTO: + // FIXME: Does it make sense to pass the ModuleSummaryIndex? + // It only seems to be needed for C++ specific optimizations. + MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); + break; + case LLVMRustOptStage::FatLTO: + MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); + break; + } + } + } else { + // We're not building any of the default pipelines but we still want to + // add the verifier, instrumentation, etc passes if they were requested + for (const auto &C : PipelineStartEPCallbacks) + C(MPM, OptLevel); + for (const auto &C : OptimizerLastEPCallbacks) + C(MPM, OptLevel); + } + + if (ExtraPassesLen) { + if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { + std::string ErrMsg = toString(std::move(Err)); + LLVMRustSetLastError(ErrMsg.c_str()); + return LLVMRustResult::Failure; + } + } + + if (NeedThinLTOBufferPasses) { + MPM.addPass(CanonicalizeAliasesPass()); + MPM.addPass(NameAnonGlobalPass()); + } + + // Upgrade all calls to old intrinsics first. + for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) + UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove + + MPM.run(*TheModule, MAM); + return LLVMRustResult::Success; +} + +// Callback to demangle function name +// Parameters: +// * name to be demangled +// * name len +// * output buffer +// * output buffer len +// Returns len of demangled string, or 0 if demangle failed. +typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); + + +namespace { + +class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { + DemangleFn Demangle; + std::vector Buf; + +public: + RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} + + // Return empty string if demangle failed + // or if name does not need to be demangled + StringRef CallDemangle(StringRef name) { + if (!Demangle) { + return StringRef(); + } + + if (Buf.size() < name.size() * 2) { + // Semangled name usually shorter than mangled, + // but allocate twice as much memory just in case + Buf.resize(name.size() * 2); + } + + auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); + if (!R) { + // Demangle failed. + return StringRef(); + } + + auto Demangled = StringRef(Buf.data(), R); + if (Demangled == name) { + // Do not print anything if demangled name is equal to mangled. + return StringRef(); + } + + return Demangled; + } + + void emitFunctionAnnot(const Function *F, + formatted_raw_ostream &OS) override { + StringRef Demangled = CallDemangle(F->getName()); + if (Demangled.empty()) { + return; + } + + OS << "; " << Demangled << "\n"; + } + + void emitInstructionAnnot(const Instruction *I, + formatted_raw_ostream &OS) override { + const char *Name; + const Value *Value; + if (const CallInst *CI = dyn_cast(I)) { + Name = "call"; + Value = CI->getCalledOperand(); + } else if (const InvokeInst* II = dyn_cast(I)) { + Name = "invoke"; + Value = II->getCalledOperand(); + } else { + // Could demangle more operations, e. g. + // `store %place, @function`. + return; + } + + if (!Value->hasName()) { + return; + } + + StringRef Demangled = CallDemangle(Value->getName()); + if (Demangled.empty()) { + return; + } + + OS << "; " << Name << " " << Demangled << "\n"; + } +}; + +} // namespace + +extern "C" LLVMRustResult +LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { + std::string ErrorInfo; + std::error_code EC; + raw_fd_ostream OS(Path, EC, sys::fs::OF_None); + if (EC) + ErrorInfo = EC.message(); + if (ErrorInfo != "") { + LLVMRustSetLastError(ErrorInfo.c_str()); + return LLVMRustResult::Failure; + } + + RustAssemblyAnnotationWriter AAW(Demangle); + formatted_raw_ostream FOS(OS); + unwrap(M)->print(FOS, &AAW); + + return LLVMRustResult::Success; +} + +extern "C" void LLVMRustPrintPasses() { + PassBuilder PB; + PB.printPassNames(outs()); +} + +extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, + size_t Len) { + auto PreserveFunctions = [=](const GlobalValue &GV) { + // Preserve LLVM-injected, ASAN-related symbols. + // See also https://github.com/rust-lang/rust/issues/113404. + if (GV.getName() == "___asan_globals_registered") { + return true; + } + + // Preserve symbols exported from Rust modules. + for (size_t I = 0; I < Len; I++) { + if (GV.getName() == Symbols[I]) { + return true; + } + } + return false; + }; + + internalizeModule(*unwrap(M), PreserveFunctions); +} + +extern "C" void +LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, + LLVMTargetMachineRef TMR) { + TargetMachine *Target = unwrap(TMR); + unwrap(Module)->setDataLayout(Target->createDataLayout()); +} + +extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { + unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); +} + +extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { + unwrap(M)->setPIELevel(PIELevel::Level::Large); +} + +extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, + LLVMRustCodeModel Model) { + auto CM = fromRust(Model); + if (!CM) + return; + unwrap(M)->setCodeModel(*CM); +} + +// Here you'll find an implementation of ThinLTO as used by the Rust compiler +// right now. This ThinLTO support is only enabled on "recent ish" versions of +// LLVM, and otherwise it's just blanket rejected from other compilers. +// +// Most of this implementation is straight copied from LLVM. At the time of +// this writing it wasn't *quite* suitable to reuse more code from upstream +// for our purposes, but we should strive to upstream this support once it's +// ready to go! I figure we may want a bit of testing locally first before +// sending this upstream to LLVM. I hear though they're quite eager to receive +// feedback like this! +// +// If you're reading this code and wondering "what in the world" or you're +// working "good lord by LLVM upgrade is *still* failing due to these bindings" +// then fear not! (ok maybe fear a little). All code here is mostly based +// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. +// +// You'll find that the general layout here roughly corresponds to the `run` +// method in that file as well as `ProcessThinLTOModule`. Functions are +// specifically commented below as well, but if you're updating this code +// or otherwise trying to understand it, the LLVM source will be useful in +// interpreting the mysteries within. +// +// Otherwise I'll apologize in advance, it probably requires a relatively +// significant investment on your part to "truly understand" what's going on +// here. Not saying I do myself, but it took me awhile staring at LLVM's source +// and various online resources about ThinLTO to make heads or tails of all +// this. + +// This is a shared data structure which *must* be threadsafe to share +// read-only amongst threads. This also corresponds basically to the arguments +// of the `ProcessThinLTOModule` function in the LLVM source. +struct LLVMRustThinLTOData { + // The combined index that is the global analysis over all modules we're + // performing ThinLTO for. This is mostly managed by LLVM. + ModuleSummaryIndex Index; + + // All modules we may look at, stored as in-memory serialized versions. This + // is later used when inlining to ensure we can extract any module to inline + // from. + StringMap ModuleMap; + + // A set that we manage of everything we *don't* want internalized. Note that + // this includes all transitive references right now as well, but it may not + // always! + DenseSet GUIDPreservedSymbols; + + // Not 100% sure what these are, but they impact what's internalized and + // what's inlined across modules, I believe. +#if LLVM_VERSION_GE(18, 0) + DenseMap ImportLists; + DenseMap ExportLists; + DenseMap ModuleToDefinedGVSummaries; +#else + StringMap ImportLists; + StringMap ExportLists; + StringMap ModuleToDefinedGVSummaries; +#endif + StringMap> ResolvedODR; + + LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} +}; + +// Just an argument to the `LLVMRustCreateThinLTOData` function below. +struct LLVMRustThinLTOModule { + const char *identifier; + const char *data; + size_t len; +}; + +// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it +// does. +static const GlobalValueSummary * +getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { + auto StrongDefForLinker = llvm::find_if( + GVSummaryList, [](const std::unique_ptr &Summary) { + auto Linkage = Summary->linkage(); + return !GlobalValue::isAvailableExternallyLinkage(Linkage) && + !GlobalValue::isWeakForLinker(Linkage); + }); + if (StrongDefForLinker != GVSummaryList.end()) + return StrongDefForLinker->get(); + + auto FirstDefForLinker = llvm::find_if( + GVSummaryList, [](const std::unique_ptr &Summary) { + auto Linkage = Summary->linkage(); + return !GlobalValue::isAvailableExternallyLinkage(Linkage); + }); + if (FirstDefForLinker == GVSummaryList.end()) + return nullptr; + return FirstDefForLinker->get(); +} + +// The main entry point for creating the global ThinLTO analysis. The structure +// here is basically the same as before threads are spawned in the `run` +// function of `lib/LTO/ThinLTOCodeGenerator.cpp`. +extern "C" LLVMRustThinLTOData* +LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, + int num_modules, + const char **preserved_symbols, + int num_symbols) { + auto Ret = std::make_unique(); + + // Load each module's summary and merge it into one combined index + for (int i = 0; i < num_modules; i++) { + auto module = &modules[i]; + StringRef buffer(module->data, module->len); + MemoryBufferRef mem_buffer(buffer, module->identifier); + + Ret->ModuleMap[module->identifier] = mem_buffer; + +#if LLVM_VERSION_GE(18, 0) + if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index)) { +#else + if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { +#endif + LLVMRustSetLastError(toString(std::move(Err)).c_str()); + return nullptr; + } + } + + // Collect for each module the list of function it defines (GUID -> Summary) + Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); + + // Convert the preserved symbols set from string to GUID, this is then needed + // for internalization. + for (int i = 0; i < num_symbols; i++) { + auto GUID = GlobalValue::getGUID(preserved_symbols[i]); + Ret->GUIDPreservedSymbols.insert(GUID); + } + + // Collect the import/export lists for all modules from the call-graph in the + // combined index + // + // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` + auto deadIsPrevailing = [&](GlobalValue::GUID G) { + return PrevailingType::Unknown; + }; + // We don't have a complete picture in our use of ThinLTO, just our immediate + // crate, so we need `ImportEnabled = false` to limit internalization. + // Otherwise, we sometimes lose `static` values -- see #60184. + computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, + deadIsPrevailing, /* ImportEnabled = */ false); + // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it + // impacts the caching. + // + // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this + // being lifted from `lib/LTO/LTO.cpp` as well + DenseMap PrevailingCopy; + for (auto &I : Ret->Index) { + if (I.second.SummaryList.size() > 1) + PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); + } + auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { + const auto &Prevailing = PrevailingCopy.find(GUID); + if (Prevailing == PrevailingCopy.end()) + return true; + return Prevailing->second == S; + }; + ComputeCrossModuleImport( + Ret->Index, + Ret->ModuleToDefinedGVSummaries, +#if LLVM_VERSION_GE(17, 0) + isPrevailing, +#endif + Ret->ImportLists, + Ret->ExportLists + ); + + auto recordNewLinkage = [&](StringRef ModuleIdentifier, + GlobalValue::GUID GUID, + GlobalValue::LinkageTypes NewLinkage) { + Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; + }; + + // Uses FromPrevailing visibility scheme which works for many binary + // formats. We probably could and should use ELF visibility scheme for many of + // our targets, however. + lto::Config conf; + thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, + Ret->GUIDPreservedSymbols); + + // Here we calculate an `ExportedGUIDs` set for use in the `isExported` + // callback below. This callback below will dictate the linkage for all + // summaries in the index, and we basically just only want to ensure that dead + // symbols are internalized. Otherwise everything that's already external + // linkage will stay as external, and internal will stay as internal. + std::set ExportedGUIDs; + for (auto &List : Ret->Index) { + for (auto &GVS: List.second.SummaryList) { + if (GlobalValue::isLocalLinkage(GVS->linkage())) + continue; + auto GUID = GVS->getOriginalName(); + if (GVS->flags().Live) + ExportedGUIDs.insert(GUID); + } + } + auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { + const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); + return (ExportList != Ret->ExportLists.end() && + ExportList->second.count(VI)) || + ExportedGUIDs.count(VI.getGUID()); + }; + thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); + + return Ret.release(); +} + +extern "C" void +LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { + delete Data; +} + +// Below are the various passes that happen *per module* when doing ThinLTO. +// +// In other words, these are the functions that are all run concurrently +// with one another, one per module. The passes here correspond to the analysis +// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the +// `ProcessThinLTOModule` function. Here they're split up into separate steps +// so rustc can save off the intermediate bytecode between each step. + +static bool +clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { + // When linking an ELF shared object, dso_local should be dropped. We + // conservatively do this for -fpic. + bool ClearDSOLocalOnDeclarations = + TM.getTargetTriple().isOSBinFormatELF() && + TM.getRelocationModel() != Reloc::Static && + Mod.getPIELevel() == PIELevel::Default; + return ClearDSOLocalOnDeclarations; +} + +extern "C" bool +LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, + LLVMTargetMachineRef TM) { + Module &Mod = *unwrap(M); + TargetMachine &Target = *unwrap(TM); + + bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); + bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); + + if (error) { + LLVMRustSetLastError("renameModuleForThinLTO failed"); + return false; + } + return true; +} + +extern "C" bool +LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { + Module &Mod = *unwrap(M); + const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); + thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); + return true; +} + +extern "C" bool +LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { + Module &Mod = *unwrap(M); + const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); + thinLTOInternalizeModule(Mod, DefinedGlobals); + return true; +} + +extern "C" bool +LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, + LLVMTargetMachineRef TM) { + Module &Mod = *unwrap(M); + TargetMachine &Target = *unwrap(TM); + + const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); + auto Loader = [&](StringRef Identifier) { + const auto &Memory = Data->ModuleMap.lookup(Identifier); + auto &Context = Mod.getContext(); + auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); + + if (!MOrErr) + return MOrErr; + + // The rest of this closure is a workaround for + // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports + // we accidentally import wasm custom sections into different modules, + // duplicating them by in the final output artifact. + // + // The issue is worked around here by manually removing the + // `wasm.custom_sections` named metadata node from any imported module. This + // we know isn't used by any optimization pass so there's no need for it to + // be imported. + // + // Note that the metadata is currently lazily loaded, so we materialize it + // here before looking up if there's metadata inside. The `FunctionImporter` + // will immediately materialize metadata anyway after an import, so this + // shouldn't be a perf hit. + if (Error Err = (*MOrErr)->materializeMetadata()) { + Expected> Ret(std::move(Err)); + return Ret; + } + + auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); + if (WasmCustomSections) + WasmCustomSections->eraseFromParent(); + + // `llvm.ident` named metadata also gets duplicated. + auto *llvmIdent = (*MOrErr)->getNamedMetadata("llvm.ident"); + if (llvmIdent) + llvmIdent->eraseFromParent(); + + return MOrErr; + }; + bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); + FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal); + Expected Result = Importer.importFunctions(Mod, ImportList); + if (!Result) { + LLVMRustSetLastError(toString(Result.takeError()).c_str()); + return false; + } + return true; +} + +// This struct and various functions are sort of a hack right now, but the +// problem is that we've got in-memory LLVM modules after we generate and +// optimize all codegen-units for one compilation in rustc. To be compatible +// with the LTO support above we need to serialize the modules plus their +// ThinLTO summary into memory. +// +// This structure is basically an owned version of a serialize module, with +// a ThinLTO summary attached. +struct LLVMRustThinLTOBuffer { + std::string data; +}; + +extern "C" LLVMRustThinLTOBuffer* +LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { + auto Ret = std::make_unique(); + { + raw_string_ostream OS(Ret->data); + { + if (is_thin) { + PassBuilder PB; + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + ModulePassManager MPM; + MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr)); + MPM.run(*unwrap(M), MAM); + } else { + WriteBitcodeToFile(*unwrap(M), OS); + } + } + } + return Ret.release(); +} + +extern "C" void +LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { + delete Buffer; +} + +extern "C" const void* +LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { + return Buffer->data.data(); +} + +extern "C" size_t +LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { + return Buffer->data.length(); +} + +// This is what we used to parse upstream bitcode for actual ThinLTO +// processing. We'll call this once per module optimized through ThinLTO, and +// it'll be called concurrently on many threads. +extern "C" LLVMModuleRef +LLVMRustParseBitcodeForLTO(LLVMContextRef Context, + const char *data, + size_t len, + const char *identifier) { + StringRef Data(data, len); + MemoryBufferRef Buffer(Data, identifier); + unwrap(Context)->enableDebugTypeODRUniquing(); + Expected> SrcOrError = + parseBitcodeFile(Buffer, *unwrap(Context)); + if (!SrcOrError) { + LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); + return nullptr; + } + return wrap(std::move(*SrcOrError).release()); +} + +// Find a section of an object file by name. Fail if the section is missing or +// empty. +extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, + size_t len, + const char *name, + size_t *out_len) { + *out_len = 0; + StringRef Data(data, len); + MemoryBufferRef Buffer(Data, ""); // The id is unused. + file_magic Type = identify_magic(Buffer.getBuffer()); + Expected> ObjFileOrError = + object::ObjectFile::createObjectFile(Buffer, Type); + if (!ObjFileOrError) { + LLVMRustSetLastError(toString(ObjFileOrError.takeError()).c_str()); + return nullptr; + } + for (const object::SectionRef &Sec : (*ObjFileOrError)->sections()) { + Expected Name = Sec.getName(); + if (Name && *Name == name) { + Expected SectionOrError = Sec.getContents(); + if (!SectionOrError) { + LLVMRustSetLastError(toString(SectionOrError.takeError()).c_str()); + return nullptr; + } + *out_len = SectionOrError->size(); + return SectionOrError->data(); + } + } + LLVMRustSetLastError("could not find requested section"); + return nullptr; +} + +// Computes the LTO cache key for the provided 'ModId' in the given 'Data', +// storing the result in 'KeyOut'. +// Currently, this cache key is a SHA-1 hash of anything that could affect +// the result of optimizing this module (e.g. module imports, exports, liveness +// of access globals, etc). +// The precise details are determined by LLVM in `computeLTOCacheKey`, which is +// used during the normal linker-plugin incremental thin-LTO process. +extern "C" void +LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) { + SmallString<40> Key; + llvm::lto::Config conf; + const auto &ImportList = Data->ImportLists.lookup(ModId); + const auto &ExportList = Data->ExportLists.lookup(ModId); + const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); + const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); + std::set CfiFunctionDefs; + std::set CfiFunctionDecls; + + // Based on the 'InProcessThinBackend' constructor in LLVM + for (auto &Name : Data->Index.cfiFunctionDefs()) + CfiFunctionDefs.insert( + GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); + for (auto &Name : Data->Index.cfiFunctionDecls()) + CfiFunctionDecls.insert( + GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); + + llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, + ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls + ); + + LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); +} diff --git a/compiler/rustc_llvm/llvm-wrapper/README b/compiler/rustc_llvm/llvm-wrapper/README new file mode 100644 index 00000000000..e1c6dd07d2b --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/README @@ -0,0 +1,16 @@ +This directory currently contains some LLVM support code. This will generally +be sent upstream to LLVM in time; for now it lives here. + +NOTE: the LLVM C++ ABI is subject to between-version breakage and must *never* +be exposed to Rust. To allow for easy auditing of that, all Rust-exposed types +must be typedef-ed as "LLVMXyz", or "LLVMRustXyz" if they were defined here. + +Functions that return a failure status and leave the error in +the LLVM last error should return an LLVMRustResult rather than an +int or anything to avoid confusion. + +When translating enums, add a single `Other` variant as the first +one to allow for new variants to be added. It should abort when used +as an input. + +All other types must not be typedef-ed as such. diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp new file mode 100644 index 00000000000..7ada2eff593 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -0,0 +1,2069 @@ +#include "LLVMWrapper.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DiagnosticHandler.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsARM.h" +#include "llvm/IR/LLVMRemarkStreamer.h" +#include "llvm/IR/Mangler.h" +#include "llvm/Remarks/RemarkStreamer.h" +#include "llvm/Remarks/RemarkSerializer.h" +#include "llvm/Remarks/RemarkFormat.h" +#include "llvm/Support/ToolOutputFile.h" +#if LLVM_VERSION_GE(16, 0) +#include "llvm/Support/ModRef.h" +#endif +#include "llvm/Object/Archive.h" +#include "llvm/Object/COFFImportFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Pass.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/Support/Signals.h" +#if LLVM_VERSION_LT(16, 0) +#include "llvm/ADT/Optional.h" +#endif + +#include + +//===----------------------------------------------------------------------=== +// +// This file defines alternate interfaces to core functions that are more +// readily callable by Rust's FFI. +// +//===----------------------------------------------------------------------=== + +using namespace llvm; +using namespace llvm::sys; +using namespace llvm::object; + +// LLVMAtomicOrdering is already an enum - don't create another +// one. +static AtomicOrdering fromRust(LLVMAtomicOrdering Ordering) { + switch (Ordering) { + case LLVMAtomicOrderingNotAtomic: + return AtomicOrdering::NotAtomic; + case LLVMAtomicOrderingUnordered: + return AtomicOrdering::Unordered; + case LLVMAtomicOrderingMonotonic: + return AtomicOrdering::Monotonic; + case LLVMAtomicOrderingAcquire: + return AtomicOrdering::Acquire; + case LLVMAtomicOrderingRelease: + return AtomicOrdering::Release; + case LLVMAtomicOrderingAcquireRelease: + return AtomicOrdering::AcquireRelease; + case LLVMAtomicOrderingSequentiallyConsistent: + return AtomicOrdering::SequentiallyConsistent; + } + + report_fatal_error("Invalid LLVMAtomicOrdering value!"); +} + +static LLVM_THREAD_LOCAL char *LastError; + +// Custom error handler for fatal LLVM errors. +// +// Notably it exits the process with code 101, unlike LLVM's default of 1. +static void FatalErrorHandler(void *UserData, + const char* Reason, + bool GenCrashDiag) { + // Do the same thing that the default error handler does. + std::cerr << "LLVM ERROR: " << Reason << std::endl; + + // Since this error handler exits the process, we have to run any cleanup that + // LLVM would run after handling the error. This might change with an LLVM + // upgrade. + sys::RunInterruptHandlers(); + + exit(101); +} + +extern "C" void LLVMRustInstallFatalErrorHandler() { + install_fatal_error_handler(FatalErrorHandler); +} + +extern "C" void LLVMRustDisableSystemDialogsOnCrash() { + sys::DisableSystemDialogsOnCrash(); +} + +extern "C" char *LLVMRustGetLastError(void) { + char *Ret = LastError; + LastError = nullptr; + return Ret; +} + +extern "C" void LLVMRustSetLastError(const char *Err) { + free((void *)LastError); + LastError = strdup(Err); +} + +extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames) { + auto ctx = new LLVMContext(); + ctx->setDiscardValueNames(shouldDiscardNames); + return wrap(ctx); +} + +extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, + const char *Triple) { + unwrap(M)->setTargetTriple(Triple::normalize(Triple)); +} + +extern "C" const char *LLVMRustPrintPassTimings(size_t *Len) { + std::string buf; + raw_string_ostream SS(buf); + TimerGroup::printAll(SS); + SS.flush(); + *Len = buf.length(); + char *CStr = (char *)malloc(*Len); + memcpy(CStr, buf.c_str(), *Len); + return CStr; +} + +extern "C" const char *LLVMRustPrintStatistics(size_t *Len) { + std::string buf; + raw_string_ostream SS(buf); + llvm::PrintStatistics(SS); + SS.flush(); + *Len = buf.length(); + char *CStr = (char *)malloc(*Len); + memcpy(CStr, buf.c_str(), *Len); + return CStr; +} + +extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, + size_t NameLen) { + return wrap(unwrap(M)->getNamedValue(StringRef(Name, NameLen))); +} + +enum class LLVMRustTailCallKind { + None, + Tail, + MustTail, + NoTail, +}; + +static CallInst::TailCallKind fromRust(LLVMRustTailCallKind Kind) { + switch (Kind) { + case LLVMRustTailCallKind::None: + return CallInst::TailCallKind::TCK_None; + case LLVMRustTailCallKind::Tail: + return CallInst::TailCallKind::TCK_Tail; + case LLVMRustTailCallKind::MustTail: + return CallInst::TailCallKind::TCK_MustTail; + case LLVMRustTailCallKind::NoTail: + return CallInst::TailCallKind::TCK_NoTail; + default: + report_fatal_error("bad CallInst::TailCallKind."); + } +} + +extern "C" void LLVMRustSetTailCallKind(LLVMValueRef Call, LLVMRustTailCallKind TCK) { + unwrap(Call)->setTailCallKind(fromRust(TCK)); +} + +extern "C" LLVMValueRef LLVMRustGetOrInsertFunction(LLVMModuleRef M, + const char *Name, + size_t NameLen, + LLVMTypeRef FunctionTy) { + return wrap(unwrap(M) + ->getOrInsertFunction(StringRef(Name, NameLen), + unwrap(FunctionTy)) + .getCallee() + ); +} + +extern "C" LLVMValueRef +LLVMRustGetOrInsertGlobal(LLVMModuleRef M, const char *Name, size_t NameLen, LLVMTypeRef Ty) { + Module *Mod = unwrap(M); + StringRef NameRef(Name, NameLen); + + // We don't use Module::getOrInsertGlobal because that returns a Constant*, + // which may either be the real GlobalVariable*, or a constant bitcast of it + // if our type doesn't match the original declaration. We always want the + // GlobalVariable* so we can access linkage, visibility, etc. + GlobalVariable *GV = Mod->getGlobalVariable(NameRef, true); + if (!GV) + GV = new GlobalVariable(*Mod, unwrap(Ty), false, + GlobalValue::ExternalLinkage, nullptr, NameRef); + return wrap(GV); +} + +extern "C" LLVMValueRef +LLVMRustInsertPrivateGlobal(LLVMModuleRef M, LLVMTypeRef Ty) { + return wrap(new GlobalVariable(*unwrap(M), + unwrap(Ty), + false, + GlobalValue::PrivateLinkage, + nullptr)); +} + +static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { + switch (Kind) { + case AlwaysInline: + return Attribute::AlwaysInline; + case ByVal: + return Attribute::ByVal; + case Cold: + return Attribute::Cold; + case InlineHint: + return Attribute::InlineHint; + case MinSize: + return Attribute::MinSize; + case Naked: + return Attribute::Naked; + case NoAlias: + return Attribute::NoAlias; + case NoCapture: + return Attribute::NoCapture; + case NoCfCheck: + return Attribute::NoCfCheck; + case NoInline: + return Attribute::NoInline; + case NonNull: + return Attribute::NonNull; + case NoRedZone: + return Attribute::NoRedZone; + case NoReturn: + return Attribute::NoReturn; + case NoUnwind: + return Attribute::NoUnwind; + case OptimizeForSize: + return Attribute::OptimizeForSize; + case ReadOnly: + return Attribute::ReadOnly; + case SExt: + return Attribute::SExt; + case StructRet: + return Attribute::StructRet; + case UWTable: + return Attribute::UWTable; + case ZExt: + return Attribute::ZExt; + case InReg: + return Attribute::InReg; + case SanitizeThread: + return Attribute::SanitizeThread; + case SanitizeAddress: + return Attribute::SanitizeAddress; + case SanitizeMemory: + return Attribute::SanitizeMemory; + case NonLazyBind: + return Attribute::NonLazyBind; + case OptimizeNone: + return Attribute::OptimizeNone; + case ReturnsTwice: + return Attribute::ReturnsTwice; + case ReadNone: + return Attribute::ReadNone; + case SanitizeHWAddress: + return Attribute::SanitizeHWAddress; + case WillReturn: + return Attribute::WillReturn; + case StackProtectReq: + return Attribute::StackProtectReq; + case StackProtectStrong: + return Attribute::StackProtectStrong; + case StackProtect: + return Attribute::StackProtect; + case NoUndef: + return Attribute::NoUndef; + case SanitizeMemTag: + return Attribute::SanitizeMemTag; + case ShadowCallStack: + return Attribute::ShadowCallStack; + case AllocSize: + return Attribute::AllocSize; + case AllocatedPointer: + return Attribute::AllocatedPointer; + case AllocAlign: + return Attribute::AllocAlign; + case SanitizeSafeStack: + return Attribute::SafeStack; + } + report_fatal_error("bad AttributeKind"); +} + +template static inline void AddAttributes(T *t, unsigned Index, + LLVMAttributeRef *Attrs, size_t AttrsLen) { + AttributeList PAL = t->getAttributes(); + AttrBuilder B(t->getContext()); + for (LLVMAttributeRef Attr : ArrayRef(Attrs, AttrsLen)) + B.addAttribute(unwrap(Attr)); + AttributeList PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B); + t->setAttributes(PALNew); +} + +extern "C" void LLVMRustAddFunctionAttributes(LLVMValueRef Fn, unsigned Index, + LLVMAttributeRef *Attrs, size_t AttrsLen) { + Function *F = unwrap(Fn); + AddAttributes(F, Index, Attrs, AttrsLen); +} + +extern "C" void LLVMRustAddCallSiteAttributes(LLVMValueRef Instr, unsigned Index, + LLVMAttributeRef *Attrs, size_t AttrsLen) { + CallBase *Call = unwrap(Instr); + AddAttributes(Call, Index, Attrs, AttrsLen); +} + +extern "C" LLVMAttributeRef LLVMRustCreateAttrNoValue(LLVMContextRef C, + LLVMRustAttribute RustAttr) { + return wrap(Attribute::get(*unwrap(C), fromRust(RustAttr))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateAlignmentAttr(LLVMContextRef C, + uint64_t Bytes) { + return wrap(Attribute::getWithAlignment(*unwrap(C), llvm::Align(Bytes))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableAttr(LLVMContextRef C, + uint64_t Bytes) { + return wrap(Attribute::getWithDereferenceableBytes(*unwrap(C), Bytes)); +} + +extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableOrNullAttr(LLVMContextRef C, + uint64_t Bytes) { + return wrap(Attribute::getWithDereferenceableOrNullBytes(*unwrap(C), Bytes)); +} + +extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C, LLVMTypeRef Ty) { + return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C, LLVMTypeRef Ty) { + return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateElementTypeAttr(LLVMContextRef C, LLVMTypeRef Ty) { + return wrap(Attribute::get(*unwrap(C), Attribute::ElementType, unwrap(Ty))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateUWTableAttr(LLVMContextRef C, bool Async) { + return wrap(Attribute::getWithUWTableKind( + *unwrap(C), Async ? UWTableKind::Async : UWTableKind::Sync)); +} + +extern "C" LLVMAttributeRef LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) { + return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, +#if LLVM_VERSION_LT(16, 0) + None +#else + std::nullopt +#endif + )); +} + +// These values **must** match ffi::AllocKindFlags. +// It _happens_ to match the LLVM values of llvm::AllocFnKind, +// but that's happenstance and we do explicit conversions before +// passing them to LLVM. +enum class LLVMRustAllocKindFlags : uint64_t { + Unknown = 0, + Alloc = 1, + Realloc = 1 << 1, + Free = 1 << 2, + Uninitialized = 1 << 3, + Zeroed = 1 << 4, + Aligned = 1 << 5, +}; + +static LLVMRustAllocKindFlags operator&(LLVMRustAllocKindFlags A, LLVMRustAllocKindFlags B) { + return static_cast(static_cast(A) & + static_cast(B)); +} + +static bool isSet(LLVMRustAllocKindFlags F) { return F != LLVMRustAllocKindFlags::Unknown; } + +static llvm::AllocFnKind allocKindFromRust(LLVMRustAllocKindFlags F) { + llvm::AllocFnKind AFK = llvm::AllocFnKind::Unknown; + if (isSet(F & LLVMRustAllocKindFlags::Alloc)) { + AFK |= llvm::AllocFnKind::Alloc; + } + if (isSet(F & LLVMRustAllocKindFlags::Realloc)) { + AFK |= llvm::AllocFnKind::Realloc; + } + if (isSet(F & LLVMRustAllocKindFlags::Free)) { + AFK |= llvm::AllocFnKind::Free; + } + if (isSet(F & LLVMRustAllocKindFlags::Uninitialized)) { + AFK |= llvm::AllocFnKind::Uninitialized; + } + if (isSet(F & LLVMRustAllocKindFlags::Zeroed)) { + AFK |= llvm::AllocFnKind::Zeroed; + } + if (isSet(F & LLVMRustAllocKindFlags::Aligned)) { + AFK |= llvm::AllocFnKind::Aligned; + } + return AFK; +} + +extern "C" LLVMAttributeRef LLVMRustCreateAllocKindAttr(LLVMContextRef C, uint64_t AllocKindArg) { + return wrap(Attribute::get(*unwrap(C), Attribute::AllocKind, + static_cast(allocKindFromRust(static_cast(AllocKindArg))))); +} + +// Simplified representation of `MemoryEffects` across the FFI boundary. +// +// Each variant corresponds to one of the static factory methods on `MemoryEffects`. +enum class LLVMRustMemoryEffects { + None, + ReadOnly, + InaccessibleMemOnly, +}; + +extern "C" LLVMAttributeRef LLVMRustCreateMemoryEffectsAttr(LLVMContextRef C, + LLVMRustMemoryEffects Effects) { +#if LLVM_VERSION_GE(16, 0) + switch (Effects) { + case LLVMRustMemoryEffects::None: + return wrap(Attribute::getWithMemoryEffects(*unwrap(C), MemoryEffects::none())); + case LLVMRustMemoryEffects::ReadOnly: + return wrap(Attribute::getWithMemoryEffects(*unwrap(C), MemoryEffects::readOnly())); + case LLVMRustMemoryEffects::InaccessibleMemOnly: + return wrap(Attribute::getWithMemoryEffects(*unwrap(C), + MemoryEffects::inaccessibleMemOnly())); + default: + report_fatal_error("bad MemoryEffects."); + } +#else + switch (Effects) { + case LLVMRustMemoryEffects::None: + return wrap(Attribute::get(*unwrap(C), Attribute::ReadNone)); + case LLVMRustMemoryEffects::ReadOnly: + return wrap(Attribute::get(*unwrap(C), Attribute::ReadOnly)); + case LLVMRustMemoryEffects::InaccessibleMemOnly: + return wrap(Attribute::get(*unwrap(C), Attribute::InaccessibleMemOnly)); + default: + report_fatal_error("bad MemoryEffects."); + } +#endif +} + +// Enable a fast-math flag +// +// https://llvm.org/docs/LangRef.html#fast-math-flags +extern "C" void LLVMRustSetFastMath(LLVMValueRef V) { + if (auto I = dyn_cast(unwrap(V))) { + I->setFast(true); + } +} + +extern "C" LLVMValueRef +LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source, + const char *Name, LLVMAtomicOrdering Order) { + Value *Ptr = unwrap(Source); + LoadInst *LI = unwrap(B)->CreateLoad(unwrap(Ty), Ptr, Name); + LI->setAtomic(fromRust(Order)); + return wrap(LI); +} + +extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B, + LLVMValueRef V, + LLVMValueRef Target, + LLVMAtomicOrdering Order) { + StoreInst *SI = unwrap(B)->CreateStore(unwrap(V), unwrap(Target)); + SI->setAtomic(fromRust(Order)); + return wrap(SI); +} + +enum class LLVMRustAsmDialect { + Att, + Intel, +}; + +static InlineAsm::AsmDialect fromRust(LLVMRustAsmDialect Dialect) { + switch (Dialect) { + case LLVMRustAsmDialect::Att: + return InlineAsm::AD_ATT; + case LLVMRustAsmDialect::Intel: + return InlineAsm::AD_Intel; + default: + report_fatal_error("bad AsmDialect."); + } +} + +extern "C" LLVMValueRef +LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen, + char *Constraints, size_t ConstraintsLen, + LLVMBool HasSideEffects, LLVMBool IsAlignStack, + LLVMRustAsmDialect Dialect, LLVMBool CanThrow) { + return wrap(InlineAsm::get(unwrap(Ty), + StringRef(AsmString, AsmStringLen), + StringRef(Constraints, ConstraintsLen), + HasSideEffects, IsAlignStack, + fromRust(Dialect), CanThrow)); +} + +extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints, + size_t ConstraintsLen) { + // llvm::Error converts to true if it is an error. + return !llvm::errorToBool(InlineAsm::verify( + unwrap(Ty), StringRef(Constraints, ConstraintsLen))); +} + +typedef DIBuilder *LLVMRustDIBuilderRef; + +template DIT *unwrapDIPtr(LLVMMetadataRef Ref) { + return (DIT *)(Ref ? unwrap(Ref) : nullptr); +} + +#define DIDescriptor DIScope +#define DIArray DINodeArray +#define unwrapDI unwrapDIPtr + +// These values **must** match debuginfo::DIFlags! They also *happen* +// to match LLVM, but that isn't required as we do giant sets of +// matching below. The value shouldn't be directly passed to LLVM. +enum class LLVMRustDIFlags : uint32_t { + FlagZero = 0, + FlagPrivate = 1, + FlagProtected = 2, + FlagPublic = 3, + FlagFwdDecl = (1 << 2), + FlagAppleBlock = (1 << 3), + FlagBlockByrefStruct = (1 << 4), + FlagVirtual = (1 << 5), + FlagArtificial = (1 << 6), + FlagExplicit = (1 << 7), + FlagPrototyped = (1 << 8), + FlagObjcClassComplete = (1 << 9), + FlagObjectPointer = (1 << 10), + FlagVector = (1 << 11), + FlagStaticMember = (1 << 12), + FlagLValueReference = (1 << 13), + FlagRValueReference = (1 << 14), + FlagExternalTypeRef = (1 << 15), + FlagIntroducedVirtual = (1 << 18), + FlagBitField = (1 << 19), + FlagNoReturn = (1 << 20), + // Do not add values that are not supported by the minimum LLVM + // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def +}; + +inline LLVMRustDIFlags operator&(LLVMRustDIFlags A, LLVMRustDIFlags B) { + return static_cast(static_cast(A) & + static_cast(B)); +} + +inline LLVMRustDIFlags operator|(LLVMRustDIFlags A, LLVMRustDIFlags B) { + return static_cast(static_cast(A) | + static_cast(B)); +} + +inline LLVMRustDIFlags &operator|=(LLVMRustDIFlags &A, LLVMRustDIFlags B) { + return A = A | B; +} + +inline bool isSet(LLVMRustDIFlags F) { return F != LLVMRustDIFlags::FlagZero; } + +inline LLVMRustDIFlags visibility(LLVMRustDIFlags F) { + return static_cast(static_cast(F) & 0x3); +} + +static DINode::DIFlags fromRust(LLVMRustDIFlags Flags) { + DINode::DIFlags Result = DINode::DIFlags::FlagZero; + + switch (visibility(Flags)) { + case LLVMRustDIFlags::FlagPrivate: + Result |= DINode::DIFlags::FlagPrivate; + break; + case LLVMRustDIFlags::FlagProtected: + Result |= DINode::DIFlags::FlagProtected; + break; + case LLVMRustDIFlags::FlagPublic: + Result |= DINode::DIFlags::FlagPublic; + break; + default: + // The rest are handled below + break; + } + + if (isSet(Flags & LLVMRustDIFlags::FlagFwdDecl)) { + Result |= DINode::DIFlags::FlagFwdDecl; + } + if (isSet(Flags & LLVMRustDIFlags::FlagAppleBlock)) { + Result |= DINode::DIFlags::FlagAppleBlock; + } + if (isSet(Flags & LLVMRustDIFlags::FlagVirtual)) { + Result |= DINode::DIFlags::FlagVirtual; + } + if (isSet(Flags & LLVMRustDIFlags::FlagArtificial)) { + Result |= DINode::DIFlags::FlagArtificial; + } + if (isSet(Flags & LLVMRustDIFlags::FlagExplicit)) { + Result |= DINode::DIFlags::FlagExplicit; + } + if (isSet(Flags & LLVMRustDIFlags::FlagPrototyped)) { + Result |= DINode::DIFlags::FlagPrototyped; + } + if (isSet(Flags & LLVMRustDIFlags::FlagObjcClassComplete)) { + Result |= DINode::DIFlags::FlagObjcClassComplete; + } + if (isSet(Flags & LLVMRustDIFlags::FlagObjectPointer)) { + Result |= DINode::DIFlags::FlagObjectPointer; + } + if (isSet(Flags & LLVMRustDIFlags::FlagVector)) { + Result |= DINode::DIFlags::FlagVector; + } + if (isSet(Flags & LLVMRustDIFlags::FlagStaticMember)) { + Result |= DINode::DIFlags::FlagStaticMember; + } + if (isSet(Flags & LLVMRustDIFlags::FlagLValueReference)) { + Result |= DINode::DIFlags::FlagLValueReference; + } + if (isSet(Flags & LLVMRustDIFlags::FlagRValueReference)) { + Result |= DINode::DIFlags::FlagRValueReference; + } + if (isSet(Flags & LLVMRustDIFlags::FlagIntroducedVirtual)) { + Result |= DINode::DIFlags::FlagIntroducedVirtual; + } + if (isSet(Flags & LLVMRustDIFlags::FlagBitField)) { + Result |= DINode::DIFlags::FlagBitField; + } + if (isSet(Flags & LLVMRustDIFlags::FlagNoReturn)) { + Result |= DINode::DIFlags::FlagNoReturn; + } + + return Result; +} + +// These values **must** match debuginfo::DISPFlags! They also *happen* +// to match LLVM, but that isn't required as we do giant sets of +// matching below. The value shouldn't be directly passed to LLVM. +enum class LLVMRustDISPFlags : uint32_t { + SPFlagZero = 0, + SPFlagVirtual = 1, + SPFlagPureVirtual = 2, + SPFlagLocalToUnit = (1 << 2), + SPFlagDefinition = (1 << 3), + SPFlagOptimized = (1 << 4), + SPFlagMainSubprogram = (1 << 5), + // Do not add values that are not supported by the minimum LLVM + // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def + // (In LLVM < 8, createFunction supported these as separate bool arguments.) +}; + +inline LLVMRustDISPFlags operator&(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { + return static_cast(static_cast(A) & + static_cast(B)); +} + +inline LLVMRustDISPFlags operator|(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { + return static_cast(static_cast(A) | + static_cast(B)); +} + +inline LLVMRustDISPFlags &operator|=(LLVMRustDISPFlags &A, LLVMRustDISPFlags B) { + return A = A | B; +} + +inline bool isSet(LLVMRustDISPFlags F) { return F != LLVMRustDISPFlags::SPFlagZero; } + +inline LLVMRustDISPFlags virtuality(LLVMRustDISPFlags F) { + return static_cast(static_cast(F) & 0x3); +} + +static DISubprogram::DISPFlags fromRust(LLVMRustDISPFlags SPFlags) { + DISubprogram::DISPFlags Result = DISubprogram::DISPFlags::SPFlagZero; + + switch (virtuality(SPFlags)) { + case LLVMRustDISPFlags::SPFlagVirtual: + Result |= DISubprogram::DISPFlags::SPFlagVirtual; + break; + case LLVMRustDISPFlags::SPFlagPureVirtual: + Result |= DISubprogram::DISPFlags::SPFlagPureVirtual; + break; + default: + // The rest are handled below + break; + } + + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagLocalToUnit)) { + Result |= DISubprogram::DISPFlags::SPFlagLocalToUnit; + } + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagDefinition)) { + Result |= DISubprogram::DISPFlags::SPFlagDefinition; + } + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagOptimized)) { + Result |= DISubprogram::DISPFlags::SPFlagOptimized; + } + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagMainSubprogram)) { + Result |= DISubprogram::DISPFlags::SPFlagMainSubprogram; + } + + return Result; +} + +enum class LLVMRustDebugEmissionKind { + NoDebug, + FullDebug, + LineTablesOnly, + DebugDirectivesOnly, +}; + +static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind) { + switch (Kind) { + case LLVMRustDebugEmissionKind::NoDebug: + return DICompileUnit::DebugEmissionKind::NoDebug; + case LLVMRustDebugEmissionKind::FullDebug: + return DICompileUnit::DebugEmissionKind::FullDebug; + case LLVMRustDebugEmissionKind::LineTablesOnly: + return DICompileUnit::DebugEmissionKind::LineTablesOnly; + case LLVMRustDebugEmissionKind::DebugDirectivesOnly: + return DICompileUnit::DebugEmissionKind::DebugDirectivesOnly; + default: + report_fatal_error("bad DebugEmissionKind."); + } +} + +enum class LLVMRustChecksumKind { + None, + MD5, + SHA1, + SHA256, +}; + +#if LLVM_VERSION_LT(16, 0) +static Optional fromRust(LLVMRustChecksumKind Kind) { +#else +static std::optional fromRust(LLVMRustChecksumKind Kind) { +#endif + switch (Kind) { + case LLVMRustChecksumKind::None: +#if LLVM_VERSION_LT(16, 0) + return None; +#else + return std::nullopt; +#endif + case LLVMRustChecksumKind::MD5: + return DIFile::ChecksumKind::CSK_MD5; + case LLVMRustChecksumKind::SHA1: + return DIFile::ChecksumKind::CSK_SHA1; + case LLVMRustChecksumKind::SHA256: + return DIFile::ChecksumKind::CSK_SHA256; + default: + report_fatal_error("bad ChecksumKind."); + } +} + +extern "C" uint32_t LLVMRustDebugMetadataVersion() { + return DEBUG_METADATA_VERSION; +} + +extern "C" uint32_t LLVMRustVersionPatch() { return LLVM_VERSION_PATCH; } + +extern "C" uint32_t LLVMRustVersionMinor() { return LLVM_VERSION_MINOR; } + +extern "C" uint32_t LLVMRustVersionMajor() { return LLVM_VERSION_MAJOR; } + +extern "C" void LLVMRustAddModuleFlag( + LLVMModuleRef M, + Module::ModFlagBehavior MergeBehavior, + const char *Name, + uint32_t Value) { + unwrap(M)->addModuleFlag(MergeBehavior, Name, Value); +} + +extern "C" bool LLVMRustHasModuleFlag(LLVMModuleRef M, const char *Name, + size_t Len) { + return unwrap(M)->getModuleFlag(StringRef(Name, Len)) != nullptr; +} + +extern "C" void LLVMRustGlobalAddMetadata( + LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) { + unwrap(Global)->addMetadata(Kind, *unwrap(MD)); +} + +extern "C" LLVMRustDIBuilderRef LLVMRustDIBuilderCreate(LLVMModuleRef M) { + return new DIBuilder(*unwrap(M)); +} + +extern "C" void LLVMRustDIBuilderDispose(LLVMRustDIBuilderRef Builder) { + delete Builder; +} + +extern "C" void LLVMRustDIBuilderFinalize(LLVMRustDIBuilderRef Builder) { + Builder->finalize(); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateCompileUnit( + LLVMRustDIBuilderRef Builder, unsigned Lang, LLVMMetadataRef FileRef, + const char *Producer, size_t ProducerLen, bool isOptimized, + const char *Flags, unsigned RuntimeVer, + const char *SplitName, size_t SplitNameLen, + LLVMRustDebugEmissionKind Kind, + uint64_t DWOId, bool SplitDebugInlining) { + auto *File = unwrapDI(FileRef); + + return wrap(Builder->createCompileUnit(Lang, File, StringRef(Producer, ProducerLen), + isOptimized, Flags, RuntimeVer, + StringRef(SplitName, SplitNameLen), + fromRust(Kind), DWOId, SplitDebugInlining)); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFile( + LLVMRustDIBuilderRef Builder, + const char *Filename, size_t FilenameLen, + const char *Directory, size_t DirectoryLen, LLVMRustChecksumKind CSKind, + const char *Checksum, size_t ChecksumLen) { + +#if LLVM_VERSION_LT(16, 0) + Optional llvmCSKind = fromRust(CSKind); +#else + std::optional llvmCSKind = fromRust(CSKind); +#endif + +#if LLVM_VERSION_LT(16, 0) + Optional> CSInfo{}; +#else + std::optional> CSInfo{}; +#endif + if (llvmCSKind) + CSInfo.emplace(*llvmCSKind, StringRef{Checksum, ChecksumLen}); + return wrap(Builder->createFile(StringRef(Filename, FilenameLen), + StringRef(Directory, DirectoryLen), + CSInfo)); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateSubroutineType(LLVMRustDIBuilderRef Builder, + LLVMMetadataRef ParameterTypes) { + return wrap(Builder->createSubroutineType( + DITypeRefArray(unwrap(ParameterTypes)))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + const char *LinkageName, size_t LinkageNameLen, + LLVMMetadataRef File, unsigned LineNo, + LLVMMetadataRef Ty, unsigned ScopeLine, LLVMRustDIFlags Flags, + LLVMRustDISPFlags SPFlags, LLVMValueRef MaybeFn, LLVMMetadataRef TParam, + LLVMMetadataRef Decl) { + DITemplateParameterArray TParams = + DITemplateParameterArray(unwrap(TParam)); + DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); + DINode::DIFlags llvmFlags = fromRust(Flags); + DISubprogram *Sub = Builder->createFunction( + unwrapDI(Scope), + StringRef(Name, NameLen), + StringRef(LinkageName, LinkageNameLen), + unwrapDI(File), LineNo, + unwrapDI(Ty), ScopeLine, llvmFlags, + llvmSPFlags, TParams, unwrapDIPtr(Decl)); + if (MaybeFn) + unwrap(MaybeFn)->setSubprogram(Sub); + return wrap(Sub); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMethod( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + const char *LinkageName, size_t LinkageNameLen, + LLVMMetadataRef File, unsigned LineNo, + LLVMMetadataRef Ty, LLVMRustDIFlags Flags, + LLVMRustDISPFlags SPFlags, LLVMMetadataRef TParam) { + DITemplateParameterArray TParams = + DITemplateParameterArray(unwrap(TParam)); + DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); + DINode::DIFlags llvmFlags = fromRust(Flags); + DISubprogram *Sub = Builder->createMethod( + unwrapDI(Scope), + StringRef(Name, NameLen), + StringRef(LinkageName, LinkageNameLen), + unwrapDI(File), LineNo, + unwrapDI(Ty), + 0, 0, nullptr, // VTable params aren't used + llvmFlags, llvmSPFlags, TParams); + return wrap(Sub); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateBasicType( + LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, + uint64_t SizeInBits, unsigned Encoding) { + return wrap(Builder->createBasicType(StringRef(Name, NameLen), SizeInBits, Encoding)); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTypedef( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Type, const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Scope) { + return wrap(Builder->createTypedef( + unwrap(Type), StringRef(Name, NameLen), unwrap(File), + LineNo, unwrapDIPtr(Scope))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef PointeeTy, + uint64_t SizeInBits, uint32_t AlignInBits, unsigned AddressSpace, + const char *Name, size_t NameLen) { + return wrap(Builder->createPointerType(unwrapDI(PointeeTy), + SizeInBits, AlignInBits, + AddressSpace, + StringRef(Name, NameLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStructType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, + uint32_t AlignInBits, LLVMRustDIFlags Flags, + LLVMMetadataRef DerivedFrom, LLVMMetadataRef Elements, + unsigned RunTimeLang, LLVMMetadataRef VTableHolder, + const char *UniqueId, size_t UniqueIdLen) { + return wrap(Builder->createStructType( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNumber, + SizeInBits, AlignInBits, fromRust(Flags), unwrapDI(DerivedFrom), + DINodeArray(unwrapDI(Elements)), RunTimeLang, + unwrapDI(VTableHolder), StringRef(UniqueId, UniqueIdLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantPart( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, + uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Discriminator, + LLVMMetadataRef Elements, const char *UniqueId, size_t UniqueIdLen) { + return wrap(Builder->createVariantPart( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNumber, + SizeInBits, AlignInBits, fromRust(Flags), unwrapDI(Discriminator), + DINodeArray(unwrapDI(Elements)), StringRef(UniqueId, UniqueIdLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMemberType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits, + uint32_t AlignInBits, uint64_t OffsetInBits, LLVMRustDIFlags Flags, + LLVMMetadataRef Ty) { + return wrap(Builder->createMemberType(unwrapDI(Scope), + StringRef(Name, NameLen), + unwrapDI(File), LineNo, + SizeInBits, AlignInBits, OffsetInBits, + fromRust(Flags), unwrapDI(Ty))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo, + uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, LLVMValueRef Discriminant, + LLVMRustDIFlags Flags, LLVMMetadataRef Ty) { + llvm::ConstantInt* D = nullptr; + if (Discriminant) { + D = unwrap(Discriminant); + } + return wrap(Builder->createVariantMemberType(unwrapDI(Scope), + StringRef(Name, NameLen), + unwrapDI(File), LineNo, + SizeInBits, AlignInBits, OffsetInBits, D, + fromRust(Flags), unwrapDI(Ty))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticMemberType( + LLVMRustDIBuilderRef Builder, + LLVMMetadataRef Scope, + const char *Name, + size_t NameLen, + LLVMMetadataRef File, + unsigned LineNo, + LLVMMetadataRef Ty, + LLVMRustDIFlags Flags, + LLVMValueRef val, + uint32_t AlignInBits +) { + return wrap(Builder->createStaticMemberType( + unwrapDI(Scope), + StringRef(Name, NameLen), + unwrapDI(File), + LineNo, + unwrapDI(Ty), + fromRust(Flags), + unwrap(val), +#if LLVM_VERSION_GE(18, 0) + llvm::dwarf::DW_TAG_member, +#endif + AlignInBits + )); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateLexicalBlock( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + LLVMMetadataRef File, unsigned Line, unsigned Col) { + return wrap(Builder->createLexicalBlock(unwrapDI(Scope), + unwrapDI(File), Line, Col)); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateLexicalBlockFile(LLVMRustDIBuilderRef Builder, + LLVMMetadataRef Scope, + LLVMMetadataRef File) { + return wrap(Builder->createLexicalBlockFile(unwrapDI(Scope), + unwrapDI(File))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Context, + const char *Name, size_t NameLen, + const char *LinkageName, size_t LinkageNameLen, + LLVMMetadataRef File, unsigned LineNo, + LLVMMetadataRef Ty, bool IsLocalToUnit, LLVMValueRef V, + LLVMMetadataRef Decl = nullptr, uint32_t AlignInBits = 0) { + llvm::GlobalVariable *InitVal = cast(unwrap(V)); + + llvm::DIExpression *InitExpr = nullptr; + if (llvm::ConstantInt *IntVal = llvm::dyn_cast(InitVal)) { + InitExpr = Builder->createConstantValueExpression( + IntVal->getValue().getSExtValue()); + } else if (llvm::ConstantFP *FPVal = + llvm::dyn_cast(InitVal)) { + InitExpr = Builder->createConstantValueExpression( + FPVal->getValueAPF().bitcastToAPInt().getZExtValue()); + } + + llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression( + unwrapDI(Context), StringRef(Name, NameLen), + StringRef(LinkageName, LinkageNameLen), + unwrapDI(File), LineNo, unwrapDI(Ty), IsLocalToUnit, + /* isDefined */ true, + InitExpr, unwrapDIPtr(Decl), + /* templateParams */ nullptr, + AlignInBits); + + InitVal->setMetadata("dbg", VarExpr); + + return wrap(VarExpr); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable( + LLVMRustDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNo, + LLVMMetadataRef Ty, bool AlwaysPreserve, LLVMRustDIFlags Flags, + unsigned ArgNo, uint32_t AlignInBits) { + if (Tag == 0x100) { // DW_TAG_auto_variable + return wrap(Builder->createAutoVariable( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNo, + unwrapDI(Ty), AlwaysPreserve, fromRust(Flags), AlignInBits)); + } else { + return wrap(Builder->createParameterVariable( + unwrapDI(Scope), StringRef(Name, NameLen), ArgNo, + unwrapDI(File), LineNo, + unwrapDI(Ty), AlwaysPreserve, fromRust(Flags))); + } +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateArrayType(LLVMRustDIBuilderRef Builder, uint64_t Size, + uint32_t AlignInBits, LLVMMetadataRef Ty, + LLVMMetadataRef Subscripts) { + return wrap( + Builder->createArrayType(Size, AlignInBits, unwrapDI(Ty), + DINodeArray(unwrapDI(Subscripts)))); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderGetOrCreateSubrange(LLVMRustDIBuilderRef Builder, int64_t Lo, + int64_t Count) { + return wrap(Builder->getOrCreateSubrange(Lo, Count)); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderGetOrCreateArray(LLVMRustDIBuilderRef Builder, + LLVMMetadataRef *Ptr, unsigned Count) { + Metadata **DataValue = unwrap(Ptr); + return wrap( + Builder->getOrCreateArray(ArrayRef(DataValue, Count)).get()); +} + +extern "C" LLVMValueRef LLVMRustDIBuilderInsertDeclareAtEnd( + LLVMRustDIBuilderRef Builder, LLVMValueRef V, LLVMMetadataRef VarInfo, + uint64_t *AddrOps, unsigned AddrOpsCount, LLVMMetadataRef DL, + LLVMBasicBlockRef InsertAtEnd) { + return wrap(Builder->insertDeclare( + unwrap(V), unwrap(VarInfo), + Builder->createExpression(llvm::ArrayRef(AddrOps, AddrOpsCount)), + DebugLoc(cast(unwrap(DL))), + unwrap(InsertAtEnd))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerator( + LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, + const uint64_t Value[2], unsigned SizeInBits, bool IsUnsigned) { + return wrap(Builder->createEnumerator(StringRef(Name, NameLen), + APSInt(APInt(SizeInBits, ArrayRef(Value, 2)), IsUnsigned))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, + uint32_t AlignInBits, LLVMMetadataRef Elements, + LLVMMetadataRef ClassTy, bool IsScoped) { + return wrap(Builder->createEnumerationType( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNumber, + SizeInBits, AlignInBits, DINodeArray(unwrapDI(Elements)), + unwrapDI(ClassTy), +#if LLVM_VERSION_GE(18, 0) + /* RunTimeLang */ 0, +#endif + "", IsScoped)); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, + uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Elements, + unsigned RunTimeLang, const char *UniqueId, size_t UniqueIdLen) { + return wrap(Builder->createUnionType( + unwrapDI(Scope), StringRef(Name, NameLen), unwrapDI(File), + LineNumber, SizeInBits, AlignInBits, fromRust(Flags), + DINodeArray(unwrapDI(Elements)), RunTimeLang, + StringRef(UniqueId, UniqueIdLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTemplateTypeParameter( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, LLVMMetadataRef Ty) { + bool IsDefault = false; // FIXME: should we ever set this true? + return wrap(Builder->createTemplateTypeParameter( + unwrapDI(Scope), StringRef(Name, NameLen), unwrapDI(Ty), IsDefault)); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateNameSpace( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, bool ExportSymbols) { + return wrap(Builder->createNameSpace( + unwrapDI(Scope), StringRef(Name, NameLen), ExportSymbols + )); +} + +extern "C" void +LLVMRustDICompositeTypeReplaceArrays(LLVMRustDIBuilderRef Builder, + LLVMMetadataRef CompositeTy, + LLVMMetadataRef Elements, + LLVMMetadataRef Params) { + DICompositeType *Tmp = unwrapDI(CompositeTy); + Builder->replaceArrays(Tmp, DINodeArray(unwrap(Elements)), + DINodeArray(unwrap(Params))); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateDebugLocation(unsigned Line, unsigned Column, + LLVMMetadataRef ScopeRef, + LLVMMetadataRef InlinedAt) { + MDNode *Scope = unwrapDIPtr(ScopeRef); + DILocation *Loc = DILocation::get( + Scope->getContext(), Line, Column, Scope, + unwrapDIPtr(InlinedAt)); + return wrap(Loc); +} + +extern "C" uint64_t LLVMRustDIBuilderCreateOpDeref() { + return dwarf::DW_OP_deref; +} + +extern "C" uint64_t LLVMRustDIBuilderCreateOpPlusUconst() { + return dwarf::DW_OP_plus_uconst; +} + +extern "C" int64_t LLVMRustDIBuilderCreateOpLLVMFragment() { + return dwarf::DW_OP_LLVM_fragment; +} + +extern "C" void LLVMRustWriteTypeToString(LLVMTypeRef Ty, RustStringRef Str) { + RawRustStringOstream OS(Str); + unwrap(Ty)->print(OS); +} + +extern "C" void LLVMRustWriteValueToString(LLVMValueRef V, + RustStringRef Str) { + RawRustStringOstream OS(Str); + if (!V) { + OS << "(null)"; + } else { + OS << "("; + unwrap(V)->getType()->print(OS); + OS << ":"; + unwrap(V)->print(OS); + OS << ")"; + } +} + +// LLVMArrayType function does not support 64-bit ElementCount +// FIXME: replace with LLVMArrayType2 when bumped minimal version to llvm-17 +// https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc +extern "C" LLVMTypeRef LLVMRustArrayType(LLVMTypeRef ElementTy, + uint64_t ElementCount) { + return wrap(ArrayType::get(unwrap(ElementTy), ElementCount)); +} + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Twine, LLVMTwineRef) + +extern "C" void LLVMRustWriteTwineToString(LLVMTwineRef T, RustStringRef Str) { + RawRustStringOstream OS(Str); + unwrap(T)->print(OS); +} + +extern "C" void LLVMRustUnpackOptimizationDiagnostic( + LLVMDiagnosticInfoRef DI, RustStringRef PassNameOut, + LLVMValueRef *FunctionOut, unsigned* Line, unsigned* Column, + RustStringRef FilenameOut, RustStringRef MessageOut) { + // Undefined to call this not on an optimization diagnostic! + llvm::DiagnosticInfoOptimizationBase *Opt = + static_cast(unwrap(DI)); + + RawRustStringOstream PassNameOS(PassNameOut); + PassNameOS << Opt->getPassName(); + *FunctionOut = wrap(&Opt->getFunction()); + + RawRustStringOstream FilenameOS(FilenameOut); + DiagnosticLocation loc = Opt->getLocation(); + if (loc.isValid()) { + *Line = loc.getLine(); + *Column = loc.getColumn(); + FilenameOS << loc.getAbsolutePath(); + } + + RawRustStringOstream MessageOS(MessageOut); + MessageOS << Opt->getMsg(); +} + +enum class LLVMRustDiagnosticLevel { + Error, + Warning, + Note, + Remark, +}; + +extern "C" void +LLVMRustUnpackInlineAsmDiagnostic(LLVMDiagnosticInfoRef DI, + LLVMRustDiagnosticLevel *LevelOut, + unsigned *CookieOut, + LLVMTwineRef *MessageOut) { + // Undefined to call this not on an inline assembly diagnostic! + llvm::DiagnosticInfoInlineAsm *IA = + static_cast(unwrap(DI)); + + *CookieOut = IA->getLocCookie(); + *MessageOut = wrap(&IA->getMsgStr()); + + switch (IA->getSeverity()) { + case DS_Error: + *LevelOut = LLVMRustDiagnosticLevel::Error; + break; + case DS_Warning: + *LevelOut = LLVMRustDiagnosticLevel::Warning; + break; + case DS_Note: + *LevelOut = LLVMRustDiagnosticLevel::Note; + break; + case DS_Remark: + *LevelOut = LLVMRustDiagnosticLevel::Remark; + break; + default: + report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); + } +} + +extern "C" void LLVMRustWriteDiagnosticInfoToString(LLVMDiagnosticInfoRef DI, + RustStringRef Str) { + RawRustStringOstream OS(Str); + DiagnosticPrinterRawOStream DP(OS); + unwrap(DI)->print(DP); +} + +enum class LLVMRustDiagnosticKind { + Other, + InlineAsm, + StackSize, + DebugMetadataVersion, + SampleProfile, + OptimizationRemark, + OptimizationRemarkMissed, + OptimizationRemarkAnalysis, + OptimizationRemarkAnalysisFPCommute, + OptimizationRemarkAnalysisAliasing, + OptimizationRemarkOther, + OptimizationFailure, + PGOProfile, + Linker, + Unsupported, + SrcMgr, +}; + +static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) { + switch (Kind) { + case DK_InlineAsm: + return LLVMRustDiagnosticKind::InlineAsm; + case DK_StackSize: + return LLVMRustDiagnosticKind::StackSize; + case DK_DebugMetadataVersion: + return LLVMRustDiagnosticKind::DebugMetadataVersion; + case DK_SampleProfile: + return LLVMRustDiagnosticKind::SampleProfile; + case DK_OptimizationRemark: + case DK_MachineOptimizationRemark: + return LLVMRustDiagnosticKind::OptimizationRemark; + case DK_OptimizationRemarkMissed: + case DK_MachineOptimizationRemarkMissed: + return LLVMRustDiagnosticKind::OptimizationRemarkMissed; + case DK_OptimizationRemarkAnalysis: + case DK_MachineOptimizationRemarkAnalysis: + return LLVMRustDiagnosticKind::OptimizationRemarkAnalysis; + case DK_OptimizationRemarkAnalysisFPCommute: + return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisFPCommute; + case DK_OptimizationRemarkAnalysisAliasing: + return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisAliasing; + case DK_PGOProfile: + return LLVMRustDiagnosticKind::PGOProfile; + case DK_Linker: + return LLVMRustDiagnosticKind::Linker; + case DK_Unsupported: + return LLVMRustDiagnosticKind::Unsupported; + case DK_SrcMgr: + return LLVMRustDiagnosticKind::SrcMgr; + default: + return (Kind >= DK_FirstRemark && Kind <= DK_LastRemark) + ? LLVMRustDiagnosticKind::OptimizationRemarkOther + : LLVMRustDiagnosticKind::Other; + } +} + +extern "C" LLVMRustDiagnosticKind +LLVMRustGetDiagInfoKind(LLVMDiagnosticInfoRef DI) { + return toRust((DiagnosticKind)unwrap(DI)->getKind()); +} + +// This is kept distinct from LLVMGetTypeKind, because when +// a new type kind is added, the Rust-side enum must be +// updated or UB will result. +extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) { + switch (unwrap(Ty)->getTypeID()) { + case Type::VoidTyID: + return LLVMVoidTypeKind; + case Type::HalfTyID: + return LLVMHalfTypeKind; + case Type::FloatTyID: + return LLVMFloatTypeKind; + case Type::DoubleTyID: + return LLVMDoubleTypeKind; + case Type::X86_FP80TyID: + return LLVMX86_FP80TypeKind; + case Type::FP128TyID: + return LLVMFP128TypeKind; + case Type::PPC_FP128TyID: + return LLVMPPC_FP128TypeKind; + case Type::LabelTyID: + return LLVMLabelTypeKind; + case Type::MetadataTyID: + return LLVMMetadataTypeKind; + case Type::IntegerTyID: + return LLVMIntegerTypeKind; + case Type::FunctionTyID: + return LLVMFunctionTypeKind; + case Type::StructTyID: + return LLVMStructTypeKind; + case Type::ArrayTyID: + return LLVMArrayTypeKind; + case Type::PointerTyID: + return LLVMPointerTypeKind; + case Type::FixedVectorTyID: + return LLVMVectorTypeKind; + case Type::X86_MMXTyID: + return LLVMX86_MMXTypeKind; + case Type::TokenTyID: + return LLVMTokenTypeKind; + case Type::ScalableVectorTyID: + return LLVMScalableVectorTypeKind; + case Type::BFloatTyID: + return LLVMBFloatTypeKind; + case Type::X86_AMXTyID: + return LLVMX86_AMXTypeKind; + default: + { + std::string error; + llvm::raw_string_ostream stream(error); + stream << "Rust does not support the TypeID: " << unwrap(Ty)->getTypeID() + << " for the type: " << *unwrap(Ty); + stream.flush(); + report_fatal_error(error.c_str()); + } + } +} + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef) + +extern "C" LLVMSMDiagnosticRef LLVMRustGetSMDiagnostic( + LLVMDiagnosticInfoRef DI, unsigned *Cookie) { + llvm::DiagnosticInfoSrcMgr *SM = static_cast(unwrap(DI)); + *Cookie = SM->getLocCookie(); + return wrap(&SM->getSMDiag()); +} + +extern "C" bool LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef, + RustStringRef MessageOut, + RustStringRef BufferOut, + LLVMRustDiagnosticLevel* LevelOut, + unsigned* LocOut, + unsigned* RangesOut, + size_t* NumRanges) { + SMDiagnostic& D = *unwrap(DRef); + RawRustStringOstream MessageOS(MessageOut); + MessageOS << D.getMessage(); + + switch (D.getKind()) { + case SourceMgr::DK_Error: + *LevelOut = LLVMRustDiagnosticLevel::Error; + break; + case SourceMgr::DK_Warning: + *LevelOut = LLVMRustDiagnosticLevel::Warning; + break; + case SourceMgr::DK_Note: + *LevelOut = LLVMRustDiagnosticLevel::Note; + break; + case SourceMgr::DK_Remark: + *LevelOut = LLVMRustDiagnosticLevel::Remark; + break; + default: + report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); + } + + if (D.getLoc() == SMLoc()) + return false; + + const SourceMgr &LSM = *D.getSourceMgr(); + const MemoryBuffer *LBuf = LSM.getMemoryBuffer(LSM.FindBufferContainingLoc(D.getLoc())); + LLVMRustStringWriteImpl(BufferOut, LBuf->getBufferStart(), LBuf->getBufferSize()); + + *LocOut = D.getLoc().getPointer() - LBuf->getBufferStart(); + + *NumRanges = std::min(*NumRanges, D.getRanges().size()); + size_t LineStart = *LocOut - (size_t)D.getColumnNo(); + for (size_t i = 0; i < *NumRanges; i++) { + RangesOut[i * 2] = LineStart + D.getRanges()[i].first; + RangesOut[i * 2 + 1] = LineStart + D.getRanges()[i].second; + } + + return true; +} + +extern "C" OperandBundleDef *LLVMRustBuildOperandBundleDef(const char *Name, + LLVMValueRef *Inputs, + unsigned NumInputs) { + return new OperandBundleDef(Name, ArrayRef(unwrap(Inputs), NumInputs)); +} + +extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { + delete Bundle; +} + +extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, + LLVMValueRef *Args, unsigned NumArgs, + OperandBundleDef **OpBundles, + unsigned NumOpBundles) { + Value *Callee = unwrap(Fn); + FunctionType *FTy = unwrap(Ty); + return wrap(unwrap(B)->CreateCall( + FTy, Callee, ArrayRef(unwrap(Args), NumArgs), + ArrayRef(*OpBundles, NumOpBundles))); +} + +extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) { + return wrap(llvm::Intrinsic::getDeclaration(unwrap(M), + (llvm::Intrinsic::ID)llvm::Intrinsic::instrprof_increment)); +} + +extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, + LLVMValueRef Size, bool IsVolatile) { + return wrap(unwrap(B)->CreateMemCpy( + unwrap(Dst), MaybeAlign(DstAlign), + unwrap(Src), MaybeAlign(SrcAlign), + unwrap(Size), IsVolatile)); +} + +extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, + LLVMValueRef Size, bool IsVolatile) { + return wrap(unwrap(B)->CreateMemMove( + unwrap(Dst), MaybeAlign(DstAlign), + unwrap(Src), MaybeAlign(SrcAlign), + unwrap(Size), IsVolatile)); +} + +extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Val, + LLVMValueRef Size, bool IsVolatile) { + return wrap(unwrap(B)->CreateMemSet( + unwrap(Dst), unwrap(Val), unwrap(Size), MaybeAlign(DstAlign), IsVolatile)); +} + +extern "C" LLVMValueRef +LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, + LLVMValueRef *Args, unsigned NumArgs, + LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, + OperandBundleDef **OpBundles, unsigned NumOpBundles, + const char *Name) { + Value *Callee = unwrap(Fn); + FunctionType *FTy = unwrap(Ty); + return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch), + ArrayRef(unwrap(Args), NumArgs), + ArrayRef(*OpBundles, NumOpBundles), + Name)); +} + +extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, + LLVMBasicBlockRef BB) { + auto Point = unwrap(BB)->getFirstInsertionPt(); + unwrap(B)->SetInsertPoint(unwrap(BB), Point); +} + +extern "C" void LLVMRustSetComdat(LLVMModuleRef M, LLVMValueRef V, + const char *Name, size_t NameLen) { + Triple TargetTriple(unwrap(M)->getTargetTriple()); + GlobalObject *GV = unwrap(V); + if (TargetTriple.supportsCOMDAT()) { + StringRef NameRef(Name, NameLen); + GV->setComdat(unwrap(M)->getOrInsertComdat(NameRef)); + } +} + +enum class LLVMRustLinkage { + ExternalLinkage = 0, + AvailableExternallyLinkage = 1, + LinkOnceAnyLinkage = 2, + LinkOnceODRLinkage = 3, + WeakAnyLinkage = 4, + WeakODRLinkage = 5, + AppendingLinkage = 6, + InternalLinkage = 7, + PrivateLinkage = 8, + ExternalWeakLinkage = 9, + CommonLinkage = 10, +}; + +static LLVMRustLinkage toRust(LLVMLinkage Linkage) { + switch (Linkage) { + case LLVMExternalLinkage: + return LLVMRustLinkage::ExternalLinkage; + case LLVMAvailableExternallyLinkage: + return LLVMRustLinkage::AvailableExternallyLinkage; + case LLVMLinkOnceAnyLinkage: + return LLVMRustLinkage::LinkOnceAnyLinkage; + case LLVMLinkOnceODRLinkage: + return LLVMRustLinkage::LinkOnceODRLinkage; + case LLVMWeakAnyLinkage: + return LLVMRustLinkage::WeakAnyLinkage; + case LLVMWeakODRLinkage: + return LLVMRustLinkage::WeakODRLinkage; + case LLVMAppendingLinkage: + return LLVMRustLinkage::AppendingLinkage; + case LLVMInternalLinkage: + return LLVMRustLinkage::InternalLinkage; + case LLVMPrivateLinkage: + return LLVMRustLinkage::PrivateLinkage; + case LLVMExternalWeakLinkage: + return LLVMRustLinkage::ExternalWeakLinkage; + case LLVMCommonLinkage: + return LLVMRustLinkage::CommonLinkage; + default: + report_fatal_error("Invalid LLVMRustLinkage value!"); + } +} + +static LLVMLinkage fromRust(LLVMRustLinkage Linkage) { + switch (Linkage) { + case LLVMRustLinkage::ExternalLinkage: + return LLVMExternalLinkage; + case LLVMRustLinkage::AvailableExternallyLinkage: + return LLVMAvailableExternallyLinkage; + case LLVMRustLinkage::LinkOnceAnyLinkage: + return LLVMLinkOnceAnyLinkage; + case LLVMRustLinkage::LinkOnceODRLinkage: + return LLVMLinkOnceODRLinkage; + case LLVMRustLinkage::WeakAnyLinkage: + return LLVMWeakAnyLinkage; + case LLVMRustLinkage::WeakODRLinkage: + return LLVMWeakODRLinkage; + case LLVMRustLinkage::AppendingLinkage: + return LLVMAppendingLinkage; + case LLVMRustLinkage::InternalLinkage: + return LLVMInternalLinkage; + case LLVMRustLinkage::PrivateLinkage: + return LLVMPrivateLinkage; + case LLVMRustLinkage::ExternalWeakLinkage: + return LLVMExternalWeakLinkage; + case LLVMRustLinkage::CommonLinkage: + return LLVMCommonLinkage; + } + report_fatal_error("Invalid LLVMRustLinkage value!"); +} + +extern "C" LLVMRustLinkage LLVMRustGetLinkage(LLVMValueRef V) { + return toRust(LLVMGetLinkage(V)); +} + +extern "C" void LLVMRustSetLinkage(LLVMValueRef V, + LLVMRustLinkage RustLinkage) { + LLVMSetLinkage(V, fromRust(RustLinkage)); +} + +extern "C" bool LLVMRustConstIntGetZExtValue(LLVMValueRef CV, uint64_t *value) { + auto C = unwrap(CV); + if (C->getBitWidth() > 64) + return false; + *value = C->getZExtValue(); + return true; +} + +// Returns true if both high and low were successfully set. Fails in case constant wasn’t any of +// the common sizes (1, 8, 16, 32, 64, 128 bits) +extern "C" bool LLVMRustConstInt128Get(LLVMValueRef CV, bool sext, uint64_t *high, uint64_t *low) +{ + auto C = unwrap(CV); + if (C->getBitWidth() > 128) { return false; } + APInt AP; + if (sext) { + AP = C->getValue().sext(128); + } else { + AP = C->getValue().zext(128); + } + *low = AP.getLoBits(64).getZExtValue(); + *high = AP.getHiBits(64).getZExtValue(); + return true; +} + +enum class LLVMRustVisibility { + Default = 0, + Hidden = 1, + Protected = 2, +}; + +static LLVMRustVisibility toRust(LLVMVisibility Vis) { + switch (Vis) { + case LLVMDefaultVisibility: + return LLVMRustVisibility::Default; + case LLVMHiddenVisibility: + return LLVMRustVisibility::Hidden; + case LLVMProtectedVisibility: + return LLVMRustVisibility::Protected; + } + report_fatal_error("Invalid LLVMRustVisibility value!"); +} + +static LLVMVisibility fromRust(LLVMRustVisibility Vis) { + switch (Vis) { + case LLVMRustVisibility::Default: + return LLVMDefaultVisibility; + case LLVMRustVisibility::Hidden: + return LLVMHiddenVisibility; + case LLVMRustVisibility::Protected: + return LLVMProtectedVisibility; + } + report_fatal_error("Invalid LLVMRustVisibility value!"); +} + +extern "C" LLVMRustVisibility LLVMRustGetVisibility(LLVMValueRef V) { + return toRust(LLVMGetVisibility(V)); +} + +extern "C" void LLVMRustSetVisibility(LLVMValueRef V, + LLVMRustVisibility RustVisibility) { + LLVMSetVisibility(V, fromRust(RustVisibility)); +} + +extern "C" void LLVMRustSetDSOLocal(LLVMValueRef Global, bool is_dso_local) { + unwrap(Global)->setDSOLocal(is_dso_local); +} + +struct LLVMRustModuleBuffer { + std::string data; +}; + +extern "C" LLVMRustModuleBuffer* +LLVMRustModuleBufferCreate(LLVMModuleRef M) { + auto Ret = std::make_unique(); + { + raw_string_ostream OS(Ret->data); + WriteBitcodeToFile(*unwrap(M), OS); + } + return Ret.release(); +} + +extern "C" void +LLVMRustModuleBufferFree(LLVMRustModuleBuffer *Buffer) { + delete Buffer; +} + +extern "C" const void* +LLVMRustModuleBufferPtr(const LLVMRustModuleBuffer *Buffer) { + return Buffer->data.data(); +} + +extern "C" size_t +LLVMRustModuleBufferLen(const LLVMRustModuleBuffer *Buffer) { + return Buffer->data.length(); +} + +extern "C" uint64_t +LLVMRustModuleCost(LLVMModuleRef M) { + auto f = unwrap(M)->functions(); + return std::distance(std::begin(f), std::end(f)); +} + +extern "C" void +LLVMRustModuleInstructionStats(LLVMModuleRef M, RustStringRef Str) +{ + RawRustStringOstream OS(Str); + llvm::json::OStream JOS(OS); + auto Module = unwrap(M); + + JOS.object([&] { + JOS.attribute("module", Module->getName()); + JOS.attribute("total", Module->getInstructionCount()); + }); +} + +// Vector reductions: +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceFAdd(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateFAddReduce(unwrap(Acc),unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceFMul(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateFMulReduce(unwrap(Acc),unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceAdd(LLVMBuilderRef B, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateAddReduce(unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceMul(LLVMBuilderRef B, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateMulReduce(unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceAnd(LLVMBuilderRef B, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateAndReduce(unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceOr(LLVMBuilderRef B, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateOrReduce(unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceXor(LLVMBuilderRef B, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateXorReduce(unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceMin(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) { + return wrap(unwrap(B)->CreateIntMinReduce(unwrap(Src), IsSigned)); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceMax(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) { + return wrap(unwrap(B)->CreateIntMaxReduce(unwrap(Src), IsSigned)); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceFMin(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { + Instruction *I = unwrap(B)->CreateFPMinReduce(unwrap(Src)); + I->setHasNoNaNs(NoNaN); + return wrap(I); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceFMax(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { + Instruction *I = unwrap(B)->CreateFPMaxReduce(unwrap(Src)); + I->setHasNoNaNs(NoNaN); + return wrap(I); +} + +extern "C" LLVMValueRef +LLVMRustBuildMinNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { + return wrap(unwrap(B)->CreateMinNum(unwrap(LHS),unwrap(RHS))); +} +extern "C" LLVMValueRef +LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { + return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS),unwrap(RHS))); +} + +// This struct contains all necessary info about a symbol exported from a DLL. +struct LLVMRustCOFFShortExport { + const char* name; + bool ordinal_present; + // The value of `ordinal` is only meaningful if `ordinal_present` is true. + uint16_t ordinal; +}; + +// Machine must be a COFF machine type, as defined in PE specs. +extern "C" LLVMRustResult LLVMRustWriteImportLibrary( + const char* ImportName, + const char* Path, + const LLVMRustCOFFShortExport* Exports, + size_t NumExports, + uint16_t Machine, + bool MinGW) +{ + std::vector ConvertedExports; + ConvertedExports.reserve(NumExports); + + for (size_t i = 0; i < NumExports; ++i) { + bool ordinal_present = Exports[i].ordinal_present; + uint16_t ordinal = ordinal_present ? Exports[i].ordinal : 0; + ConvertedExports.push_back(llvm::object::COFFShortExport{ + Exports[i].name, // Name + std::string{}, // ExtName + std::string{}, // SymbolName + std::string{}, // AliasTarget + ordinal, // Ordinal + ordinal_present, // Noname + false, // Data + false, // Private + false // Constant + }); + } + + auto Error = llvm::object::writeImportLibrary( + ImportName, + Path, + ConvertedExports, + static_cast(Machine), + MinGW); + if (Error) { + std::string errorString; + llvm::raw_string_ostream stream(errorString); + stream << Error; + stream.flush(); + LLVMRustSetLastError(errorString.c_str()); + return LLVMRustResult::Failure; + } else { + return LLVMRustResult::Success; + } +} + +// Transfers ownership of DiagnosticHandler unique_ptr to the caller. +extern "C" DiagnosticHandler * +LLVMRustContextGetDiagnosticHandler(LLVMContextRef C) { + std::unique_ptr DH = unwrap(C)->getDiagnosticHandler(); + return DH.release(); +} + +// Sets unique_ptr to object of DiagnosticHandler to provide custom diagnostic +// handling. Ownership of the handler is moved to the LLVMContext. +extern "C" void LLVMRustContextSetDiagnosticHandler(LLVMContextRef C, + DiagnosticHandler *DH) { + unwrap(C)->setDiagnosticHandler(std::unique_ptr(DH)); +} + +using LLVMDiagnosticHandlerTy = DiagnosticHandler::DiagnosticHandlerTy; + +// Configures a diagnostic handler that invokes provided callback when a +// backend needs to emit a diagnostic. +// +// When RemarkAllPasses is true, remarks are enabled for all passes. Otherwise +// the RemarkPasses array specifies individual passes for which remarks will be +// enabled. +// +// If RemarkFilePath is not NULL, optimization remarks will be streamed directly into this file, +// bypassing the diagnostics handler. +extern "C" void LLVMRustContextConfigureDiagnosticHandler( + LLVMContextRef C, LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, + void *DiagnosticHandlerContext, bool RemarkAllPasses, + const char * const * RemarkPasses, size_t RemarkPassesLen, + const char * RemarkFilePath, + bool PGOAvailable +) { + + class RustDiagnosticHandler final : public DiagnosticHandler { + public: + RustDiagnosticHandler( + LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, + void *DiagnosticHandlerContext, + bool RemarkAllPasses, + std::vector RemarkPasses, + std::unique_ptr RemarksFile, + std::unique_ptr RemarkStreamer, + std::unique_ptr LlvmRemarkStreamer + ) + : DiagnosticHandlerCallback(DiagnosticHandlerCallback), + DiagnosticHandlerContext(DiagnosticHandlerContext), + RemarkAllPasses(RemarkAllPasses), + RemarkPasses(std::move(RemarkPasses)), + RemarksFile(std::move(RemarksFile)), + RemarkStreamer(std::move(RemarkStreamer)), + LlvmRemarkStreamer(std::move(LlvmRemarkStreamer)) {} + + virtual bool handleDiagnostics(const DiagnosticInfo &DI) override { + // If this diagnostic is one of the optimization remark kinds, we can check if it's enabled + // before emitting it. This can avoid many short-lived allocations when unpacking the + // diagnostic and converting its various C++ strings into rust strings. + // FIXME: some diagnostic infos still allocate before we get here, and avoiding that would be + // good in the future. That will require changing a few call sites in LLVM. + if (auto *OptDiagBase = dyn_cast(&DI)) { + if (OptDiagBase->isEnabled()) { + if (this->LlvmRemarkStreamer) { + this->LlvmRemarkStreamer->emit(*OptDiagBase); + return true; + } + } else { + return true; + } + } + if (DiagnosticHandlerCallback) { + DiagnosticHandlerCallback(DI, DiagnosticHandlerContext); + return true; + } + return false; + } + + bool isAnalysisRemarkEnabled(StringRef PassName) const override { + return isRemarkEnabled(PassName); + } + + bool isMissedOptRemarkEnabled(StringRef PassName) const override { + return isRemarkEnabled(PassName); + } + + bool isPassedOptRemarkEnabled(StringRef PassName) const override { + return isRemarkEnabled(PassName); + } + + bool isAnyRemarkEnabled() const override { + return RemarkAllPasses || !RemarkPasses.empty(); + } + + private: + bool isRemarkEnabled(StringRef PassName) const { + if (RemarkAllPasses) + return true; + + for (auto &Pass : RemarkPasses) + if (Pass == PassName) + return true; + + return false; + } + + LLVMDiagnosticHandlerTy DiagnosticHandlerCallback = nullptr; + void *DiagnosticHandlerContext = nullptr; + + bool RemarkAllPasses = false; + std::vector RemarkPasses; + + // Since LlvmRemarkStreamer contains a pointer to RemarkStreamer, the ordering of the three + // members below is important. + std::unique_ptr RemarksFile; + std::unique_ptr RemarkStreamer; + std::unique_ptr LlvmRemarkStreamer; + }; + + std::vector Passes; + for (size_t I = 0; I != RemarkPassesLen; ++I) + { + Passes.push_back(RemarkPasses[I]); + } + + // We need to hold onto both the streamers and the opened file + std::unique_ptr RemarkFile; + std::unique_ptr RemarkStreamer; + std::unique_ptr LlvmRemarkStreamer; + + if (RemarkFilePath != nullptr) { + if (PGOAvailable) { + // Enable PGO hotness data for remarks, if available + unwrap(C)->setDiagnosticsHotnessRequested(true); + } + + std::error_code EC; + RemarkFile = std::make_unique( + RemarkFilePath, + EC, + llvm::sys::fs::OF_TextWithCRLF + ); + if (EC) { + std::string Error = std::string("Cannot create remark file: ") + + toString(errorCodeToError(EC)); + report_fatal_error(Twine(Error)); + } + + // Do not delete the file after we gather remarks + RemarkFile->keep(); + + auto RemarkSerializer = remarks::createRemarkSerializer( + llvm::remarks::Format::YAML, + remarks::SerializerMode::Separate, + RemarkFile->os() + ); + if (Error E = RemarkSerializer.takeError()) + { + std::string Error = std::string("Cannot create remark serializer: ") + toString(std::move(E)); + report_fatal_error(Twine(Error)); + } + RemarkStreamer = std::make_unique(std::move(*RemarkSerializer)); + LlvmRemarkStreamer = std::make_unique(*RemarkStreamer); + } + + unwrap(C)->setDiagnosticHandler(std::make_unique( + DiagnosticHandlerCallback, + DiagnosticHandlerContext, + RemarkAllPasses, + Passes, + std::move(RemarkFile), + std::move(RemarkStreamer), + std::move(LlvmRemarkStreamer) + )); +} + +extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) { + RawRustStringOstream OS(Str); + GlobalValue *GV = unwrap(V); + Mangler().getNameWithPrefix(OS, GV, true); +} + +extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) { + auto *CB = unwrap(CallSite); + switch (CB->getIntrinsicID()) { + case Intrinsic::arm_ldrex: + return 0; + case Intrinsic::arm_strex: + return 1; + } + return -1; +} + +extern "C" bool LLVMRustIsBitcode(char *ptr, size_t len) { + return identify_magic(StringRef(ptr, len)) == file_magic::bitcode; +} + +extern "C" bool LLVMRustIsNonGVFunctionPointerTy(LLVMValueRef V) { + if (unwrap(V)->getType()->isPointerTy()) { + if (auto *GV = dyn_cast(unwrap(V))) { + if (GV->getValueType()->isFunctionTy()) + return false; + } + return true; + } + return false; +} + +extern "C" bool LLVMRustLLVMHasZlibCompressionForDebugSymbols() { +#if LLVM_VERSION_GE(16, 0) + return llvm::compression::zlib::isAvailable(); +#else + return false; +#endif +} + +extern "C" bool LLVMRustLLVMHasZstdCompressionForDebugSymbols() { +#if LLVM_VERSION_GE(16, 0) + return llvm::compression::zstd::isAvailable(); +#else + return false; +#endif +} diff --git a/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp new file mode 100644 index 00000000000..bf00d11edf6 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp @@ -0,0 +1,96 @@ +// Derived from code in LLVM, which is: +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// Derived from: +// * https://github.com/llvm/llvm-project/blob/8ef3e895ad8ab1724e2b87cabad1dacdc7a397a3/llvm/include/llvm/Object/ArchiveWriter.h +// * https://github.com/llvm/llvm-project/blob/8ef3e895ad8ab1724e2b87cabad1dacdc7a397a3/llvm/lib/Object/ArchiveWriter.cpp + +#include "llvm/ADT/SmallString.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/Object/ObjectFile.h" + +using namespace llvm; +using namespace llvm::sys; +using namespace llvm::object; + +static bool isArchiveSymbol(const object::BasicSymbolRef &S) { + Expected SymFlagsOrErr = S.getFlags(); + if (!SymFlagsOrErr) + // FIXME: Actually report errors helpfully. + report_fatal_error(SymFlagsOrErr.takeError()); + if (*SymFlagsOrErr & object::SymbolRef::SF_FormatSpecific) + return false; + if (!(*SymFlagsOrErr & object::SymbolRef::SF_Global)) + return false; + if (*SymFlagsOrErr & object::SymbolRef::SF_Undefined) + return false; + return true; +} + +typedef void *(*LLVMRustGetSymbolsCallback)(void *, const char *); +typedef void *(*LLVMRustGetSymbolsErrorCallback)(const char *); + +// Note: This is implemented in C++ instead of using the C api from Rust as IRObjectFile doesn't +// implement getSymbolName, only printSymbolName, which is inaccessible from the C api. +extern "C" void *LLVMRustGetSymbols( + char *BufPtr, size_t BufLen, void *State, LLVMRustGetSymbolsCallback Callback, + LLVMRustGetSymbolsErrorCallback ErrorCallback) { + std::unique_ptr Buf = + MemoryBuffer::getMemBuffer(StringRef(BufPtr, BufLen), StringRef("LLVMRustGetSymbolsObject"), + false); + SmallString<0> SymNameBuf; + raw_svector_ostream SymName(SymNameBuf); + + // In the scenario when LLVMContext is populated SymbolicFile will contain a + // reference to it, thus SymbolicFile should be destroyed first. + LLVMContext Context; + std::unique_ptr Obj; + + const file_magic Type = identify_magic(Buf->getBuffer()); + if (!object::SymbolicFile::isSymbolicFile(Type, &Context)) { + return 0; + } + + if (Type == file_magic::bitcode) { + auto ObjOrErr = object::SymbolicFile::createSymbolicFile( + Buf->getMemBufferRef(), file_magic::bitcode, &Context); + if (!ObjOrErr) { + Error E = ObjOrErr.takeError(); + SmallString<0> ErrorBuf; + raw_svector_ostream Error(ErrorBuf); + Error << E << '\0'; + return ErrorCallback(Error.str().data()); + } + Obj = std::move(*ObjOrErr); + } else { + auto ObjOrErr = object::SymbolicFile::createSymbolicFile(Buf->getMemBufferRef()); + if (!ObjOrErr) { + Error E = ObjOrErr.takeError(); + SmallString<0> ErrorBuf; + raw_svector_ostream Error(ErrorBuf); + Error << E << '\0'; + return ErrorCallback(Error.str().data()); + } + Obj = std::move(*ObjOrErr); + } + + + for (const object::BasicSymbolRef &S : Obj->symbols()) { + if (!isArchiveSymbol(S)) + continue; + if (Error E = S.printName(SymName)) { + SmallString<0> ErrorBuf; + raw_svector_ostream Error(ErrorBuf); + Error << E << '\0'; + return ErrorCallback(Error.str().data()); + } + SymName << '\0'; + if (void *E = Callback(State, SymNameBuf.str().data())) { + return E; + } + SymNameBuf.clear(); + } + return 0; +} diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs new file mode 100644 index 00000000000..ca0aec71052 --- /dev/null +++ b/compiler/rustc_llvm/src/lib.rs @@ -0,0 +1,209 @@ +#![deny(rustc::untranslatable_diagnostic)] +#![deny(rustc::diagnostic_outside_of_impl)] +#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] +#![doc(rust_logo)] +#![feature(rustdoc_internals)] +#![allow(internal_features)] + +// NOTE: This crate only exists to allow linking on mingw targets. + +use libc::{c_char, size_t}; +use std::cell::RefCell; +use std::slice; + +#[repr(C)] +pub struct RustString { + pub bytes: RefCell>, +} + +impl RustString { + pub fn len(&self) -> usize { + self.bytes.borrow().len() + } + + pub fn is_empty(&self) -> bool { + self.bytes.borrow().is_empty() + } +} + +/// Appending to a Rust string -- used by RawRustStringOstream. +#[no_mangle] +pub unsafe extern "C" fn LLVMRustStringWriteImpl( + sr: &RustString, + ptr: *const c_char, + size: size_t, +) { + let slice = slice::from_raw_parts(ptr as *const u8, size); + + sr.bytes.borrow_mut().extend_from_slice(slice); +} + +/// Initialize targets enabled by the build script via `cfg(llvm_component = "...")`. +/// N.B., this function can't be moved to `rustc_codegen_llvm` because of the `cfg`s. +pub fn initialize_available_targets() { + macro_rules! init_target( + ($cfg:meta, $($method:ident),*) => { { + #[cfg($cfg)] + fn init() { + extern "C" { + $(fn $method();)* + } + unsafe { + $($method();)* + } + } + #[cfg(not($cfg))] + fn init() { } + init(); + } } + ); + init_target!( + llvm_component = "x86", + LLVMInitializeX86TargetInfo, + LLVMInitializeX86Target, + LLVMInitializeX86TargetMC, + LLVMInitializeX86AsmPrinter, + LLVMInitializeX86AsmParser + ); + init_target!( + llvm_component = "arm", + LLVMInitializeARMTargetInfo, + LLVMInitializeARMTarget, + LLVMInitializeARMTargetMC, + LLVMInitializeARMAsmPrinter, + LLVMInitializeARMAsmParser + ); + init_target!( + llvm_component = "aarch64", + LLVMInitializeAArch64TargetInfo, + LLVMInitializeAArch64Target, + LLVMInitializeAArch64TargetMC, + LLVMInitializeAArch64AsmPrinter, + LLVMInitializeAArch64AsmParser + ); + init_target!( + llvm_component = "amdgpu", + LLVMInitializeAMDGPUTargetInfo, + LLVMInitializeAMDGPUTarget, + LLVMInitializeAMDGPUTargetMC, + LLVMInitializeAMDGPUAsmPrinter, + LLVMInitializeAMDGPUAsmParser + ); + init_target!( + llvm_component = "avr", + LLVMInitializeAVRTargetInfo, + LLVMInitializeAVRTarget, + LLVMInitializeAVRTargetMC, + LLVMInitializeAVRAsmPrinter, + LLVMInitializeAVRAsmParser + ); + init_target!( + llvm_component = "m68k", + LLVMInitializeM68kTargetInfo, + LLVMInitializeM68kTarget, + LLVMInitializeM68kTargetMC, + LLVMInitializeM68kAsmPrinter, + LLVMInitializeM68kAsmParser + ); + init_target!( + llvm_component = "csky", + LLVMInitializeCSKYTargetInfo, + LLVMInitializeCSKYTarget, + LLVMInitializeCSKYTargetMC, + LLVMInitializeCSKYAsmPrinter, + LLVMInitializeCSKYAsmParser + ); + init_target!( + llvm_component = "loongarch", + LLVMInitializeLoongArchTargetInfo, + LLVMInitializeLoongArchTarget, + LLVMInitializeLoongArchTargetMC, + LLVMInitializeLoongArchAsmPrinter, + LLVMInitializeLoongArchAsmParser + ); + init_target!( + llvm_component = "mips", + LLVMInitializeMipsTargetInfo, + LLVMInitializeMipsTarget, + LLVMInitializeMipsTargetMC, + LLVMInitializeMipsAsmPrinter, + LLVMInitializeMipsAsmParser + ); + init_target!( + llvm_component = "powerpc", + LLVMInitializePowerPCTargetInfo, + LLVMInitializePowerPCTarget, + LLVMInitializePowerPCTargetMC, + LLVMInitializePowerPCAsmPrinter, + LLVMInitializePowerPCAsmParser + ); + init_target!( + llvm_component = "systemz", + LLVMInitializeSystemZTargetInfo, + LLVMInitializeSystemZTarget, + LLVMInitializeSystemZTargetMC, + LLVMInitializeSystemZAsmPrinter, + LLVMInitializeSystemZAsmParser + ); + init_target!( + llvm_component = "jsbackend", + LLVMInitializeJSBackendTargetInfo, + LLVMInitializeJSBackendTarget, + LLVMInitializeJSBackendTargetMC + ); + init_target!( + llvm_component = "msp430", + LLVMInitializeMSP430TargetInfo, + LLVMInitializeMSP430Target, + LLVMInitializeMSP430TargetMC, + LLVMInitializeMSP430AsmPrinter, + LLVMInitializeMSP430AsmParser + ); + init_target!( + llvm_component = "riscv", + LLVMInitializeRISCVTargetInfo, + LLVMInitializeRISCVTarget, + LLVMInitializeRISCVTargetMC, + LLVMInitializeRISCVAsmPrinter, + LLVMInitializeRISCVAsmParser + ); + init_target!( + llvm_component = "sparc", + LLVMInitializeSparcTargetInfo, + LLVMInitializeSparcTarget, + LLVMInitializeSparcTargetMC, + LLVMInitializeSparcAsmPrinter, + LLVMInitializeSparcAsmParser + ); + init_target!( + llvm_component = "nvptx", + LLVMInitializeNVPTXTargetInfo, + LLVMInitializeNVPTXTarget, + LLVMInitializeNVPTXTargetMC, + LLVMInitializeNVPTXAsmPrinter + ); + init_target!( + llvm_component = "hexagon", + LLVMInitializeHexagonTargetInfo, + LLVMInitializeHexagonTarget, + LLVMInitializeHexagonTargetMC, + LLVMInitializeHexagonAsmPrinter, + LLVMInitializeHexagonAsmParser + ); + init_target!( + llvm_component = "webassembly", + LLVMInitializeWebAssemblyTargetInfo, + LLVMInitializeWebAssemblyTarget, + LLVMInitializeWebAssemblyTargetMC, + LLVMInitializeWebAssemblyAsmPrinter, + LLVMInitializeWebAssemblyAsmParser + ); + init_target!( + llvm_component = "bpf", + LLVMInitializeBPFTargetInfo, + LLVMInitializeBPFTarget, + LLVMInitializeBPFTargetMC, + LLVMInitializeBPFAsmPrinter, + LLVMInitializeBPFAsmParser + ); +} From d45a18af13cda220f0e362714e5a28eb5a5392f6 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 29 Nov 2023 16:35:04 +0000 Subject: [PATCH 02/70] revert to rustc version 1.64 --- compiler/rustc_llvm/Cargo.toml | 8 +- compiler/rustc_llvm/build.rs | 67 +- .../llvm-wrapper/ArchiveWrapper.cpp | 21 +- .../llvm-wrapper/CoverageMappingWrapper.cpp | 155 +--- .../rustc_llvm/llvm-wrapper/LLVMWrapper.h | 14 +- .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 821 ++++++++++-------- .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 601 ++++++------- .../rustc_llvm/llvm-wrapper/SymbolWrapper.cpp | 96 -- compiler/rustc_llvm/src/lib.rs | 23 +- 9 files changed, 805 insertions(+), 1001 deletions(-) delete mode 100644 compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp diff --git a/compiler/rustc_llvm/Cargo.toml b/compiler/rustc_llvm/Cargo.toml index 58e219e5a46..34556df3c6d 100644 --- a/compiler/rustc_llvm/Cargo.toml +++ b/compiler/rustc_llvm/Cargo.toml @@ -3,12 +3,12 @@ name = "rustc_llvm" version = "0.0.0" edition = "2021" +[features] +static-libstdcpp = [] +emscripten = [] + [dependencies] -# tidy-alphabetical-start libc = "0.2.73" -# tidy-alphabetical-end [build-dependencies] -# tidy-alphabetical-start cc = "1.0.69" -# tidy-alphabetical-end diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index ed1e8771323..62ef5804dce 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -10,9 +10,7 @@ const OPTIONAL_COMPONENTS: &[&str] = &[ "aarch64", "amdgpu", "avr", - "loongarch", "m68k", - "csky", "mips", "powerpc", "systemz", @@ -60,7 +58,7 @@ fn restore_library_path() { /// Supposed to be used for all variables except those set for build scripts by cargo /// fn tracked_env_var_os + Display>(key: K) -> Option { - println!("cargo:rerun-if-env-changed={key}"); + println!("cargo:rerun-if-env-changed={}", key); env::var_os(key) } @@ -86,7 +84,7 @@ fn output(cmd: &mut Command) -> String { let output = match cmd.stderr(Stdio::inherit()).output() { Ok(status) => status, Err(e) => { - println!("\n\nfailed to execute command: {cmd:?}\nerror: {e}\n\n"); + println!("\n\nfailed to execute command: {:?}\nerror: {}\n\n", cmd, e); std::process::exit(1); } }; @@ -102,7 +100,7 @@ fn output(cmd: &mut Command) -> String { fn main() { for component in REQUIRED_COMPONENTS.iter().chain(OPTIONAL_COMPONENTS.iter()) { - println!("cargo:rustc-check-cfg=values(llvm_component,\"{component}\")"); + println!("cargo:rustc-check-cfg=values(llvm_component,\"{}\")", component); } if tracked_env_var_os("RUST_CHECK").is_some() { @@ -166,12 +164,12 @@ fn main() { for component in REQUIRED_COMPONENTS { if !components.contains(component) { - panic!("require llvm component {component} but wasn't found"); + panic!("require llvm component {} but wasn't found", component); } } for component in components.iter() { - println!("cargo:rustc-cfg=llvm_component=\"{component}\""); + println!("cargo:rustc-cfg=llvm_component=\"{}\"", component); } // Link in our own LLVM shims, compiled with the same flags as LLVM @@ -224,7 +222,6 @@ fn main() { .file("llvm-wrapper/RustWrapper.cpp") .file("llvm-wrapper/ArchiveWrapper.cpp") .file("llvm-wrapper/CoverageMappingWrapper.cpp") - .file("llvm-wrapper/SymbolWrapper.cpp") .file("llvm-wrapper/Linker.cpp") .cpp(true) .cpp_link_stdlib(None) // we handle this below @@ -240,38 +237,11 @@ fn main() { if !is_crossed { cmd.arg("--system-libs"); - } - - // We need libkstat for getHostCPUName on SPARC builds. - // See also: https://github.com/llvm/llvm-project/issues/64186 - if target.starts_with("sparcv9") && target.contains("solaris") { - println!("cargo:rustc-link-lib=kstat"); - } - - if (target.starts_with("arm") && !target.contains("freebsd")) - || target.starts_with("mips-") - || target.starts_with("mipsel-") - || target.starts_with("powerpc-") - { - // 32-bit targets need to link libatomic. - println!("cargo:rustc-link-lib=atomic"); } else if target.contains("windows-gnu") { println!("cargo:rustc-link-lib=shell32"); println!("cargo:rustc-link-lib=uuid"); - } else if target.contains("haiku") - || target.contains("darwin") - || (is_crossed && (target.contains("dragonfly") || target.contains("solaris"))) - { - println!("cargo:rustc-link-lib=z"); - } else if target.contains("netbsd") { - // On NetBSD/i386, gcc and g++ is built for i486 (to maximize backward compat) - // However, LLVM insists on using 64-bit atomics. - // This gives rise to a need to link rust itself with -latomic for these targets - if target.starts_with("i586") || target.starts_with("i686") { - println!("cargo:rustc-link-lib=atomic"); - } + } else if target.contains("netbsd") || target.contains("haiku") || target.contains("darwin") { println!("cargo:rustc-link-lib=z"); - println!("cargo:rustc-link-lib=execinfo"); } cmd.args(&components); @@ -303,7 +273,7 @@ fn main() { } let kind = if name.starts_with("LLVM") { llvm_kind } else { "dylib" }; - println!("cargo:rustc-link-lib={kind}={name}"); + println!("cargo:rustc-link-lib={}={}", kind, name); } // LLVM ldflags @@ -322,11 +292,11 @@ fn main() { println!("cargo:rustc-link-search=native={}", stripped.replace(&host, &target)); } } else if let Some(stripped) = lib.strip_prefix("-LIBPATH:") { - println!("cargo:rustc-link-search=native={stripped}"); + println!("cargo:rustc-link-search=native={}", stripped); } else if let Some(stripped) = lib.strip_prefix("-l") { - println!("cargo:rustc-link-lib={stripped}"); + println!("cargo:rustc-link-lib={}", stripped); } else if let Some(stripped) = lib.strip_prefix("-L") { - println!("cargo:rustc-link-search=native={stripped}"); + println!("cargo:rustc-link-search=native={}", stripped); } } @@ -338,9 +308,9 @@ fn main() { if let Some(s) = llvm_linker_flags { for lib in s.into_string().unwrap().split_whitespace() { if let Some(stripped) = lib.strip_prefix("-l") { - println!("cargo:rustc-link-lib={stripped}"); + println!("cargo:rustc-link-lib={}", stripped); } else if let Some(stripped) = lib.strip_prefix("-L") { - println!("cargo:rustc-link-search=native={stripped}"); + println!("cargo:rustc-link-search=native={}", stripped); } } } @@ -353,12 +323,11 @@ fn main() { } else if target.contains("darwin") || target.contains("freebsd") || target.contains("windows-gnullvm") - || target.contains("aix") { "c++" } else if target.contains("netbsd") && llvm_static_stdcpp.is_some() { // NetBSD uses a separate library when relocation is required - "stdc++_p" + "stdc++_pic" } else if llvm_use_libcxx.is_some() { "c++" } else { @@ -366,10 +335,10 @@ fn main() { }; // RISC-V GCC erroneously requires libatomic for sub-word - // atomic operations. Some BSD uses Clang as its system + // atomic operations. FreeBSD uses Clang as its system // compiler and provides no libatomic in its base system so // does not want this. - if target.starts_with("riscv") && !target.contains("freebsd") && !target.contains("openbsd") { + if !target.contains("freebsd") && target.starts_with("riscv") { println!("cargo:rustc-link-lib=atomic"); } @@ -380,14 +349,14 @@ fn main() { let path = PathBuf::from(s); println!("cargo:rustc-link-search=native={}", path.parent().unwrap().display()); if target.contains("windows") { - println!("cargo:rustc-link-lib=static:-bundle={stdcppname}"); + println!("cargo:rustc-link-lib=static:-bundle={}", stdcppname); } else { - println!("cargo:rustc-link-lib=static={stdcppname}"); + println!("cargo:rustc-link-lib=static={}", stdcppname); } } else if cxxflags.contains("stdlib=libc++") { println!("cargo:rustc-link-lib=c++"); } else { - println!("cargo:rustc-link-lib={stdcppname}"); + println!("cargo:rustc-link-lib={}", stdcppname); } } diff --git a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp index 54fdc84c77d..97541e615da 100644 --- a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp @@ -39,7 +39,6 @@ enum class LLVMRustArchiveKind { BSD, DARWIN, COFF, - AIX_BIG, }; static Archive::Kind fromRust(LLVMRustArchiveKind Kind) { @@ -52,8 +51,6 @@ static Archive::Kind fromRust(LLVMRustArchiveKind Kind) { return Archive::K_DARWIN; case LLVMRustArchiveKind::COFF: return Archive::K_COFF; - case LLVMRustArchiveKind::AIX_BIG: - return Archive::K_AIXBIG; default: report_fatal_error("Bad ArchiveKind."); } @@ -157,6 +154,19 @@ LLVMRustArchiveChildName(LLVMRustArchiveChildConstRef Child, size_t *Size) { return Name.data(); } +extern "C" const char *LLVMRustArchiveChildData(LLVMRustArchiveChildRef Child, + size_t *Size) { + StringRef Buf; + Expected BufOrErr = Child->getBuffer(); + if (!BufOrErr) { + LLVMRustSetLastError(toString(BufOrErr.takeError()).c_str()); + return nullptr; + } + Buf = BufOrErr.get(); + *Size = Buf.size(); + return Buf.data(); +} + extern "C" LLVMRustArchiveMemberRef LLVMRustArchiveMemberNew(char *Filename, char *Name, LLVMRustArchiveChildRef Child) { @@ -203,12 +213,7 @@ LLVMRustWriteArchive(char *Dst, size_t NumMembers, } } -#if LLVM_VERSION_LT(18, 0) auto Result = writeArchive(Dst, Members, WriteSymbtab, Kind, true, false); -#else - auto SymtabMode = WriteSymbtab ? SymtabWritingMode::NormalSymtab : SymtabWritingMode::NoSymtab; - auto Result = writeArchive(Dst, Members, SymtabMode, Kind, true, false); -#endif if (!Result) return LLVMRustResult::Success; LLVMRustSetLastError(toString(std::move(Result)).c_str()); diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp index d61ec0b641c..154f554d607 100644 --- a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp @@ -8,118 +8,35 @@ using namespace llvm; -// FFI equivalent of enum `llvm::coverage::Counter::CounterKind` -// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L97-L99 -enum class LLVMRustCounterKind { - Zero = 0, - CounterValueReference = 1, - Expression = 2, -}; - -// FFI equivalent of struct `llvm::coverage::Counter` -// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L94-L149 -struct LLVMRustCounter { - LLVMRustCounterKind CounterKind; - uint32_t ID; -}; - -static coverage::Counter fromRust(LLVMRustCounter Counter) { - switch (Counter.CounterKind) { - case LLVMRustCounterKind::Zero: - return coverage::Counter::getZero(); - case LLVMRustCounterKind::CounterValueReference: - return coverage::Counter::getCounter(Counter.ID); - case LLVMRustCounterKind::Expression: - return coverage::Counter::getExpression(Counter.ID); - } - report_fatal_error("Bad LLVMRustCounterKind!"); -} - -// FFI equivalent of enum `llvm::coverage::CounterMappingRegion::RegionKind` -// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L213-L234 -enum class LLVMRustCounterMappingRegionKind { - CodeRegion = 0, - ExpansionRegion = 1, - SkippedRegion = 2, - GapRegion = 3, - BranchRegion = 4, -}; - -static coverage::CounterMappingRegion::RegionKind -fromRust(LLVMRustCounterMappingRegionKind Kind) { - switch (Kind) { - case LLVMRustCounterMappingRegionKind::CodeRegion: - return coverage::CounterMappingRegion::CodeRegion; - case LLVMRustCounterMappingRegionKind::ExpansionRegion: - return coverage::CounterMappingRegion::ExpansionRegion; - case LLVMRustCounterMappingRegionKind::SkippedRegion: - return coverage::CounterMappingRegion::SkippedRegion; - case LLVMRustCounterMappingRegionKind::GapRegion: - return coverage::CounterMappingRegion::GapRegion; - case LLVMRustCounterMappingRegionKind::BranchRegion: - return coverage::CounterMappingRegion::BranchRegion; - } - report_fatal_error("Bad LLVMRustCounterMappingRegionKind!"); -} - -// FFI equivalent of struct `llvm::coverage::CounterMappingRegion` -// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L211-L304 struct LLVMRustCounterMappingRegion { - LLVMRustCounter Count; - LLVMRustCounter FalseCount; + coverage::Counter Count; + coverage::Counter FalseCount; uint32_t FileID; uint32_t ExpandedFileID; uint32_t LineStart; uint32_t ColumnStart; uint32_t LineEnd; uint32_t ColumnEnd; - LLVMRustCounterMappingRegionKind Kind; + coverage::CounterMappingRegion::RegionKind Kind; }; -// FFI equivalent of enum `llvm::coverage::CounterExpression::ExprKind` -// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L154 -enum class LLVMRustCounterExprKind { - Subtract = 0, - Add = 1, -}; - -// FFI equivalent of struct `llvm::coverage::CounterExpression` -// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L160 -struct LLVMRustCounterExpression { - LLVMRustCounterExprKind Kind; - LLVMRustCounter LHS; - LLVMRustCounter RHS; -}; - -static coverage::CounterExpression::ExprKind -fromRust(LLVMRustCounterExprKind Kind) { - switch (Kind) { - case LLVMRustCounterExprKind::Subtract: - return coverage::CounterExpression::Subtract; - case LLVMRustCounterExprKind::Add: - return coverage::CounterExpression::Add; - } - report_fatal_error("Bad LLVMRustCounterExprKind!"); -} - extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer( - const char *const Filenames[], + const char* const Filenames[], size_t FilenamesLen, - const size_t *const Lengths, - size_t LengthsLen, RustStringRef BufferOut) { - if (FilenamesLen != LengthsLen) { - report_fatal_error( - "Mismatched lengths in LLVMRustCoverageWriteFilenamesSectionToBuffer"); - } - +#if LLVM_VERSION_GE(13,0) SmallVector FilenameRefs; - FilenameRefs.reserve(FilenamesLen); for (size_t i = 0; i < FilenamesLen; i++) { - FilenameRefs.emplace_back(Filenames[i], Lengths[i]); + FilenameRefs.push_back(std::string(Filenames[i])); + } +#else + SmallVector FilenameRefs; + for (size_t i = 0; i < FilenamesLen; i++) { + FilenameRefs.push_back(StringRef(Filenames[i])); } - auto FilenamesWriter = - coverage::CoverageFilenamesSectionWriter(ArrayRef(FilenameRefs)); +#endif + auto FilenamesWriter = coverage::CoverageFilenamesSectionWriter( + makeArrayRef(FilenameRefs)); RawRustStringOstream OS(BufferOut); FilenamesWriter.write(OS); } @@ -127,51 +44,41 @@ extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer( extern "C" void LLVMRustCoverageWriteMappingToBuffer( const unsigned *VirtualFileMappingIDs, unsigned NumVirtualFileMappingIDs, - const LLVMRustCounterExpression *RustExpressions, + const coverage::CounterExpression *Expressions, unsigned NumExpressions, - const LLVMRustCounterMappingRegion *RustMappingRegions, + LLVMRustCounterMappingRegion *RustMappingRegions, unsigned NumMappingRegions, RustStringRef BufferOut) { // Convert from FFI representation to LLVM representation. SmallVector MappingRegions; MappingRegions.reserve(NumMappingRegions); - for (const auto &Region : ArrayRef( - RustMappingRegions, NumMappingRegions)) { + for (const auto &Region : makeArrayRef(RustMappingRegions, NumMappingRegions)) { MappingRegions.emplace_back( - fromRust(Region.Count), fromRust(Region.FalseCount), - Region.FileID, Region.ExpandedFileID, + Region.Count, Region.FalseCount, Region.FileID, Region.ExpandedFileID, Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd, - fromRust(Region.Kind)); - } - - std::vector Expressions; - Expressions.reserve(NumExpressions); - for (const auto &Expression : - ArrayRef(RustExpressions, NumExpressions)) { - Expressions.emplace_back(fromRust(Expression.Kind), - fromRust(Expression.LHS), - fromRust(Expression.RHS)); + Region.Kind); } - auto CoverageMappingWriter = coverage::CoverageMappingWriter( - ArrayRef(VirtualFileMappingIDs, NumVirtualFileMappingIDs), - Expressions, + makeArrayRef(VirtualFileMappingIDs, NumVirtualFileMappingIDs), + makeArrayRef(Expressions, NumExpressions), MappingRegions); RawRustStringOstream OS(BufferOut); CoverageMappingWriter.write(OS); } -extern "C" LLVMValueRef LLVMRustCoverageCreatePGOFuncNameVar( - LLVMValueRef F, - const char *FuncName, - size_t FuncNameLen) { - StringRef FuncNameRef(FuncName, FuncNameLen); +extern "C" LLVMValueRef LLVMRustCoverageCreatePGOFuncNameVar(LLVMValueRef F, const char *FuncName) { + StringRef FuncNameRef(FuncName); return wrap(createPGOFuncNameVar(*cast(unwrap(F)), FuncNameRef)); } +extern "C" uint64_t LLVMRustCoverageHashCString(const char *StrVal) { + StringRef StrRef(StrVal); + return IndexedInstrProf::ComputeHash(StrRef); +} + extern "C" uint64_t LLVMRustCoverageHashByteArray( const char *Bytes, - size_t NumBytes) { + unsigned NumBytes) { StringRef StrRef(Bytes, NumBytes); return IndexedInstrProf::ComputeHash(StrRef); } @@ -202,5 +109,9 @@ extern "C" void LLVMRustCoverageWriteMappingVarNameToString(RustStringRef Str) { } extern "C" uint32_t LLVMRustCoverageMappingVersion() { +#if LLVM_VERSION_GE(13, 0) return coverage::CovMapVersion::Version6; +#else + return coverage::CovMapVersion::Version5; +#endif } diff --git a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h index 142384e6d0c..015c1c52bef 100644 --- a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h +++ b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h @@ -4,6 +4,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Triple.h" #include "llvm/Analysis/Lint.h" #include "llvm/Analysis/Passes.h" #include "llvm/IR/IRBuilder.h" @@ -14,7 +15,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/FormattedStream.h" -#include "llvm/Support/JSON.h" +#include "llvm/Support/Host.h" #include "llvm/Support/Memory.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/TargetSelect.h" @@ -25,6 +26,7 @@ #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Scalar.h" +#include "llvm/Transforms/Vectorize.h" #define LLVM_VERSION_GE(major, minor) \ (LLVM_VERSION_MAJOR > (major) || \ @@ -42,12 +44,6 @@ #include "llvm/IR/IRPrintingPasses.h" #include "llvm/Linker/Linker.h" -#if LLVM_VERSION_GE(16, 0) -#include "llvm/TargetParser/Triple.h" -#else -#include "llvm/ADT/Triple.h" -#endif - extern "C" void LLVMRustSetLastError(const char *); enum class LLVMRustResult { Success, Failure }; @@ -80,6 +76,7 @@ enum LLVMRustAttribute { OptimizeNone = 24, ReturnsTwice = 25, ReadNone = 26, + InaccessibleMemOnly = 27, SanitizeHWAddress = 28, WillReturn = 29, StackProtectReq = 30, @@ -90,9 +87,10 @@ enum LLVMRustAttribute { NoCfCheck = 35, ShadowCallStack = 36, AllocSize = 37, +#if LLVM_VERSION_GE(15, 0) AllocatedPointer = 38, AllocAlign = 39, - SanitizeSafeStack = 40, +#endif }; typedef struct OpaqueRustString *RustStringRef; diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index d2cd79b456a..0a6bd49992d 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -1,7 +1,5 @@ #include -#include -#include #include #include @@ -10,13 +8,12 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/CodeGen/CommandFlags.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/InitializePasses.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/AssemblyAnnotationWriter.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Verifier.h" -#include "llvm/MC/TargetRegistry.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Object/IRObjectFile.h" #include "llvm/Passes/PassBuilder.h" @@ -24,19 +21,21 @@ #include "llvm/Passes/StandardInstrumentations.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/FileSystem.h" -#if LLVM_VERSION_GE(17, 0) -#include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Support/Host.h" +#if LLVM_VERSION_LT(14, 0) +#include "llvm/Support/TargetRegistry.h" +#else +#include "llvm/MC/TargetRegistry.h" #endif #include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/FunctionImport.h" -#include "llvm/Transforms/IPO/Internalize.h" -#include "llvm/Transforms/IPO/LowerTypeTests.h" -#include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" #include "llvm/Transforms/Utils/AddDiscriminators.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" #include "llvm/LTO/LTO.h" -#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/Bitcode/BitcodeWriterPass.h" +#include "llvm-c/Transforms/PassManagerBuilder.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" @@ -52,25 +51,37 @@ using namespace llvm; -static codegen::RegisterCodeGenFlags CGF; - typedef struct LLVMOpaquePass *LLVMPassRef; typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) -extern "C" void LLVMRustTimeTraceProfilerInitialize() { +extern "C" void LLVMInitializePasses() { + PassRegistry &Registry = *PassRegistry::getPassRegistry(); + initializeCore(Registry); + initializeCodeGen(Registry); + initializeScalarOpts(Registry); + initializeVectorization(Registry); + initializeIPO(Registry); + initializeAnalysis(Registry); + initializeTransformUtils(Registry); + initializeInstCombine(Registry); + initializeInstrumentation(Registry); + initializeTarget(Registry); +} + +extern "C" void LLVMTimeTraceProfilerInitialize() { timeTraceProfilerInitialize( /* TimeTraceGranularity */ 0, /* ProcName */ "rustc"); } -extern "C" void LLVMRustTimeTraceProfilerFinishThread() { +extern "C" void LLVMTimeTraceProfilerFinishThread() { timeTraceProfilerFinishThread(); } -extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) { +extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { StringRef FN(FileName); std::error_code EC; raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); @@ -79,6 +90,167 @@ extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) { timeTraceProfilerCleanup(); } +extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { +#if LLVM_VERSION_LT(15, 0) + StringRef SR(PassName); + PassRegistry *PR = PassRegistry::getPassRegistry(); + + const PassInfo *PI = PR->getPassInfo(SR); + if (PI) { + return wrap(PI->createPass()); + } + return nullptr; +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) { +#if LLVM_VERSION_LT(15, 0) + const bool CompileKernel = false; + const bool UseAfterScope = true; + + return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope)); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) { +#if LLVM_VERSION_LT(15, 0) + const bool CompileKernel = false; + + return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover)); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) { +#if LLVM_VERSION_LT(15, 0) + const bool CompileKernel = false; + + return wrap(createMemorySanitizerLegacyPassPass( + MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel})); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() { +#if LLVM_VERSION_LT(15, 0) + return wrap(createThreadSanitizerLegacyPassPass()); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) { +#if LLVM_VERSION_LT(15, 0) + const bool CompileKernel = false; + + return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover)); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { +#if LLVM_VERSION_LT(15, 0) + assert(RustPass); + Pass *Pass = unwrap(RustPass); + PassManagerBase *PMB = unwrap(PMR); + PMB->add(Pass); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() { +#if LLVM_VERSION_LT(15, 0) + return LLVMPassManagerBuilderCreate(); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) { +#if LLVM_VERSION_LT(15, 0) + LLVMPassManagerBuilderDispose(PMB); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager( + LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { +#if LLVM_VERSION_LT(15, 0) + LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager( + LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { +#if LLVM_VERSION_LT(15, 0) + LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager( + LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) { +#if LLVM_VERSION_LT(15, 0) + LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" +void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( + LLVMPassManagerBuilderRef PMBR, + LLVMPassManagerRef PMR +) { +#if LLVM_VERSION_LT(15, 0) + unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold( + LLVMPassManagerBuilderRef PMB, unsigned Threshold) { +#if LLVM_VERSION_LT(15, 0) + LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" +void LLVMRustAddLastExtensionPasses( + LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { +#if LLVM_VERSION_LT(15, 0) + auto AddExtensionPasses = [Passes, NumPasses]( + const PassManagerBuilder &Builder, PassManagerBase &PM) { + for (size_t I = 0; I < NumPasses; I++) { + PM.add(unwrap(Passes[I])); + } + }; + // Add the passes to both of the pre-finalization extension points, + // so they are run for optimized and non-optimized builds. + unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, + AddExtensionPasses); + unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, + AddExtensionPasses); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + #ifdef LLVM_COMPONENT_X86 #define SUBTARGET_X86 SUBTARGET(X86) #else @@ -109,12 +281,6 @@ extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) { #define SUBTARGET_M68K #endif -#ifdef LLVM_COMPONENT_CSKY -#define SUBTARGET_CSKY SUBTARGET(CSKY) -#else -#define SUBTARGET_CSKY -#endif - #ifdef LLVM_COMPONENT_MIPS #define SUBTARGET_MIPS SUBTARGET(Mips) #else @@ -157,19 +323,12 @@ extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) { #define SUBTARGET_HEXAGON #endif -#ifdef LLVM_COMPONENT_LOONGARCH -#define SUBTARGET_LOONGARCH SUBTARGET(LoongArch) -#else -#define SUBTARGET_LOONGARCH -#endif - #define GEN_SUBTARGETS \ SUBTARGET_X86 \ SUBTARGET_ARM \ SUBTARGET_AARCH64 \ SUBTARGET_AVR \ SUBTARGET_M68K \ - SUBTARGET_CSKY \ SUBTARGET_MIPS \ SUBTARGET_PPC \ SUBTARGET_SYSTEMZ \ @@ -177,7 +336,6 @@ extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) { SUBTARGET_SPARC \ SUBTARGET_HEXAGON \ SUBTARGET_RISCV \ - SUBTARGET_LOONGARCH \ #define SUBTARGET(x) \ namespace llvm { \ @@ -204,12 +362,7 @@ enum class LLVMRustCodeModel { None, }; -#if LLVM_VERSION_LT(16, 0) -static Optional -#else -static std::optional -#endif -fromRust(LLVMRustCodeModel Model) { +static Optional fromRust(LLVMRustCodeModel Model) { switch (Model) { case LLVMRustCodeModel::Tiny: return CodeModel::Tiny; @@ -222,11 +375,7 @@ fromRust(LLVMRustCodeModel Model) { case LLVMRustCodeModel::Large: return CodeModel::Large; case LLVMRustCodeModel::None: -#if LLVM_VERSION_LT(16, 0) return None; -#else - return std::nullopt; -#endif default: report_fatal_error("Bad CodeModel."); } @@ -239,22 +388,16 @@ enum class LLVMRustCodeGenOptLevel { Aggressive, }; -#if LLVM_VERSION_GE(18, 0) - using CodeGenOptLevelEnum = llvm::CodeGenOptLevel; -#else - using CodeGenOptLevelEnum = llvm::CodeGenOpt::Level; -#endif - -static CodeGenOptLevelEnum fromRust(LLVMRustCodeGenOptLevel Level) { +static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { switch (Level) { case LLVMRustCodeGenOptLevel::None: - return CodeGenOptLevelEnum::None; + return CodeGenOpt::None; case LLVMRustCodeGenOptLevel::Less: - return CodeGenOptLevelEnum::Less; + return CodeGenOpt::Less; case LLVMRustCodeGenOptLevel::Default: - return CodeGenOptLevelEnum::Default; + return CodeGenOpt::Default; case LLVMRustCodeGenOptLevel::Aggressive: - return CodeGenOptLevelEnum::Aggressive; + return CodeGenOpt::Aggressive; default: report_fatal_error("Bad CodeGenOptLevel."); } @@ -269,6 +412,10 @@ enum class LLVMRustPassBuilderOptLevel { Oz, }; +#if LLVM_VERSION_LT(14,0) +using OptimizationLevel = PassBuilder::OptimizationLevel; +#endif + static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { switch (Level) { case LLVMRustPassBuilderOptLevel::O0: @@ -315,6 +462,7 @@ static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { report_fatal_error("Bad RelocModel."); } +#ifdef LLVM_RUSTLLVM /// getLongestEntryLength - Return the length of the longest entry in the table. template static size_t getLongestEntryLength(ArrayRef Table) { @@ -324,78 +472,55 @@ static size_t getLongestEntryLength(ArrayRef Table) { return MaxLen; } -using PrintBackendInfo = void(void*, const char* Data, size_t Len); - -extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM, - const char* TargetCPU, - PrintBackendInfo Print, - void* Out) { +extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { const TargetMachine *Target = unwrap(TM); - const Triple::ArchType HostArch = Triple(sys::getDefaultTargetTriple()).getArch(); - const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); - - std::ostringstream Buf; - -#if LLVM_VERSION_GE(17, 0) const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); - const ArrayRef CPUTable = MCInfo->getAllProcessorDescriptions(); -#else - Buf << "Full target CPU help is not supported by this LLVM version.\n\n"; - SubtargetSubTypeKV TargetCPUKV = { TargetCPU, {{}}, {{}} }; - const ArrayRef CPUTable = TargetCPUKV; -#endif + const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); + const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); + const ArrayRef CPUTable = MCInfo->getCPUTable(); unsigned MaxCPULen = getLongestEntryLength(CPUTable); - Buf << "Available CPUs for this target:\n"; - // Don't print the "native" entry when the user specifies --target with a - // different arch since that could be wrong or misleading. + printf("Available CPUs for this target:\n"); if (HostArch == TargetArch) { - MaxCPULen = std::max(MaxCPULen, (unsigned) std::strlen("native")); const StringRef HostCPU = sys::getHostCPUName(); - Buf << " " << std::left << std::setw(MaxCPULen) << "native" - << " - Select the CPU of the current host " - "(currently " << HostCPU.str() << ").\n"; + printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", + MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); } - for (auto &CPU : CPUTable) { - // Compare cpu against current target to label the default - if (strcmp(CPU.Key, TargetCPU) == 0) { - Buf << " " << std::left << std::setw(MaxCPULen) << CPU.Key - << " - This is the default target CPU for the current build target " - "(currently " << Target->getTargetTriple().str() << ")."; - } - else { - Buf << " " << CPU.Key; - } - Buf << "\n"; - } - - const auto &BufString = Buf.str(); - Print(Out, BufString.data(), BufString.size()); + for (auto &CPU : CPUTable) + printf(" %-*s\n", MaxCPULen, CPU.Key); + printf("\n"); } extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { -#ifdef LLVM_RUSTLLVM const TargetMachine *Target = unwrap(TM); const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); const ArrayRef FeatTable = MCInfo->getFeatureTable(); return FeatTable.size(); -#else - return 0; -#endif } extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, const char** Feature, const char** Desc) { -#ifdef LLVM_RUSTLLVM const TargetMachine *Target = unwrap(TM); const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); const ArrayRef FeatTable = MCInfo->getFeatureTable(); const SubtargetFeatureKV Feat = FeatTable[Index]; *Feature = Feat.Key; *Desc = Feat.Desc; -#endif } +#else + +extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { + printf("Target CPU help is not supported by this LLVM version.\n\n"); +} + +extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) { + return 0; +} + +extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {} +#endif + extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { StringRef Name = sys::getHostCPUName(); *len = Name.size(); @@ -415,11 +540,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( bool EmitStackSizeSection, bool RelaxELFRelocations, bool UseInitArray, - const char *SplitDwarfFile, - const char *OutputObjFile, - const char *DebugInfoCompression, - bool ForceEmulatedTls, - const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) { + const char *SplitDwarfFile) { auto OptLevel = fromRust(RustOptLevel); auto RM = fromRust(RustReloc); @@ -434,7 +555,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( return nullptr; } - TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Trip); + TargetOptions Options; Options.FloatABIType = FloatABI::Default; if (UseSoftFloat) { @@ -449,31 +570,9 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( if (SplitDwarfFile) { Options.MCOptions.SplitDwarfFile = SplitDwarfFile; } - if (OutputObjFile) { - Options.ObjectFilenameForDebug = OutputObjFile; - } -#if LLVM_VERSION_GE(16, 0) - if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) { - Options.CompressDebugSections = DebugCompressionType::Zlib; - } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) { - Options.CompressDebugSections = DebugCompressionType::Zstd; - } else if (!strcmp("none", DebugInfoCompression)) { - Options.CompressDebugSections = DebugCompressionType::None; - } -#endif - Options.RelaxELFRelocations = RelaxELFRelocations; Options.UseInitArray = UseInitArray; -#if LLVM_VERSION_LT(17, 0) - if (ForceEmulatedTls) { - Options.ExplicitEmulatedTLS = true; - Options.EmulatedTLS = true; - } -#else - Options.EmulatedTLS = ForceEmulatedTls || Trip.hasDefaultEmulatedTLS(); -#endif - if (TrapUnreachable) { // Tell LLVM to codegen `unreachable` into an explicit trap instruction. // This limits the extent of possible undefined behavior in some cases, as @@ -488,49 +587,54 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( Options.EmitStackSizeSection = EmitStackSizeSection; - - if (ArgsCstrBuff != nullptr) - { - int buffer_offset = 0; - assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); - - const size_t arg0_len = std::strlen(ArgsCstrBuff); - char* arg0 = new char[arg0_len + 1]; - memcpy(arg0, ArgsCstrBuff, arg0_len); - arg0[arg0_len] = '\0'; - buffer_offset += arg0_len + 1; - - const int num_cmd_arg_strings = - std::count(&ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0'); - - std::string* cmd_arg_strings = new std::string[num_cmd_arg_strings]; - for (int i = 0; i < num_cmd_arg_strings; ++i) - { - assert(buffer_offset < ArgsCstrBuffLen); - const int len = std::strlen(ArgsCstrBuff + buffer_offset); - cmd_arg_strings[i] = std::string(&ArgsCstrBuff[buffer_offset], len); - buffer_offset += len + 1; - } - - assert(buffer_offset == ArgsCstrBuffLen); - - Options.MCOptions.Argv0 = arg0; - Options.MCOptions.CommandLineArgs = - llvm::ArrayRef(cmd_arg_strings, num_cmd_arg_strings); - } - TargetMachine *TM = TheTarget->createTargetMachine( Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); return wrap(TM); } extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { + delete unwrap(TM); +} - MCTargetOptions& MCOptions = unwrap(TM)->Options.MCOptions; - delete[] MCOptions.Argv0; - delete[] MCOptions.CommandLineArgs.data(); +extern "C" void LLVMRustConfigurePassManagerBuilder( + LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, + bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, + const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath, + int SizeLevel) { +#if LLVM_VERSION_LT(15, 0) + unwrap(PMBR)->MergeFunctions = MergeFunctions; + unwrap(PMBR)->SLPVectorize = SLPVectorize; + unwrap(PMBR)->OptLevel = fromRust(OptLevel); + unwrap(PMBR)->LoopVectorize = LoopVectorize; + unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; + unwrap(PMBR)->SizeLevel = SizeLevel; + unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0; - delete unwrap(TM); + if (PGOGenPath) { + assert(!PGOUsePath && !PGOSampleUsePath); + unwrap(PMBR)->EnablePGOInstrGen = true; + unwrap(PMBR)->PGOInstrGen = PGOGenPath; + } else if (PGOUsePath) { + assert(!PGOSampleUsePath); + unwrap(PMBR)->PGOInstrUse = PGOUsePath; + } else if (PGOSampleUsePath) { + unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath; + } +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` +// field of a PassManagerBuilder, we expose our own method of doing so. +extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, + LLVMModuleRef M, + bool DisableSimplifyLibCalls) { + Triple TargetTriple(unwrap(M)->getTargetTriple()); + TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); + if (DisableSimplifyLibCalls) + TLI->disableAllFunctions(); + unwrap(PMBR)->LibraryInfo = TLI; } // Unfortunately, the LLVM C API doesn't provide a way to create the @@ -544,9 +648,30 @@ extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); } +// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over +// all the functions in a module, so we do that manually here. You'll find +// similar code in clang's BackendUtil.cpp file. +extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, + LLVMModuleRef M) { + llvm::legacy::FunctionPassManager *P = + unwrap(PMR); + P->doInitialization(); + + // Upgrade all calls to old intrinsics first. + for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) + UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove + + for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; + ++I) + if (!I->isDeclaration()) + P->run(*I); + + P->doFinalization(); +} + extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { // Initializing the command-line options more than once is not allowed. So, - // check if they've already been initialized. (This could happen if we're + // check if they've already been initialized. (This could happen if we're // being called from rustpkg, for example). If the arguments change, then // that's just kinda unfortunate. static bool Initialized = false; @@ -564,17 +689,9 @@ enum class LLVMRustFileType { static CodeGenFileType fromRust(LLVMRustFileType Type) { switch (Type) { case LLVMRustFileType::AssemblyFile: -#if LLVM_VERSION_GE(18, 0) - return CodeGenFileType::AssemblyFile; -#else return CGFT_AssemblyFile; -#endif case LLVMRustFileType::ObjectFile: -#if LLVM_VERSION_GE(18, 0) - return CodeGenFileType::ObjectFile; -#else return CGFT_ObjectFile; -#endif default: report_fatal_error("Bad FileType."); } @@ -628,14 +745,14 @@ extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmS extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { - if (const auto *Cast = any_cast(&WrappedIr)) - return (*Cast)->getName().str(); - if (const auto *Cast = any_cast(&WrappedIr)) - return (*Cast)->getName().str(); - if (const auto *Cast = any_cast(&WrappedIr)) - return (*Cast)->getName().str(); - if (const auto *Cast = any_cast(&WrappedIr)) - return (*Cast)->getName(); + if (any_isa(WrappedIr)) + return any_cast(WrappedIr)->getName().str(); + if (any_isa(WrappedIr)) + return any_cast(WrappedIr)->getName().str(); + if (any_isa(WrappedIr)) + return any_cast(WrappedIr)->getName().str(); + if (any_isa(WrappedIr)) + return any_cast(WrappedIr)->getName(); return ""; } @@ -686,32 +803,26 @@ enum class LLVMRustOptStage { struct LLVMRustSanitizerOptions { bool SanitizeAddress; bool SanitizeAddressRecover; - bool SanitizeCFI; - bool SanitizeKCFI; bool SanitizeMemory; bool SanitizeMemoryRecover; int SanitizeMemoryTrackOrigins; bool SanitizeThread; bool SanitizeHWAddress; bool SanitizeHWAddressRecover; - bool SanitizeKernelAddress; - bool SanitizeKernelAddressRecover; }; extern "C" LLVMRustResult -LLVMRustOptimize( +LLVMRustOptimizeWithNewPassManager( LLVMModuleRef ModuleRef, LLVMTargetMachineRef TMRef, LLVMRustPassBuilderOptLevel OptLevelRust, LLVMRustOptStage OptStage, - bool IsLinkerPluginLTO, bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, LLVMRustSanitizerOptions *SanitizerOptions, const char *PGOGenPath, const char *PGOUsePath, - bool InstrumentCoverage, const char *InstrProfileOutput, - bool InstrumentGCOV, + bool InstrumentCoverage, bool InstrumentGCOV, const char *PGOSampleUsePath, bool DebugInfoForProfiling, void* LlvmSelfProfiler, LLVMRustSelfProfileBeforePassCallback BeforePassCallback, @@ -734,80 +845,43 @@ LLVMRustOptimize( bool DebugPassManager = false; PassInstrumentationCallbacks PIC; -#if LLVM_VERSION_LT(16, 0) StandardInstrumentations SI(DebugPassManager); -#else - StandardInstrumentations SI(TheModule->getContext(), DebugPassManager); -#endif SI.registerCallbacks(PIC); if (LlvmSelfProfiler){ LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); } -#if LLVM_VERSION_LT(16, 0) Optional PGOOpt; -#else - std::optional PGOOpt; -#endif -#if LLVM_VERSION_GE(17, 0) - auto FS = vfs::getRealFileSystem(); -#endif if (PGOGenPath) { assert(!PGOUsePath && !PGOSampleUsePath); - PGOOpt = PGOOptions(PGOGenPath, "", "", -#if LLVM_VERSION_GE(17, 0) - "", - FS, -#endif - PGOOptions::IRInstr, PGOOptions::NoCSAction, - DebugInfoForProfiling); + PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr, + PGOOptions::NoCSAction, DebugInfoForProfiling); } else if (PGOUsePath) { assert(!PGOSampleUsePath); - PGOOpt = PGOOptions(PGOUsePath, "", "", -#if LLVM_VERSION_GE(17, 0) - "", - FS, -#endif - PGOOptions::IRUse, PGOOptions::NoCSAction, - DebugInfoForProfiling); + PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse, + PGOOptions::NoCSAction, DebugInfoForProfiling); } else if (PGOSampleUsePath) { - PGOOpt = PGOOptions(PGOSampleUsePath, "", "", -#if LLVM_VERSION_GE(17, 0) - "", - FS, -#endif - PGOOptions::SampleUse, PGOOptions::NoCSAction, - DebugInfoForProfiling); + PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse, + PGOOptions::NoCSAction, DebugInfoForProfiling); } else if (DebugInfoForProfiling) { - PGOOpt = PGOOptions("", "", "", -#if LLVM_VERSION_GE(17, 0) - "", - FS, -#endif - PGOOptions::NoAction, PGOOptions::NoCSAction, - DebugInfoForProfiling); + PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction, + PGOOptions::NoCSAction, DebugInfoForProfiling); } +#if LLVM_VERSION_GE(13, 0) PassBuilder PB(TM, PTO, PGOOpt, &PIC); LoopAnalysisManager LAM; FunctionAnalysisManager FAM; CGSCCAnalysisManager CGAM; ModuleAnalysisManager MAM; - - if (LLVMPluginsLen) { - auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); - SmallVector Plugins; - PluginsStr.split(Plugins, ',', -1, false); - for (auto PluginPath: Plugins) { - auto Plugin = PassPlugin::Load(PluginPath.str()); - if (!Plugin) { - LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); - return LLVMRustResult::Failure; - } - Plugin->registerPassBuilderCallbacks(PB); - } - } +#else + PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC); + LoopAnalysisManager LAM(DebugPassManager); + FunctionAnalysisManager FAM(DebugPassManager); + CGSCCAnalysisManager CGAM(DebugPassManager); + ModuleAnalysisManager MAM(DebugPassManager); +#endif FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); @@ -830,18 +904,6 @@ LLVMRustOptimize( std::vector> OptimizerLastEPCallbacks; - if (!IsLinkerPluginLTO - && SanitizerOptions && SanitizerOptions->SanitizeCFI - && !NoPrepopulatePasses) { - PipelineStartEPCallbacks.push_back( - [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr, - /*ImportSummary=*/nullptr, - /*DropTypeTests=*/false)); - } - ); - } - if (VerifyIR) { PipelineStartEPCallbacks.push_back( [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { @@ -860,14 +922,8 @@ LLVMRustOptimize( if (InstrumentCoverage) { PipelineStartEPCallbacks.push_back( - [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) { + [](ModulePassManager &MPM, OptimizationLevel Level) { InstrProfOptions Options; - if (InstrProfileOutput) { - Options.InstrProfileOutput = InstrProfileOutput; - } - // cargo run tests in multhreading mode by default - // so use atomics for coverage counters - Options.Atomic = true; MPM.addPass(InstrProfiling(Options, false)); } ); @@ -878,16 +934,15 @@ LLVMRustOptimize( MemorySanitizerOptions Options( SanitizerOptions->SanitizeMemoryTrackOrigins, SanitizerOptions->SanitizeMemoryRecover, - /*CompileKernel=*/false, - /*EagerChecks=*/true); + /*CompileKernel=*/false); OptimizerLastEPCallbacks.push_back( [Options](ModulePassManager &MPM, OptimizationLevel Level) { -#if LLVM_VERSION_LT(16, 0) +#if LLVM_VERSION_GE(14, 0) MPM.addPass(ModuleMemorySanitizerPass(Options)); - MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); #else MPM.addPass(MemorySanitizerPass(Options)); #endif + MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); } ); } @@ -895,27 +950,36 @@ LLVMRustOptimize( if (SanitizerOptions->SanitizeThread) { OptimizerLastEPCallbacks.push_back( [](ModulePassManager &MPM, OptimizationLevel Level) { +#if LLVM_VERSION_GE(14, 0) MPM.addPass(ModuleThreadSanitizerPass()); +#else + MPM.addPass(ThreadSanitizerPass()); +#endif MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); } ); } - if (SanitizerOptions->SanitizeAddress || SanitizerOptions->SanitizeKernelAddress) { + if (SanitizerOptions->SanitizeAddress) { OptimizerLastEPCallbacks.push_back( [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { - auto CompileKernel = SanitizerOptions->SanitizeKernelAddress; +#if LLVM_VERSION_LT(15, 0) + MPM.addPass(RequireAnalysisPass()); +#endif +#if LLVM_VERSION_GE(14, 0) AddressSanitizerOptions opts = AddressSanitizerOptions{ - CompileKernel, - SanitizerOptions->SanitizeAddressRecover - || SanitizerOptions->SanitizeKernelAddressRecover, + /*CompileKernel=*/false, + SanitizerOptions->SanitizeAddressRecover, /*UseAfterScope=*/true, AsanDetectStackUseAfterReturnMode::Runtime, }; -#if LLVM_VERSION_LT(16, 0) MPM.addPass(ModuleAddressSanitizerPass(opts)); #else - MPM.addPass(AddressSanitizerPass(opts)); + MPM.addPass(ModuleAddressSanitizerPass( + /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover)); + MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( + /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover, + /*UseAfterScope=*/true))); #endif } ); @@ -923,19 +987,42 @@ LLVMRustOptimize( if (SanitizerOptions->SanitizeHWAddress) { OptimizerLastEPCallbacks.push_back( [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +#if LLVM_VERSION_GE(14, 0) HWAddressSanitizerOptions opts( /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, /*DisableOptimization=*/false); MPM.addPass(HWAddressSanitizerPass(opts)); +#else + MPM.addPass(HWAddressSanitizerPass( + /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover)); +#endif } ); } } + if (LLVMPluginsLen) { + auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); + SmallVector Plugins; + PluginsStr.split(Plugins, ',', -1, false); + for (auto PluginPath: Plugins) { + auto Plugin = PassPlugin::Load(PluginPath.str()); + if (!Plugin) { + LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); + continue; + } + Plugin->registerPassBuilderCallbacks(PB); + } + } + +#if LLVM_VERSION_GE(13, 0) ModulePassManager MPM; +#else + ModulePassManager MPM(DebugPassManager); +#endif bool NeedThinLTOBufferPasses = UseThinLTOBuffers; if (!NoPrepopulatePasses) { - // The pre-link pipelines don't support O0 and require using buildO0DefaultPipeline() instead. + // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead. // At the same time, the LTO pipelines do support O0 and using them is required. bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; if (OptLevel == OptimizationLevel::O0 && !IsLTO) { @@ -1121,20 +1208,35 @@ LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { } extern "C" void LLVMRustPrintPasses() { - PassBuilder PB; - PB.printPassNames(outs()); + LLVMInitializePasses(); + struct MyListener : PassRegistrationListener { + void passEnumerate(const PassInfo *Info) { + StringRef PassArg = Info->getPassArgument(); + StringRef PassName = Info->getPassName(); + if (!PassArg.empty()) { + // These unsigned->signed casts could theoretically overflow, but + // realistically never will (and even if, the result is implementation + // defined rather plain UB). + printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), + (int)PassName.size(), PassName.data()); + } + } + } Listener; + + PassRegistry *PR = PassRegistry::getPassRegistry(); + PR->enumerateWith(&Listener); +} + +extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, + bool AddLifetimes) { + unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); } extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, size_t Len) { - auto PreserveFunctions = [=](const GlobalValue &GV) { - // Preserve LLVM-injected, ASAN-related symbols. - // See also https://github.com/rust-lang/rust/issues/113404. - if (GV.getName() == "___asan_globals_registered") { - return true; - } + llvm::legacy::PassManager passes; - // Preserve symbols exported from Rust modules. + auto PreserveFunctions = [=](const GlobalValue &GV) { for (size_t I = 0; I < Len; I++) { if (GV.getName() == Symbols[I]) { return true; @@ -1143,7 +1245,9 @@ extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, return false; }; - internalizeModule(*unwrap(M), PreserveFunctions); + passes.add(llvm::createInternalizePass(PreserveFunctions)); + + passes.run(*unwrap(M)); } extern "C" void @@ -1164,7 +1268,7 @@ extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, LLVMRustCodeModel Model) { auto CM = fromRust(Model); - if (!CM) + if (!CM.hasValue()) return; unwrap(M)->setCodeModel(*CM); } @@ -1217,15 +1321,9 @@ struct LLVMRustThinLTOData { // Not 100% sure what these are, but they impact what's internalized and // what's inlined across modules, I believe. -#if LLVM_VERSION_GE(18, 0) - DenseMap ImportLists; - DenseMap ExportLists; - DenseMap ModuleToDefinedGVSummaries; -#else StringMap ImportLists; StringMap ExportLists; StringMap ModuleToDefinedGVSummaries; -#endif StringMap> ResolvedODR; LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} @@ -1279,11 +1377,7 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, Ret->ModuleMap[module->identifier] = mem_buffer; -#if LLVM_VERSION_GE(18, 0) - if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index)) { -#else if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { -#endif LLVMRustSetLastError(toString(std::move(Err)).c_str()); return nullptr; } @@ -1311,6 +1405,13 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, // Otherwise, we sometimes lose `static` values -- see #60184. computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, deadIsPrevailing, /* ImportEnabled = */ false); + ComputeCrossModuleImport( + Ret->Index, + Ret->ModuleToDefinedGVSummaries, + Ret->ImportLists, + Ret->ExportLists + ); + // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it // impacts the caching. // @@ -1327,29 +1428,23 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, return true; return Prevailing->second == S; }; - ComputeCrossModuleImport( - Ret->Index, - Ret->ModuleToDefinedGVSummaries, -#if LLVM_VERSION_GE(17, 0) - isPrevailing, -#endif - Ret->ImportLists, - Ret->ExportLists - ); - auto recordNewLinkage = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID, GlobalValue::LinkageTypes NewLinkage) { Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; }; +#if LLVM_VERSION_GE(13,0) // Uses FromPrevailing visibility scheme which works for many binary // formats. We probably could and should use ELF visibility scheme for many of // our targets, however. lto::Config conf; thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, Ret->GUIDPreservedSymbols); - +#else + thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, + Ret->GUIDPreservedSymbols); +#endif // Here we calculate an `ExportedGUIDs` set for use in the `isExported` // callback below. This callback below will dictate the linkage for all // summaries in the index, and we basically just only want to ensure that dead @@ -1420,7 +1515,11 @@ extern "C" bool LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { Module &Mod = *unwrap(M); const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); +#if LLVM_VERSION_GE(14, 0) thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); +#else + thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); +#endif return true; } @@ -1470,11 +1569,6 @@ LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, if (WasmCustomSections) WasmCustomSections->eraseFromParent(); - // `llvm.ident` named metadata also gets duplicated. - auto *llvmIdent = (*MOrErr)->getNamedMetadata("llvm.ident"); - if (llvmIdent) - llvmIdent->eraseFromParent(); - return MOrErr; }; bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); @@ -1505,23 +1599,13 @@ LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { { raw_string_ostream OS(Ret->data); { + legacy::PassManager PM; if (is_thin) { - PassBuilder PB; - LoopAnalysisManager LAM; - FunctionAnalysisManager FAM; - CGSCCAnalysisManager CGAM; - ModuleAnalysisManager MAM; - PB.registerModuleAnalyses(MAM); - PB.registerCGSCCAnalyses(CGAM); - PB.registerFunctionAnalyses(FAM); - PB.registerLoopAnalyses(LAM); - PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); - ModulePassManager MPM; - MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr)); - MPM.run(*unwrap(M), MAM); + PM.add(createWriteThinLTOBitcodePass(OS)); } else { - WriteBitcodeToFile(*unwrap(M), OS); + PM.add(createBitcodeWriterPass(OS)); } + PM.run(*unwrap(M)); } } return Ret.release(); @@ -1543,7 +1627,7 @@ LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { } // This is what we used to parse upstream bitcode for actual ThinLTO -// processing. We'll call this once per module optimized through ThinLTO, and +// processing. We'll call this once per module optimized through ThinLTO, and // it'll be called concurrently on many threads. extern "C" LLVMModuleRef LLVMRustParseBitcodeForLTO(LLVMContextRef Context, @@ -1562,36 +1646,87 @@ LLVMRustParseBitcodeForLTO(LLVMContextRef Context, return wrap(std::move(*SrcOrError).release()); } -// Find a section of an object file by name. Fail if the section is missing or -// empty. -extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, - size_t len, - const char *name, - size_t *out_len) { +// Find the bitcode section in the object file data and return it as a slice. +// Fail if the bitcode section is present but empty. +// +// On success, the return value is the pointer to the start of the slice and +// `out_len` is filled with the (non-zero) length. On failure, the return value +// is `nullptr` and `out_len` is set to zero. +extern "C" const char* +LLVMRustGetBitcodeSliceFromObjectData(const char *data, + size_t len, + size_t *out_len) { *out_len = 0; + StringRef Data(data, len); MemoryBufferRef Buffer(Data, ""); // The id is unused. - file_magic Type = identify_magic(Buffer.getBuffer()); - Expected> ObjFileOrError = - object::ObjectFile::createObjectFile(Buffer, Type); - if (!ObjFileOrError) { - LLVMRustSetLastError(toString(ObjFileOrError.takeError()).c_str()); + + Expected BitcodeOrError = + object::IRObjectFile::findBitcodeInMemBuffer(Buffer); + if (!BitcodeOrError) { + LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str()); return nullptr; } - for (const object::SectionRef &Sec : (*ObjFileOrError)->sections()) { - Expected Name = Sec.getName(); - if (Name && *Name == name) { - Expected SectionOrError = Sec.getContents(); - if (!SectionOrError) { - LLVMRustSetLastError(toString(SectionOrError.takeError()).c_str()); - return nullptr; - } - *out_len = SectionOrError->size(); - return SectionOrError->data(); + + *out_len = BitcodeOrError->getBufferSize(); + return BitcodeOrError->getBufferStart(); +} + +// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See +// the comment in `back/lto.rs` for why this exists. +extern "C" void +LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, + DICompileUnit **A, + DICompileUnit **B) { + Module *M = unwrap(Mod); + DICompileUnit **Cur = A; + DICompileUnit **Next = B; + for (DICompileUnit *CU : M->debug_compile_units()) { + *Cur = CU; + Cur = Next; + Next = nullptr; + if (Cur == nullptr) + break; + } +} + +// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See +// the comment in `back/lto.rs` for why this exists. +extern "C" void +LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { + Module *M = unwrap(Mod); + + // If the original source module didn't have a `DICompileUnit` then try to + // merge all the existing compile units. If there aren't actually any though + // then there's not much for us to do so return. + if (Unit == nullptr) { + for (DICompileUnit *CU : M->debug_compile_units()) { + Unit = CU; + break; } + if (Unit == nullptr) + return; } - LLVMRustSetLastError("could not find requested section"); - return nullptr; + + // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and + // process it recursively. Note that we used to specifically iterate over + // instructions to ensure we feed everything into it, but `processModule` + // started doing this the same way in LLVM 7 (commit d769eb36ab2b8). + DebugInfoFinder Finder; + Finder.processModule(*M); + + // After we've found all our debuginfo, rewrite all subprograms to point to + // the same `DICompileUnit`. + for (auto &F : Finder.subprograms()) { + F->replaceUnit(Unit); + } + + // Erase any other references to other `DICompileUnit` instances, the verifier + // will later ensure that we don't actually have any other stale references to + // worry about. + auto *MD = M->getNamedMetadata("llvm.dbg.cu"); + MD->clearOperands(); + MD->addOperand(Unit); } // Computes the LTO cache key for the provided 'ModId' in the given 'Data', diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 7ada2eff593..5f5b5de790e 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1,5 +1,4 @@ #include "LLVMWrapper.h" -#include "llvm/ADT/Statistic.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DiagnosticHandler.h" #include "llvm/IR/DiagnosticInfo.h" @@ -8,24 +7,14 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsARM.h" -#include "llvm/IR/LLVMRemarkStreamer.h" #include "llvm/IR/Mangler.h" -#include "llvm/Remarks/RemarkStreamer.h" -#include "llvm/Remarks/RemarkSerializer.h" -#include "llvm/Remarks/RemarkFormat.h" -#include "llvm/Support/ToolOutputFile.h" -#if LLVM_VERSION_GE(16, 0) -#include "llvm/Support/ModRef.h" -#endif #include "llvm/Object/Archive.h" #include "llvm/Object/COFFImportFile.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Pass.h" -#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/Bitcode/BitcodeWriterPass.h" #include "llvm/Support/Signals.h" -#if LLVM_VERSION_LT(16, 0) #include "llvm/ADT/Optional.h" -#endif #include @@ -69,7 +58,11 @@ static LLVM_THREAD_LOCAL char *LastError; // // Notably it exits the process with code 101, unlike LLVM's default of 1. static void FatalErrorHandler(void *UserData, +#if LLVM_VERSION_LT(14, 0) + const std::string& Reason, +#else const char* Reason, +#endif bool GenCrashDiag) { // Do the same thing that the default error handler does. std::cerr << "LLVM ERROR: " << Reason << std::endl; @@ -112,26 +105,9 @@ extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, unwrap(M)->setTargetTriple(Triple::normalize(Triple)); } -extern "C" const char *LLVMRustPrintPassTimings(size_t *Len) { - std::string buf; - raw_string_ostream SS(buf); - TimerGroup::printAll(SS); - SS.flush(); - *Len = buf.length(); - char *CStr = (char *)malloc(*Len); - memcpy(CStr, buf.c_str(), *Len); - return CStr; -} - -extern "C" const char *LLVMRustPrintStatistics(size_t *Len) { - std::string buf; - raw_string_ostream SS(buf); - llvm::PrintStatistics(SS); - SS.flush(); - *Len = buf.length(); - char *CStr = (char *)malloc(*Len); - memcpy(CStr, buf.c_str(), *Len); - return CStr; +extern "C" void LLVMRustPrintPassTimings() { + raw_fd_ostream OS(2, false); // stderr. + TimerGroup::printAll(OS); } extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, @@ -139,32 +115,6 @@ extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, return wrap(unwrap(M)->getNamedValue(StringRef(Name, NameLen))); } -enum class LLVMRustTailCallKind { - None, - Tail, - MustTail, - NoTail, -}; - -static CallInst::TailCallKind fromRust(LLVMRustTailCallKind Kind) { - switch (Kind) { - case LLVMRustTailCallKind::None: - return CallInst::TailCallKind::TCK_None; - case LLVMRustTailCallKind::Tail: - return CallInst::TailCallKind::TCK_Tail; - case LLVMRustTailCallKind::MustTail: - return CallInst::TailCallKind::TCK_MustTail; - case LLVMRustTailCallKind::NoTail: - return CallInst::TailCallKind::TCK_NoTail; - default: - report_fatal_error("bad CallInst::TailCallKind."); - } -} - -extern "C" void LLVMRustSetTailCallKind(LLVMValueRef Call, LLVMRustTailCallKind TCK) { - unwrap(Call)->setTailCallKind(fromRust(TCK)); -} - extern "C" LLVMValueRef LLVMRustGetOrInsertFunction(LLVMModuleRef M, const char *Name, size_t NameLen, @@ -201,6 +151,10 @@ LLVMRustInsertPrivateGlobal(LLVMModuleRef M, LLVMTypeRef Ty) { nullptr)); } +extern "C" LLVMTypeRef LLVMRustMetadataTypeInContext(LLVMContextRef C) { + return wrap(Type::getMetadataTy(*unwrap(C))); +} + static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { switch (Kind) { case AlwaysInline: @@ -259,6 +213,8 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { return Attribute::ReturnsTwice; case ReadNone: return Attribute::ReadNone; + case InaccessibleMemOnly: + return Attribute::InaccessibleMemOnly; case SanitizeHWAddress: return Attribute::SanitizeHWAddress; case WillReturn: @@ -277,12 +233,12 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { return Attribute::ShadowCallStack; case AllocSize: return Attribute::AllocSize; +#if LLVM_VERSION_GE(15, 0) case AllocatedPointer: return Attribute::AllocatedPointer; case AllocAlign: return Attribute::AllocAlign; - case SanitizeSafeStack: - return Attribute::SafeStack; +#endif } report_fatal_error("bad AttributeKind"); } @@ -290,10 +246,18 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { template static inline void AddAttributes(T *t, unsigned Index, LLVMAttributeRef *Attrs, size_t AttrsLen) { AttributeList PAL = t->getAttributes(); + AttributeList PALNew; +#if LLVM_VERSION_LT(14, 0) + AttrBuilder B; + for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen)) + B.addAttribute(unwrap(Attr)); + PALNew = PAL.addAttributes(t->getContext(), Index, B); +#else AttrBuilder B(t->getContext()); - for (LLVMAttributeRef Attr : ArrayRef(Attrs, AttrsLen)) + for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen)) B.addAttribute(unwrap(Attr)); - AttributeList PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B); + PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B); +#endif t->setAttributes(PALNew); } @@ -338,24 +302,28 @@ extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C, LLVMTy } extern "C" LLVMAttributeRef LLVMRustCreateElementTypeAttr(LLVMContextRef C, LLVMTypeRef Ty) { +#if LLVM_VERSION_GE(15, 0) return wrap(Attribute::get(*unwrap(C), Attribute::ElementType, unwrap(Ty))); +#else + report_fatal_error("Should not be needed on LLVM < 15"); +#endif } extern "C" LLVMAttributeRef LLVMRustCreateUWTableAttr(LLVMContextRef C, bool Async) { +#if LLVM_VERSION_LT(15, 0) + return wrap(Attribute::get(*unwrap(C), Attribute::UWTable)); +#else return wrap(Attribute::getWithUWTableKind( *unwrap(C), Async ? UWTableKind::Async : UWTableKind::Sync)); +#endif } extern "C" LLVMAttributeRef LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) { - return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, -#if LLVM_VERSION_LT(16, 0) - None -#else - std::nullopt -#endif - )); + return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, None)); } +#if LLVM_VERSION_GE(15, 0) + // These values **must** match ffi::AllocKindFlags. // It _happens_ to match the LLVM values of llvm::AllocFnKind, // but that's happenstance and we do explicit conversions before @@ -399,46 +367,15 @@ static llvm::AllocFnKind allocKindFromRust(LLVMRustAllocKindFlags F) { } return AFK; } +#endif extern "C" LLVMAttributeRef LLVMRustCreateAllocKindAttr(LLVMContextRef C, uint64_t AllocKindArg) { +#if LLVM_VERSION_GE(15, 0) return wrap(Attribute::get(*unwrap(C), Attribute::AllocKind, static_cast(allocKindFromRust(static_cast(AllocKindArg))))); -} - -// Simplified representation of `MemoryEffects` across the FFI boundary. -// -// Each variant corresponds to one of the static factory methods on `MemoryEffects`. -enum class LLVMRustMemoryEffects { - None, - ReadOnly, - InaccessibleMemOnly, -}; - -extern "C" LLVMAttributeRef LLVMRustCreateMemoryEffectsAttr(LLVMContextRef C, - LLVMRustMemoryEffects Effects) { -#if LLVM_VERSION_GE(16, 0) - switch (Effects) { - case LLVMRustMemoryEffects::None: - return wrap(Attribute::getWithMemoryEffects(*unwrap(C), MemoryEffects::none())); - case LLVMRustMemoryEffects::ReadOnly: - return wrap(Attribute::getWithMemoryEffects(*unwrap(C), MemoryEffects::readOnly())); - case LLVMRustMemoryEffects::InaccessibleMemOnly: - return wrap(Attribute::getWithMemoryEffects(*unwrap(C), - MemoryEffects::inaccessibleMemOnly())); - default: - report_fatal_error("bad MemoryEffects."); - } #else - switch (Effects) { - case LLVMRustMemoryEffects::None: - return wrap(Attribute::get(*unwrap(C), Attribute::ReadNone)); - case LLVMRustMemoryEffects::ReadOnly: - return wrap(Attribute::get(*unwrap(C), Attribute::ReadOnly)); - case LLVMRustMemoryEffects::InaccessibleMemOnly: - return wrap(Attribute::get(*unwrap(C), Attribute::InaccessibleMemOnly)); - default: - report_fatal_error("bad MemoryEffects."); - } + report_fatal_error( + "allockind attributes are new in LLVM 15 and should not be used on older LLVMs"); #endif } @@ -469,6 +406,51 @@ extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B, return wrap(SI); } +// FIXME: Use the C-API LLVMBuildAtomicCmpXchg and LLVMSetWeak +// once we raise our minimum support to LLVM 10. +extern "C" LLVMValueRef +LLVMRustBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Target, + LLVMValueRef Old, LLVMValueRef Source, + LLVMAtomicOrdering Order, + LLVMAtomicOrdering FailureOrder, LLVMBool Weak) { +#if LLVM_VERSION_GE(13,0) + // Rust probably knows the alignment of the target value and should be able to + // specify something more precise than MaybeAlign here. See also + // https://reviews.llvm.org/D97224 which may be a useful reference. + AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg( + unwrap(Target), unwrap(Old), unwrap(Source), llvm::MaybeAlign(), fromRust(Order), + fromRust(FailureOrder)); +#else + AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg( + unwrap(Target), unwrap(Old), unwrap(Source), fromRust(Order), + fromRust(FailureOrder)); +#endif + ACXI->setWeak(Weak); + return wrap(ACXI); +} + +enum class LLVMRustSynchronizationScope { + SingleThread, + CrossThread, +}; + +static SyncScope::ID fromRust(LLVMRustSynchronizationScope Scope) { + switch (Scope) { + case LLVMRustSynchronizationScope::SingleThread: + return SyncScope::SingleThread; + case LLVMRustSynchronizationScope::CrossThread: + return SyncScope::System; + default: + report_fatal_error("bad SynchronizationScope."); + } +} + +extern "C" LLVMValueRef +LLVMRustBuildAtomicFence(LLVMBuilderRef B, LLVMAtomicOrdering Order, + LLVMRustSynchronizationScope Scope) { + return wrap(unwrap(B)->CreateFence(fromRust(Order), fromRust(Scope))); +} + enum class LLVMRustAsmDialect { Att, Intel, @@ -490,18 +472,36 @@ LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen, char *Constraints, size_t ConstraintsLen, LLVMBool HasSideEffects, LLVMBool IsAlignStack, LLVMRustAsmDialect Dialect, LLVMBool CanThrow) { +#if LLVM_VERSION_GE(13, 0) return wrap(InlineAsm::get(unwrap(Ty), StringRef(AsmString, AsmStringLen), StringRef(Constraints, ConstraintsLen), HasSideEffects, IsAlignStack, fromRust(Dialect), CanThrow)); +#else + return wrap(InlineAsm::get(unwrap(Ty), + StringRef(AsmString, AsmStringLen), + StringRef(Constraints, ConstraintsLen), + HasSideEffects, IsAlignStack, + fromRust(Dialect))); +#endif } extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints, size_t ConstraintsLen) { +#if LLVM_VERSION_LT(15, 0) + return InlineAsm::Verify(unwrap(Ty), + StringRef(Constraints, ConstraintsLen)); +#else // llvm::Error converts to true if it is an error. return !llvm::errorToBool(InlineAsm::verify( unwrap(Ty), StringRef(Constraints, ConstraintsLen))); +#endif +} + +extern "C" void LLVMRustAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm, + size_t AsmLen) { + unwrap(M)->appendModuleInlineAsm(StringRef(Asm, AsmLen)); } typedef DIBuilder *LLVMRustDIBuilderRef; @@ -701,7 +701,6 @@ enum class LLVMRustDebugEmissionKind { NoDebug, FullDebug, LineTablesOnly, - DebugDirectivesOnly, }; static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind) { @@ -712,8 +711,6 @@ static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind) return DICompileUnit::DebugEmissionKind::FullDebug; case LLVMRustDebugEmissionKind::LineTablesOnly: return DICompileUnit::DebugEmissionKind::LineTablesOnly; - case LLVMRustDebugEmissionKind::DebugDirectivesOnly: - return DICompileUnit::DebugEmissionKind::DebugDirectivesOnly; default: report_fatal_error("bad DebugEmissionKind."); } @@ -726,18 +723,10 @@ enum class LLVMRustChecksumKind { SHA256, }; -#if LLVM_VERSION_LT(16, 0) static Optional fromRust(LLVMRustChecksumKind Kind) { -#else -static std::optional fromRust(LLVMRustChecksumKind Kind) { -#endif switch (Kind) { case LLVMRustChecksumKind::None: -#if LLVM_VERSION_LT(16, 0) return None; -#else - return std::nullopt; -#endif case LLVMRustChecksumKind::MD5: return DIFile::ChecksumKind::CSK_MD5; case LLVMRustChecksumKind::SHA1: @@ -772,6 +761,10 @@ extern "C" bool LLVMRustHasModuleFlag(LLVMModuleRef M, const char *Name, return unwrap(M)->getModuleFlag(StringRef(Name, Len)) != nullptr; } +extern "C" LLVMValueRef LLVMRustMetadataAsValue(LLVMContextRef C, LLVMMetadataRef MD) { + return wrap(MetadataAsValue::get(*unwrap(C), unwrap(MD))); +} + extern "C" void LLVMRustGlobalAddMetadata( LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) { unwrap(Global)->addMetadata(Kind, *unwrap(MD)); @@ -809,18 +802,8 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFile( const char *Filename, size_t FilenameLen, const char *Directory, size_t DirectoryLen, LLVMRustChecksumKind CSKind, const char *Checksum, size_t ChecksumLen) { - -#if LLVM_VERSION_LT(16, 0) Optional llvmCSKind = fromRust(CSKind); -#else - std::optional llvmCSKind = fromRust(CSKind); -#endif - -#if LLVM_VERSION_LT(16, 0) Optional> CSInfo{}; -#else - std::optional> CSInfo{}; -#endif if (llvmCSKind) CSInfo.emplace(*llvmCSKind, StringRef{Checksum, ChecksumLen}); return wrap(Builder->createFile(StringRef(Filename, FilenameLen), @@ -859,28 +842,6 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( return wrap(Sub); } -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMethod( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, - const char *LinkageName, size_t LinkageNameLen, - LLVMMetadataRef File, unsigned LineNo, - LLVMMetadataRef Ty, LLVMRustDIFlags Flags, - LLVMRustDISPFlags SPFlags, LLVMMetadataRef TParam) { - DITemplateParameterArray TParams = - DITemplateParameterArray(unwrap(TParam)); - DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); - DINode::DIFlags llvmFlags = fromRust(Flags); - DISubprogram *Sub = Builder->createMethod( - unwrapDI(Scope), - StringRef(Name, NameLen), - StringRef(LinkageName, LinkageNameLen), - unwrapDI(File), LineNo, - unwrapDI(Ty), - 0, 0, nullptr, // VTable params aren't used - llvmFlags, llvmSPFlags, TParams); - return wrap(Sub); -} - extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateBasicType( LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, uint64_t SizeInBits, unsigned Encoding) { @@ -963,33 +924,6 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType( fromRust(Flags), unwrapDI(Ty))); } -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticMemberType( - LLVMRustDIBuilderRef Builder, - LLVMMetadataRef Scope, - const char *Name, - size_t NameLen, - LLVMMetadataRef File, - unsigned LineNo, - LLVMMetadataRef Ty, - LLVMRustDIFlags Flags, - LLVMValueRef val, - uint32_t AlignInBits -) { - return wrap(Builder->createStaticMemberType( - unwrapDI(Scope), - StringRef(Name, NameLen), - unwrapDI(File), - LineNo, - unwrapDI(Ty), - fromRust(Flags), - unwrap(val), -#if LLVM_VERSION_GE(18, 0) - llvm::dwarf::DW_TAG_member, -#endif - AlignInBits - )); -} - extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateLexicalBlock( LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, LLVMMetadataRef File, unsigned Line, unsigned Col) { @@ -1093,9 +1027,8 @@ extern "C" LLVMValueRef LLVMRustDIBuilderInsertDeclareAtEnd( extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerator( LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, - const uint64_t Value[2], unsigned SizeInBits, bool IsUnsigned) { - return wrap(Builder->createEnumerator(StringRef(Name, NameLen), - APSInt(APInt(SizeInBits, ArrayRef(Value, 2)), IsUnsigned))); + int64_t Value, bool IsUnsigned) { + return wrap(Builder->createEnumerator(StringRef(Name, NameLen), Value, IsUnsigned)); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType( @@ -1108,11 +1041,7 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType( unwrapDI(Scope), StringRef(Name, NameLen), unwrapDI(File), LineNumber, SizeInBits, AlignInBits, DINodeArray(unwrapDI(Elements)), - unwrapDI(ClassTy), -#if LLVM_VERSION_GE(18, 0) - /* RunTimeLang */ 0, -#endif - "", IsScoped)); + unwrapDI(ClassTy), "", IsScoped)); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType( @@ -1173,10 +1102,6 @@ extern "C" uint64_t LLVMRustDIBuilderCreateOpPlusUconst() { return dwarf::DW_OP_plus_uconst; } -extern "C" int64_t LLVMRustDIBuilderCreateOpLLVMFragment() { - return dwarf::DW_OP_LLVM_fragment; -} - extern "C" void LLVMRustWriteTypeToString(LLVMTypeRef Ty, RustStringRef Str) { RawRustStringOstream OS(Str); unwrap(Ty)->print(OS); @@ -1197,8 +1122,6 @@ extern "C" void LLVMRustWriteValueToString(LLVMValueRef V, } // LLVMArrayType function does not support 64-bit ElementCount -// FIXME: replace with LLVMArrayType2 when bumped minimal version to llvm-17 -// https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc extern "C" LLVMTypeRef LLVMRustArrayType(LLVMTypeRef ElementTy, uint64_t ElementCount) { return wrap(ArrayType::get(unwrap(ElementTy), ElementCount)); @@ -1327,8 +1250,10 @@ static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) { return LLVMRustDiagnosticKind::Linker; case DK_Unsupported: return LLVMRustDiagnosticKind::Unsupported; +#if LLVM_VERSION_GE(13, 0) case DK_SrcMgr: return LLVMRustDiagnosticKind::SrcMgr; +#endif default: return (Kind >= DK_FirstRemark && Kind <= DK_LastRemark) ? LLVMRustDiagnosticKind::OptimizationRemarkOther @@ -1386,25 +1311,46 @@ extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) { return LLVMBFloatTypeKind; case Type::X86_AMXTyID: return LLVMX86_AMXTypeKind; - default: - { - std::string error; - llvm::raw_string_ostream stream(error); - stream << "Rust does not support the TypeID: " << unwrap(Ty)->getTypeID() - << " for the type: " << *unwrap(Ty); - stream.flush(); - report_fatal_error(error.c_str()); - } +#if LLVM_VERSION_GE(15, 0) && LLVM_VERSION_LT(16, 0) + case Type::DXILPointerTyID: + report_fatal_error("Rust does not support DirectX typed pointers."); + break; +#endif +#if LLVM_VERSION_GE(16, 0) + case Type::TypedPointerTyID: + report_fatal_error("Rust does not support typed pointers."); + break; +#endif } + report_fatal_error("Unhandled TypeID."); } DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef) +#if LLVM_VERSION_LT(13, 0) +using LLVMInlineAsmDiagHandlerTy = LLVMContext::InlineAsmDiagHandlerTy; +#else +using LLVMInlineAsmDiagHandlerTy = void*; +#endif + +extern "C" void LLVMRustSetInlineAsmDiagnosticHandler( + LLVMContextRef C, LLVMInlineAsmDiagHandlerTy H, void *CX) { + // Diagnostic handlers were unified in LLVM change 5de2d189e6ad, so starting + // with LLVM 13 this function is gone. +#if LLVM_VERSION_LT(13, 0) + unwrap(C)->setInlineAsmDiagnosticHandler(H, CX); +#endif +} + extern "C" LLVMSMDiagnosticRef LLVMRustGetSMDiagnostic( LLVMDiagnosticInfoRef DI, unsigned *Cookie) { +#if LLVM_VERSION_GE(13, 0) llvm::DiagnosticInfoSrcMgr *SM = static_cast(unwrap(DI)); *Cookie = SM->getLocCookie(); return wrap(&SM->getSMDiag()); +#else + report_fatal_error("Shouldn't get called on older versions"); +#endif } extern "C" bool LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef, @@ -1454,10 +1400,65 @@ extern "C" bool LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef, return true; } +extern "C" LLVMValueRef LLVMRustBuildCleanupPad(LLVMBuilderRef B, + LLVMValueRef ParentPad, + unsigned ArgCount, + LLVMValueRef *LLArgs, + const char *Name) { + Value **Args = unwrap(LLArgs); + if (ParentPad == nullptr) { + Type *Ty = Type::getTokenTy(unwrap(B)->getContext()); + ParentPad = wrap(Constant::getNullValue(Ty)); + } + return wrap(unwrap(B)->CreateCleanupPad( + unwrap(ParentPad), ArrayRef(Args, ArgCount), Name)); +} + +extern "C" LLVMValueRef LLVMRustBuildCleanupRet(LLVMBuilderRef B, + LLVMValueRef CleanupPad, + LLVMBasicBlockRef UnwindBB) { + CleanupPadInst *Inst = cast(unwrap(CleanupPad)); + return wrap(unwrap(B)->CreateCleanupRet(Inst, unwrap(UnwindBB))); +} + +extern "C" LLVMValueRef +LLVMRustBuildCatchPad(LLVMBuilderRef B, LLVMValueRef ParentPad, + unsigned ArgCount, LLVMValueRef *LLArgs, const char *Name) { + Value **Args = unwrap(LLArgs); + return wrap(unwrap(B)->CreateCatchPad( + unwrap(ParentPad), ArrayRef(Args, ArgCount), Name)); +} + +extern "C" LLVMValueRef LLVMRustBuildCatchRet(LLVMBuilderRef B, + LLVMValueRef Pad, + LLVMBasicBlockRef BB) { + return wrap(unwrap(B)->CreateCatchRet(cast(unwrap(Pad)), + unwrap(BB))); +} + +extern "C" LLVMValueRef LLVMRustBuildCatchSwitch(LLVMBuilderRef B, + LLVMValueRef ParentPad, + LLVMBasicBlockRef BB, + unsigned NumHandlers, + const char *Name) { + if (ParentPad == nullptr) { + Type *Ty = Type::getTokenTy(unwrap(B)->getContext()); + ParentPad = wrap(Constant::getNullValue(Ty)); + } + return wrap(unwrap(B)->CreateCatchSwitch(unwrap(ParentPad), unwrap(BB), + NumHandlers, Name)); +} + +extern "C" void LLVMRustAddHandler(LLVMValueRef CatchSwitchRef, + LLVMBasicBlockRef Handler) { + Value *CatchSwitch = unwrap(CatchSwitchRef); + cast(CatchSwitch)->addHandler(unwrap(Handler)); +} + extern "C" OperandBundleDef *LLVMRustBuildOperandBundleDef(const char *Name, LLVMValueRef *Inputs, unsigned NumInputs) { - return new OperandBundleDef(Name, ArrayRef(unwrap(Inputs), NumInputs)); + return new OperandBundleDef(Name, makeArrayRef(unwrap(Inputs), NumInputs)); } extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { @@ -1466,13 +1467,13 @@ extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, - OperandBundleDef **OpBundles, - unsigned NumOpBundles) { + OperandBundleDef *Bundle) { Value *Callee = unwrap(Fn); FunctionType *FTy = unwrap(Ty); + unsigned Len = Bundle ? 1 : 0; + ArrayRef Bundles = makeArrayRef(Bundle, Len); return wrap(unwrap(B)->CreateCall( - FTy, Callee, ArrayRef(unwrap(Args), NumArgs), - ArrayRef(*OpBundles, NumOpBundles))); + FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), Bundles)); } extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) { @@ -1512,14 +1513,14 @@ extern "C" LLVMValueRef LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, - OperandBundleDef **OpBundles, unsigned NumOpBundles, - const char *Name) { + OperandBundleDef *Bundle, const char *Name) { Value *Callee = unwrap(Fn); FunctionType *FTy = unwrap(Ty); + unsigned Len = Bundle ? 1 : 0; + ArrayRef Bundles = makeArrayRef(Bundle, Len); return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch), - ArrayRef(unwrap(Args), NumArgs), - ArrayRef(*OpBundles, NumOpBundles), - Name)); + makeArrayRef(unwrap(Args), NumArgs), + Bundles, Name)); } extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, @@ -1618,12 +1619,14 @@ extern "C" void LLVMRustSetLinkage(LLVMValueRef V, LLVMSetLinkage(V, fromRust(RustLinkage)); } -extern "C" bool LLVMRustConstIntGetZExtValue(LLVMValueRef CV, uint64_t *value) { - auto C = unwrap(CV); - if (C->getBitWidth() > 64) - return false; - *value = C->getZExtValue(); - return true; +extern "C" LLVMValueRef LLVMRustConstInBoundsGEP2(LLVMTypeRef Ty, + LLVMValueRef ConstantVal, + LLVMValueRef *ConstantIndices, + unsigned NumIndices) { + ArrayRef IdxList(unwrap(ConstantIndices, NumIndices), + NumIndices); + Constant *Val = unwrap(ConstantVal); + return wrap(ConstantExpr::getInBoundsGetElementPtr(unwrap(Ty), Val, IdxList)); } // Returns true if both high and low were successfully set. Fails in case constant wasn’t any of @@ -1633,11 +1636,19 @@ extern "C" bool LLVMRustConstInt128Get(LLVMValueRef CV, bool sext, uint64_t *hig auto C = unwrap(CV); if (C->getBitWidth() > 128) { return false; } APInt AP; +#if LLVM_VERSION_GE(15, 0) if (sext) { AP = C->getValue().sext(128); } else { AP = C->getValue().zext(128); } +#else + if (sext) { + AP = C->getValue().sextOrSelf(128); + } else { + AP = C->getValue().zextOrSelf(128); + } +#endif *low = AP.getLoBits(64).getZExtValue(); *high = AP.getHiBits(64).getZExtValue(); return true; @@ -1677,6 +1688,12 @@ extern "C" LLVMRustVisibility LLVMRustGetVisibility(LLVMValueRef V) { return toRust(LLVMGetVisibility(V)); } +// Oh hey, a binding that makes sense for once? (because LLVM’s own do not) +extern "C" LLVMValueRef LLVMRustBuildIntCast(LLVMBuilderRef B, LLVMValueRef Val, + LLVMTypeRef DestTy, bool isSigned) { + return wrap(unwrap(B)->CreateIntCast(unwrap(Val), unwrap(DestTy), isSigned, "")); +} + extern "C" void LLVMRustSetVisibility(LLVMValueRef V, LLVMRustVisibility RustVisibility) { LLVMSetVisibility(V, fromRust(RustVisibility)); @@ -1695,7 +1712,11 @@ LLVMRustModuleBufferCreate(LLVMModuleRef M) { auto Ret = std::make_unique(); { raw_string_ostream OS(Ret->data); - WriteBitcodeToFile(*unwrap(M), OS); + { + legacy::PassManager PM; + PM.add(createBitcodeWriterPass(OS)); + PM.run(*unwrap(M)); + } } return Ret.release(); } @@ -1721,19 +1742,6 @@ LLVMRustModuleCost(LLVMModuleRef M) { return std::distance(std::begin(f), std::end(f)); } -extern "C" void -LLVMRustModuleInstructionStats(LLVMModuleRef M, RustStringRef Str) -{ - RawRustStringOstream OS(Str); - llvm::json::OStream JOS(OS); - auto Module = unwrap(M); - - JOS.object([&] { - JOS.attribute("module", Module->getName()); - JOS.attribute("total", Module->getInstructionCount()); - }); -} - // Vector reductions: extern "C" LLVMValueRef LLVMRustBuildVectorReduceFAdd(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { @@ -1869,52 +1877,23 @@ using LLVMDiagnosticHandlerTy = DiagnosticHandler::DiagnosticHandlerTy; // When RemarkAllPasses is true, remarks are enabled for all passes. Otherwise // the RemarkPasses array specifies individual passes for which remarks will be // enabled. -// -// If RemarkFilePath is not NULL, optimization remarks will be streamed directly into this file, -// bypassing the diagnostics handler. extern "C" void LLVMRustContextConfigureDiagnosticHandler( LLVMContextRef C, LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, void *DiagnosticHandlerContext, bool RemarkAllPasses, - const char * const * RemarkPasses, size_t RemarkPassesLen, - const char * RemarkFilePath, - bool PGOAvailable -) { + const char * const * RemarkPasses, size_t RemarkPassesLen) { class RustDiagnosticHandler final : public DiagnosticHandler { public: - RustDiagnosticHandler( - LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, - void *DiagnosticHandlerContext, - bool RemarkAllPasses, - std::vector RemarkPasses, - std::unique_ptr RemarksFile, - std::unique_ptr RemarkStreamer, - std::unique_ptr LlvmRemarkStreamer - ) + RustDiagnosticHandler(LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, + void *DiagnosticHandlerContext, + bool RemarkAllPasses, + std::vector RemarkPasses) : DiagnosticHandlerCallback(DiagnosticHandlerCallback), DiagnosticHandlerContext(DiagnosticHandlerContext), RemarkAllPasses(RemarkAllPasses), - RemarkPasses(std::move(RemarkPasses)), - RemarksFile(std::move(RemarksFile)), - RemarkStreamer(std::move(RemarkStreamer)), - LlvmRemarkStreamer(std::move(LlvmRemarkStreamer)) {} + RemarkPasses(RemarkPasses) {} virtual bool handleDiagnostics(const DiagnosticInfo &DI) override { - // If this diagnostic is one of the optimization remark kinds, we can check if it's enabled - // before emitting it. This can avoid many short-lived allocations when unpacking the - // diagnostic and converting its various C++ strings into rust strings. - // FIXME: some diagnostic infos still allocate before we get here, and avoiding that would be - // good in the future. That will require changing a few call sites in LLVM. - if (auto *OptDiagBase = dyn_cast(&DI)) { - if (OptDiagBase->isEnabled()) { - if (this->LlvmRemarkStreamer) { - this->LlvmRemarkStreamer->emit(*OptDiagBase); - return true; - } - } else { - return true; - } - } if (DiagnosticHandlerCallback) { DiagnosticHandlerCallback(DI, DiagnosticHandlerContext); return true; @@ -1955,69 +1934,14 @@ extern "C" void LLVMRustContextConfigureDiagnosticHandler( bool RemarkAllPasses = false; std::vector RemarkPasses; - - // Since LlvmRemarkStreamer contains a pointer to RemarkStreamer, the ordering of the three - // members below is important. - std::unique_ptr RemarksFile; - std::unique_ptr RemarkStreamer; - std::unique_ptr LlvmRemarkStreamer; }; std::vector Passes; for (size_t I = 0; I != RemarkPassesLen; ++I) - { Passes.push_back(RemarkPasses[I]); - } - - // We need to hold onto both the streamers and the opened file - std::unique_ptr RemarkFile; - std::unique_ptr RemarkStreamer; - std::unique_ptr LlvmRemarkStreamer; - - if (RemarkFilePath != nullptr) { - if (PGOAvailable) { - // Enable PGO hotness data for remarks, if available - unwrap(C)->setDiagnosticsHotnessRequested(true); - } - - std::error_code EC; - RemarkFile = std::make_unique( - RemarkFilePath, - EC, - llvm::sys::fs::OF_TextWithCRLF - ); - if (EC) { - std::string Error = std::string("Cannot create remark file: ") + - toString(errorCodeToError(EC)); - report_fatal_error(Twine(Error)); - } - - // Do not delete the file after we gather remarks - RemarkFile->keep(); - - auto RemarkSerializer = remarks::createRemarkSerializer( - llvm::remarks::Format::YAML, - remarks::SerializerMode::Separate, - RemarkFile->os() - ); - if (Error E = RemarkSerializer.takeError()) - { - std::string Error = std::string("Cannot create remark serializer: ") + toString(std::move(E)); - report_fatal_error(Twine(Error)); - } - RemarkStreamer = std::make_unique(std::move(*RemarkSerializer)); - LlvmRemarkStreamer = std::make_unique(*RemarkStreamer); - } unwrap(C)->setDiagnosticHandler(std::make_unique( - DiagnosticHandlerCallback, - DiagnosticHandlerContext, - RemarkAllPasses, - Passes, - std::move(RemarkFile), - std::move(RemarkStreamer), - std::move(LlvmRemarkStreamer) - )); + DiagnosticHandlerCallback, DiagnosticHandlerContext, RemarkAllPasses, Passes)); } extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) { @@ -2026,7 +1950,16 @@ extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) { Mangler().getNameWithPrefix(OS, GV, true); } +// LLVMGetAggregateElement was added in LLVM 15. For earlier LLVM versions just +// use its implementation. +#if LLVM_VERSION_LT(15, 0) +extern "C" LLVMValueRef LLVMGetAggregateElement(LLVMValueRef C, unsigned Idx) { + return wrap(unwrap(C)->getAggregateElement(Idx)); +} +#endif + extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) { +#if LLVM_VERSION_GE(15, 0) auto *CB = unwrap(CallSite); switch (CB->getIntrinsicID()) { case Intrinsic::arm_ldrex: @@ -2034,36 +1967,6 @@ extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) { case Intrinsic::arm_strex: return 1; } - return -1; -} - -extern "C" bool LLVMRustIsBitcode(char *ptr, size_t len) { - return identify_magic(StringRef(ptr, len)) == file_magic::bitcode; -} - -extern "C" bool LLVMRustIsNonGVFunctionPointerTy(LLVMValueRef V) { - if (unwrap(V)->getType()->isPointerTy()) { - if (auto *GV = dyn_cast(unwrap(V))) { - if (GV->getValueType()->isFunctionTy()) - return false; - } - return true; - } - return false; -} - -extern "C" bool LLVMRustLLVMHasZlibCompressionForDebugSymbols() { -#if LLVM_VERSION_GE(16, 0) - return llvm::compression::zlib::isAvailable(); -#else - return false; -#endif -} - -extern "C" bool LLVMRustLLVMHasZstdCompressionForDebugSymbols() { -#if LLVM_VERSION_GE(16, 0) - return llvm::compression::zstd::isAvailable(); -#else - return false; #endif + return -1; } diff --git a/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp deleted file mode 100644 index bf00d11edf6..00000000000 --- a/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp +++ /dev/null @@ -1,96 +0,0 @@ -// Derived from code in LLVM, which is: -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -// Derived from: -// * https://github.com/llvm/llvm-project/blob/8ef3e895ad8ab1724e2b87cabad1dacdc7a397a3/llvm/include/llvm/Object/ArchiveWriter.h -// * https://github.com/llvm/llvm-project/blob/8ef3e895ad8ab1724e2b87cabad1dacdc7a397a3/llvm/lib/Object/ArchiveWriter.cpp - -#include "llvm/ADT/SmallString.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/Object/ObjectFile.h" - -using namespace llvm; -using namespace llvm::sys; -using namespace llvm::object; - -static bool isArchiveSymbol(const object::BasicSymbolRef &S) { - Expected SymFlagsOrErr = S.getFlags(); - if (!SymFlagsOrErr) - // FIXME: Actually report errors helpfully. - report_fatal_error(SymFlagsOrErr.takeError()); - if (*SymFlagsOrErr & object::SymbolRef::SF_FormatSpecific) - return false; - if (!(*SymFlagsOrErr & object::SymbolRef::SF_Global)) - return false; - if (*SymFlagsOrErr & object::SymbolRef::SF_Undefined) - return false; - return true; -} - -typedef void *(*LLVMRustGetSymbolsCallback)(void *, const char *); -typedef void *(*LLVMRustGetSymbolsErrorCallback)(const char *); - -// Note: This is implemented in C++ instead of using the C api from Rust as IRObjectFile doesn't -// implement getSymbolName, only printSymbolName, which is inaccessible from the C api. -extern "C" void *LLVMRustGetSymbols( - char *BufPtr, size_t BufLen, void *State, LLVMRustGetSymbolsCallback Callback, - LLVMRustGetSymbolsErrorCallback ErrorCallback) { - std::unique_ptr Buf = - MemoryBuffer::getMemBuffer(StringRef(BufPtr, BufLen), StringRef("LLVMRustGetSymbolsObject"), - false); - SmallString<0> SymNameBuf; - raw_svector_ostream SymName(SymNameBuf); - - // In the scenario when LLVMContext is populated SymbolicFile will contain a - // reference to it, thus SymbolicFile should be destroyed first. - LLVMContext Context; - std::unique_ptr Obj; - - const file_magic Type = identify_magic(Buf->getBuffer()); - if (!object::SymbolicFile::isSymbolicFile(Type, &Context)) { - return 0; - } - - if (Type == file_magic::bitcode) { - auto ObjOrErr = object::SymbolicFile::createSymbolicFile( - Buf->getMemBufferRef(), file_magic::bitcode, &Context); - if (!ObjOrErr) { - Error E = ObjOrErr.takeError(); - SmallString<0> ErrorBuf; - raw_svector_ostream Error(ErrorBuf); - Error << E << '\0'; - return ErrorCallback(Error.str().data()); - } - Obj = std::move(*ObjOrErr); - } else { - auto ObjOrErr = object::SymbolicFile::createSymbolicFile(Buf->getMemBufferRef()); - if (!ObjOrErr) { - Error E = ObjOrErr.takeError(); - SmallString<0> ErrorBuf; - raw_svector_ostream Error(ErrorBuf); - Error << E << '\0'; - return ErrorCallback(Error.str().data()); - } - Obj = std::move(*ObjOrErr); - } - - - for (const object::BasicSymbolRef &S : Obj->symbols()) { - if (!isArchiveSymbol(S)) - continue; - if (Error E = S.printName(SymName)) { - SmallString<0> ErrorBuf; - raw_svector_ostream Error(ErrorBuf); - Error << E << '\0'; - return ErrorCallback(Error.str().data()); - } - SymName << '\0'; - if (void *E = Callback(State, SymNameBuf.str().data())) { - return E; - } - SymNameBuf.clear(); - } - return 0; -} diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs index ca0aec71052..8eade02a408 100644 --- a/compiler/rustc_llvm/src/lib.rs +++ b/compiler/rustc_llvm/src/lib.rs @@ -1,9 +1,4 @@ -#![deny(rustc::untranslatable_diagnostic)] -#![deny(rustc::diagnostic_outside_of_impl)] #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![doc(rust_logo)] -#![feature(rustdoc_internals)] -#![allow(internal_features)] // NOTE: This crate only exists to allow linking on mingw targets. @@ -33,7 +28,7 @@ pub unsafe extern "C" fn LLVMRustStringWriteImpl( ptr: *const c_char, size: size_t, ) { - let slice = slice::from_raw_parts(ptr as *const u8, size); + let slice = slice::from_raw_parts(ptr as *const u8, size as usize); sr.bytes.borrow_mut().extend_from_slice(slice); } @@ -105,22 +100,6 @@ pub fn initialize_available_targets() { LLVMInitializeM68kAsmPrinter, LLVMInitializeM68kAsmParser ); - init_target!( - llvm_component = "csky", - LLVMInitializeCSKYTargetInfo, - LLVMInitializeCSKYTarget, - LLVMInitializeCSKYTargetMC, - LLVMInitializeCSKYAsmPrinter, - LLVMInitializeCSKYAsmParser - ); - init_target!( - llvm_component = "loongarch", - LLVMInitializeLoongArchTargetInfo, - LLVMInitializeLoongArchTarget, - LLVMInitializeLoongArchTargetMC, - LLVMInitializeLoongArchAsmPrinter, - LLVMInitializeLoongArchAsmParser - ); init_target!( llvm_component = "mips", LLVMInitializeMipsTargetInfo, From 9a92066b02aa6e6a5436e773345f9a0000ff7254 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 29 Nov 2023 16:44:27 +0000 Subject: [PATCH 03/70] add llvm wrappers to cargo build --- Cargo.lock | 1 + Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index fd79975371c..ce4361a8c2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2304,6 +2304,7 @@ dependencies = [ "plc_xml", "pretty_assertions", "regex", + "rustc_llvm", "serde", "serde_json", "serial_test", diff --git a/Cargo.toml b/Cargo.toml index 9e3163e9a4c..fa3f50484f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ plc_source = { path = "./compiler/plc_source" } plc_ast = { path = "./compiler/plc_ast" } plc_util = { path = "./compiler/plc_util" } plc_diagnostics = { path = "./compiler/plc_diagnostics" } +rustc_llvm = { path = "./compiler/rustc_llvm" } logos = "0.12.0" thiserror = "1.0" clap = { version = "3.0", features = ["derive"] } From d533cfe19116dbcf2e751541e0899eaae4cf37fd Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 29 Nov 2023 16:44:46 +0000 Subject: [PATCH 04/70] add llvm wrapper build settings --- scripts/build.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scripts/build.sh b/scripts/build.sh index 536c8389400..3f5f35837cf 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -18,6 +18,9 @@ package=0 target="" CONTAINER_NAME='rust-llvm' +# There may be a better way to set these +export REAL_LIBRARY_PATH_VAR="/usr/lib/llvm-14/lib" +export PATH="$PATH:/usr/lib/llvm-14/bin" source "${BASH_SOURCE%/*}/common.sh" @@ -113,6 +116,25 @@ function run_std_build() { fi } +# # Builds the LLVM coverage wrapper functions +# function run_build_llvm_wrappers() { +# CARGO_OPTIONS=$(set_cargo_options) + +# # Run cargo build with release or debug flags +# echo "Build starting" +# echo "-----------------------------------" +# cmd="cargo build $CARGO_OPTIONS -p " +# log "Running $cmd" +# eval "$cmd" +# echo "-----------------------------------" +# if [[ ${PIPESTATUS[0]} -ne 0 ]]; then +# echo "Build failed" +# exit 1 +# else +# echo "Build done" +# fi +# } + function run_check() { CARGO_OPTIONS=$(set_cargo_options) log "Running cargo check" From ff1a358f857ad6fd8b66c10a9f92fcbdfb469731 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 29 Nov 2023 16:51:01 +0000 Subject: [PATCH 05/70] add readme.md --- compiler/rustc_llvm/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 compiler/rustc_llvm/README.md diff --git a/compiler/rustc_llvm/README.md b/compiler/rustc_llvm/README.md new file mode 100644 index 00000000000..204eb75d94a --- /dev/null +++ b/compiler/rustc_llvm/README.md @@ -0,0 +1,5 @@ +# `rustc-llvm` + +This package serves to wrap some of the LLVM functions which are not natively exposed as part of the LLVM C-API, in a Rust-friendly way. + +This code is taken directly from the [Rust compiler source code (version 1.64.0)](https://github.com/rust-lang/rust/tree/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52), which is the last version of the Rust compiler to use LLVM 14 (which is currently the version used by `ruSTy`). The Rust compiler uses this code to interface with LLVM in order to add code coverage instrumentation to Rust binaries, among other features. From a77f58981ee9e3a3af9d08a5ec7fd4bb675c714d Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 29 Nov 2023 17:06:01 +0000 Subject: [PATCH 06/70] import rustc_codegen_llvm --- compiler/rustc_codegen_llvm/Cargo.toml | 36 + compiler/rustc_codegen_llvm/README.md | 7 + compiler/rustc_codegen_llvm/src/abi.rs | 599 ++++ compiler/rustc_codegen_llvm/src/allocator.rs | 157 + compiler/rustc_codegen_llvm/src/asm.rs | 1037 +++++++ compiler/rustc_codegen_llvm/src/attributes.rs | 449 +++ .../rustc_codegen_llvm/src/back/archive.rs | 383 +++ compiler/rustc_codegen_llvm/src/back/lto.rs | 936 ++++++ .../rustc_codegen_llvm/src/back/profiling.rs | 58 + compiler/rustc_codegen_llvm/src/back/write.rs | 1212 ++++++++ compiler/rustc_codegen_llvm/src/base.rs | 173 ++ compiler/rustc_codegen_llvm/src/builder.rs | 1508 ++++++++++ compiler/rustc_codegen_llvm/src/callee.rs | 194 ++ compiler/rustc_codegen_llvm/src/common.rs | 359 +++ compiler/rustc_codegen_llvm/src/consts.rs | 577 ++++ compiler/rustc_codegen_llvm/src/context.rs | 1014 +++++++ .../src/coverageinfo/mapgen.rs | 334 +++ .../src/coverageinfo/mod.rs | 385 +++ .../src/debuginfo/create_scope_map.rs | 126 + .../rustc_codegen_llvm/src/debuginfo/doc.md | 131 + .../rustc_codegen_llvm/src/debuginfo/gdb.rs | 120 + .../src/debuginfo/metadata.rs | 1618 +++++++++++ .../src/debuginfo/metadata/enums/cpp_like.rs | 514 ++++ .../src/debuginfo/metadata/enums/mod.rs | 437 +++ .../src/debuginfo/metadata/enums/native.rs | 441 +++ .../src/debuginfo/metadata/type_map.rs | 267 ++ .../rustc_codegen_llvm/src/debuginfo/mod.rs | 614 ++++ .../src/debuginfo/namespace.rs | 48 + .../rustc_codegen_llvm/src/debuginfo/utils.rs | 99 + compiler/rustc_codegen_llvm/src/declare.rs | 146 + compiler/rustc_codegen_llvm/src/intrinsic.rs | 1924 +++++++++++++ compiler/rustc_codegen_llvm/src/lib.rs | 442 +++ .../rustc_codegen_llvm/src/llvm/archive_ro.rs | 105 + .../rustc_codegen_llvm/src/llvm/diagnostic.rs | 213 ++ compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2547 +++++++++++++++++ compiler/rustc_codegen_llvm/src/llvm/mod.rs | 318 ++ compiler/rustc_codegen_llvm/src/llvm_util.rs | 562 ++++ compiler/rustc_codegen_llvm/src/mono_item.rs | 150 + compiler/rustc_codegen_llvm/src/type_.rs | 319 +++ compiler/rustc_codegen_llvm/src/type_of.rs | 418 +++ compiler/rustc_codegen_llvm/src/va_arg.rs | 214 ++ compiler/rustc_codegen_llvm/src/value.rs | 32 + 42 files changed, 21223 insertions(+) create mode 100644 compiler/rustc_codegen_llvm/Cargo.toml create mode 100644 compiler/rustc_codegen_llvm/README.md create mode 100644 compiler/rustc_codegen_llvm/src/abi.rs create mode 100644 compiler/rustc_codegen_llvm/src/allocator.rs create mode 100644 compiler/rustc_codegen_llvm/src/asm.rs create mode 100644 compiler/rustc_codegen_llvm/src/attributes.rs create mode 100644 compiler/rustc_codegen_llvm/src/back/archive.rs create mode 100644 compiler/rustc_codegen_llvm/src/back/lto.rs create mode 100644 compiler/rustc_codegen_llvm/src/back/profiling.rs create mode 100644 compiler/rustc_codegen_llvm/src/back/write.rs create mode 100644 compiler/rustc_codegen_llvm/src/base.rs create mode 100644 compiler/rustc_codegen_llvm/src/builder.rs create mode 100644 compiler/rustc_codegen_llvm/src/callee.rs create mode 100644 compiler/rustc_codegen_llvm/src/common.rs create mode 100644 compiler/rustc_codegen_llvm/src/consts.rs create mode 100644 compiler/rustc_codegen_llvm/src/context.rs create mode 100644 compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs create mode 100644 compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/doc.md create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/mod.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/utils.rs create mode 100644 compiler/rustc_codegen_llvm/src/declare.rs create mode 100644 compiler/rustc_codegen_llvm/src/intrinsic.rs create mode 100644 compiler/rustc_codegen_llvm/src/lib.rs create mode 100644 compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs create mode 100644 compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs create mode 100644 compiler/rustc_codegen_llvm/src/llvm/ffi.rs create mode 100644 compiler/rustc_codegen_llvm/src/llvm/mod.rs create mode 100644 compiler/rustc_codegen_llvm/src/llvm_util.rs create mode 100644 compiler/rustc_codegen_llvm/src/mono_item.rs create mode 100644 compiler/rustc_codegen_llvm/src/type_.rs create mode 100644 compiler/rustc_codegen_llvm/src/type_of.rs create mode 100644 compiler/rustc_codegen_llvm/src/va_arg.rs create mode 100644 compiler/rustc_codegen_llvm/src/value.rs diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml new file mode 100644 index 00000000000..f9a5463efcd --- /dev/null +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "rustc_codegen_llvm" +version = "0.0.0" +edition = "2021" + +[lib] +test = false +doctest = false + +[dependencies] +bitflags = "1.0" +cstr = "0.2" +libc = "0.2" +libloading = "0.7.1" +measureme = "10.0.0" +tracing = "0.1" +rustc_middle = { path = "../rustc_middle" } +rustc-demangle = "0.1.21" +rustc_attr = { path = "../rustc_attr" } +rustc_codegen_ssa = { path = "../rustc_codegen_ssa" } +rustc_data_structures = { path = "../rustc_data_structures" } +rustc_errors = { path = "../rustc_errors" } +rustc_fs_util = { path = "../rustc_fs_util" } +rustc_hir = { path = "../rustc_hir" } +rustc_index = { path = "../rustc_index" } +rustc_llvm = { path = "../rustc_llvm" } +rustc_macros = { path = "../rustc_macros" } +rustc_metadata = { path = "../rustc_metadata" } +rustc_query_system = { path = "../rustc_query_system" } +rustc_session = { path = "../rustc_session" } +rustc_serialize = { path = "../rustc_serialize" } +rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } +rustc_target = { path = "../rustc_target" } +smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } +rustc_ast = { path = "../rustc_ast" } +rustc_span = { path = "../rustc_span" } diff --git a/compiler/rustc_codegen_llvm/README.md b/compiler/rustc_codegen_llvm/README.md new file mode 100644 index 00000000000..afec60d017e --- /dev/null +++ b/compiler/rustc_codegen_llvm/README.md @@ -0,0 +1,7 @@ +The `codegen` crate contains the code to convert from MIR into LLVM IR, +and then from LLVM IR into machine code. In general it contains code +that runs towards the end of the compilation process. + +For more information about how codegen works, see the [rustc dev guide]. + +[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/backend/codegen.html diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs new file mode 100644 index 00000000000..9eb3574e77b --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -0,0 +1,599 @@ +use crate::attributes; +use crate::builder::Builder; +use crate::context::CodegenCx; +use crate::llvm::{self, Attribute, AttributePlace}; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; + +use rustc_codegen_ssa::mir::operand::OperandValue; +use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::MemFlags; +use rustc_middle::bug; +use rustc_middle::ty::layout::LayoutOf; +pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; +use rustc_middle::ty::Ty; +use rustc_session::config; +use rustc_target::abi::call::ArgAbi; +pub use rustc_target::abi::call::*; +use rustc_target::abi::{self, HasDataLayout, Int}; +pub use rustc_target::spec::abi::Abi; + +use libc::c_uint; +use smallvec::SmallVec; + +pub trait ArgAttributesExt { + fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); + fn apply_attrs_to_callsite( + &self, + idx: AttributePlace, + cx: &CodegenCx<'_, '_>, + callsite: &Value, + ); +} + +fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { + // LLVM prior to version 12 had known miscompiles in the presence of + // noalias attributes (see #54878), but we don't support earlier + // versions at all anymore. We now enable mutable noalias by default. + cx.tcx.sess.opts.unstable_opts.mutable_noalias.unwrap_or(true) +} + +const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = + [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; + +const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [ + (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), + (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture), + (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), + (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), + (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), +]; + +fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> { + let mut regular = this.regular; + + let mut attrs = SmallVec::new(); + + // ABI-affecting attributes must always be applied + for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES { + if regular.contains(attr) { + attrs.push(llattr.create_attr(cx.llcx)); + } + } + if let Some(align) = this.pointee_align { + attrs.push(llvm::CreateAlignmentAttr(cx.llcx, align.bytes())); + } + match this.arg_ext { + ArgExtension::None => {} + ArgExtension::Zext => attrs.push(llvm::AttributeKind::ZExt.create_attr(cx.llcx)), + ArgExtension::Sext => attrs.push(llvm::AttributeKind::SExt.create_attr(cx.llcx)), + } + + // Only apply remaining attributes when optimizing + if cx.sess().opts.optimize != config::OptLevel::No { + let deref = this.pointee_size.bytes(); + if deref != 0 { + if regular.contains(ArgAttribute::NonNull) { + attrs.push(llvm::CreateDereferenceableAttr(cx.llcx, deref)); + } else { + attrs.push(llvm::CreateDereferenceableOrNullAttr(cx.llcx, deref)); + } + regular -= ArgAttribute::NonNull; + } + for (attr, llattr) in OPTIMIZATION_ATTRIBUTES { + if regular.contains(attr) { + attrs.push(llattr.create_attr(cx.llcx)); + } + } + if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { + attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx)); + } + } + + attrs +} + +impl ArgAttributesExt for ArgAttributes { + fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) { + let attrs = get_attrs(self, cx); + attributes::apply_to_llfn(llfn, idx, &attrs); + } + + fn apply_attrs_to_callsite( + &self, + idx: AttributePlace, + cx: &CodegenCx<'_, '_>, + callsite: &Value, + ) { + let attrs = get_attrs(self, cx); + attributes::apply_to_callsite(callsite, idx, &attrs); + } +} + +pub trait LlvmType { + fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type; +} + +impl LlvmType for Reg { + fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { + match self.kind { + RegKind::Integer => cx.type_ix(self.size.bits()), + RegKind::Float => match self.size.bits() { + 32 => cx.type_f32(), + 64 => cx.type_f64(), + _ => bug!("unsupported float: {:?}", self), + }, + RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()), + } + } +} + +impl LlvmType for CastTarget { + fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { + let rest_ll_unit = self.rest.unit.llvm_type(cx); + let (rest_count, rem_bytes) = if self.rest.unit.size.bytes() == 0 { + (0, 0) + } else { + ( + self.rest.total.bytes() / self.rest.unit.size.bytes(), + self.rest.total.bytes() % self.rest.unit.size.bytes(), + ) + }; + + if self.prefix.iter().all(|x| x.is_none()) { + // Simplify to a single unit when there is no prefix and size <= unit size + if self.rest.total <= self.rest.unit.size { + return rest_ll_unit; + } + + // Simplify to array when all chunks are the same size and type + if rem_bytes == 0 { + return cx.type_array(rest_ll_unit, rest_count); + } + } + + // Create list of fields in the main structure + let mut args: Vec<_> = self + .prefix + .iter() + .flat_map(|option_reg| option_reg.map(|reg| reg.llvm_type(cx))) + .chain((0..rest_count).map(|_| rest_ll_unit)) + .collect(); + + // Append final integer + if rem_bytes != 0 { + // Only integers can be really split further. + assert_eq!(self.rest.unit.kind, RegKind::Integer); + args.push(cx.type_ix(rem_bytes * 8)); + } + + cx.type_struct(&args, false) + } +} + +pub trait ArgAbiExt<'ll, 'tcx> { + fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn store( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + val: &'ll Value, + dst: PlaceRef<'tcx, &'ll Value>, + ); + fn store_fn_arg( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + idx: &mut usize, + dst: PlaceRef<'tcx, &'ll Value>, + ); +} + +impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { + /// Gets the LLVM type for a place of the original Rust type of + /// this argument/return, i.e., the result of `type_of::type_of`. + fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + self.layout.llvm_type(cx) + } + + /// Stores a direct/indirect value described by this ArgAbi into a + /// place for the original Rust type of this argument/return. + /// Can be used for both storing formal arguments into Rust variables + /// or results of call/invoke instructions into their destinations. + fn store( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + val: &'ll Value, + dst: PlaceRef<'tcx, &'ll Value>, + ) { + if self.is_ignore() { + return; + } + if self.is_sized_indirect() { + OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) + } else if self.is_unsized_indirect() { + bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); + } else if let PassMode::Cast(cast) = self.mode { + // FIXME(eddyb): Figure out when the simpler Store is safe, clang + // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. + let can_store_through_cast_ptr = false; + if can_store_through_cast_ptr { + let cast_ptr_llty = bx.type_ptr_to(cast.llvm_type(bx)); + let cast_dst = bx.pointercast(dst.llval, cast_ptr_llty); + bx.store(val, cast_dst, self.layout.align.abi); + } else { + // The actual return type is a struct, but the ABI + // adaptation code has cast it into some scalar type. The + // code that follows is the only reliable way I have + // found to do a transform like i64 -> {i32,i32}. + // Basically we dump the data onto the stack then memcpy it. + // + // Other approaches I tried: + // - Casting rust ret pointer to the foreign type and using Store + // is (a) unsafe if size of foreign type > size of rust type and + // (b) runs afoul of strict aliasing rules, yielding invalid + // assembly under -O (specifically, the store gets removed). + // - Truncating foreign type to correct integral type and then + // bitcasting to the struct type yields invalid cast errors. + + // We instead thus allocate some scratch space... + let scratch_size = cast.size(bx); + let scratch_align = cast.align(bx); + let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); + bx.lifetime_start(llscratch, scratch_size); + + // ... where we first store the value... + bx.store(val, llscratch, scratch_align); + + // ... and then memcpy it to the intended destination. + bx.memcpy( + dst.llval, + self.layout.align.abi, + llscratch, + scratch_align, + bx.const_usize(self.layout.size.bytes()), + MemFlags::empty(), + ); + + bx.lifetime_end(llscratch, scratch_size); + } + } else { + OperandValue::Immediate(val).store(bx, dst); + } + } + + fn store_fn_arg( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + idx: &mut usize, + dst: PlaceRef<'tcx, &'ll Value>, + ) { + let mut next = || { + let val = llvm::get_param(bx.llfn(), *idx as c_uint); + *idx += 1; + val + }; + match self.mode { + PassMode::Ignore => {} + PassMode::Pair(..) => { + OperandValue::Pair(next(), next()).store(bx, dst); + } + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst); + } + PassMode::Direct(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } + | PassMode::Cast(_) => { + let next_arg = next(); + self.store(bx, next_arg, dst); + } + } + } +} + +impl<'ll, 'tcx> ArgAbiMethods<'tcx> for Builder<'_, 'll, 'tcx> { + fn store_fn_arg( + &mut self, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + idx: &mut usize, + dst: PlaceRef<'tcx, Self::Value>, + ) { + arg_abi.store_fn_arg(self, idx, dst) + } + fn store_arg( + &mut self, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + val: &'ll Value, + dst: PlaceRef<'tcx, &'ll Value>, + ) { + arg_abi.store(self, val, dst) + } + fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + arg_abi.memory_ty(self) + } +} + +pub trait FnAbiLlvmExt<'ll, 'tcx> { + fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn llvm_cconv(&self) -> llvm::CallConv; + fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value); + fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value); +} + +impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { + fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + // Ignore "extra" args from the call site for C variadic functions. + // Only the "fixed" args are part of the LLVM function signature. + let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; + + let args_capacity: usize = args.iter().map(|arg| + if arg.pad.is_some() { 1 } else { 0 } + + if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } + ).sum(); + let mut llargument_tys = Vec::with_capacity( + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, + ); + + let llreturn_ty = match self.ret.mode { + PassMode::Ignore => cx.type_void(), + PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), + PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Indirect { .. } => { + llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); + cx.type_void() + } + }; + + for arg in args { + // add padding + if let Some(ty) = arg.pad { + llargument_tys.push(ty.llvm_type(cx)); + } + + let llarg_ty = match arg.mode { + PassMode::Ignore => continue, + PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), + PassMode::Pair(..) => { + llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true)); + llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); + continue; + } + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + let ptr_ty = cx.tcx.mk_mut_ptr(arg.layout.ty); + let ptr_layout = cx.layout_of(ptr_ty); + llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); + llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); + continue; + } + PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { + cx.type_ptr_to(arg.memory_ty(cx)) + } + }; + llargument_tys.push(llarg_ty); + } + + if self.c_variadic { + cx.type_variadic_func(&llargument_tys, llreturn_ty) + } else { + cx.type_func(&llargument_tys, llreturn_ty) + } + } + + fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + unsafe { + llvm::LLVMPointerType( + self.llvm_type(cx), + cx.data_layout().instruction_address_space.0 as c_uint, + ) + } + } + + fn llvm_cconv(&self) -> llvm::CallConv { + match self.conv { + Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, + Conv::RustCold => llvm::ColdCallConv, + Conv::AmdGpuKernel => llvm::AmdGpuKernel, + Conv::AvrInterrupt => llvm::AvrInterrupt, + Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, + Conv::ArmAapcs => llvm::ArmAapcsCallConv, + Conv::Msp430Intr => llvm::Msp430Intr, + Conv::PtxKernel => llvm::PtxKernel, + Conv::X86Fastcall => llvm::X86FastcallCallConv, + Conv::X86Intr => llvm::X86_Intr, + Conv::X86Stdcall => llvm::X86StdcallCallConv, + Conv::X86ThisCall => llvm::X86_ThisCall, + Conv::X86VectorCall => llvm::X86_VectorCall, + Conv::X86_64SysV => llvm::X86_64_SysV, + Conv::X86_64Win64 => llvm::X86_64_Win64, + } + } + + fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { + let mut func_attrs = SmallVec::<[_; 2]>::new(); + if self.ret.layout.abi.is_uninhabited() { + func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); + } + if !self.can_unwind { + func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx)); + } + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs }); + + let mut i = 0; + let mut apply = |attrs: &ArgAttributes| { + attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn); + i += 1; + i - 1 + }; + match self.ret.mode { + PassMode::Direct(ref attrs) => { + attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); + } + PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + assert!(!on_stack); + let i = apply(attrs); + let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); + } + PassMode::Cast(cast) => { + cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); + } + _ => {} + } + for arg in &self.args { + if arg.pad.is_some() { + apply(&ArgAttributes::new()); + } + match arg.mode { + PassMode::Ignore => {} + PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + let i = apply(attrs); + let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); + } + PassMode::Direct(ref attrs) + | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + apply(attrs); + } + PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { + assert!(!on_stack); + apply(attrs); + apply(extra_attrs); + } + PassMode::Pair(ref a, ref b) => { + apply(a); + apply(b); + } + PassMode::Cast(cast) => { + apply(&cast.attrs); + } + } + } + } + + fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { + let mut func_attrs = SmallVec::<[_; 2]>::new(); + if self.ret.layout.abi.is_uninhabited() { + func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx)); + } + if !self.can_unwind { + func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(bx.cx.llcx)); + } + attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &{ func_attrs }); + + let mut i = 0; + let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { + attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite); + i += 1; + i - 1 + }; + match self.ret.mode { + PassMode::Direct(ref attrs) => { + attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); + } + PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + assert!(!on_stack); + let i = apply(bx.cx, attrs); + let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); + attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); + } + PassMode::Cast(cast) => { + cast.attrs.apply_attrs_to_callsite( + llvm::AttributePlace::ReturnValue, + &bx.cx, + callsite, + ); + } + _ => {} + } + if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { + // If the value is a boolean, the range is 0..2 and that ultimately + // become 0..0 when the type becomes i1, which would be rejected + // by the LLVM verifier. + if let Int(..) = scalar.primitive() { + if !scalar.is_bool() && !scalar.is_always_valid(bx) { + bx.range_metadata(callsite, scalar.valid_range(bx)); + } + } + } + for arg in &self.args { + if arg.pad.is_some() { + apply(bx.cx, &ArgAttributes::new()); + } + match arg.mode { + PassMode::Ignore => {} + PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + let i = apply(bx.cx, attrs); + let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Argument(i), + &[byval], + ); + } + PassMode::Direct(ref attrs) + | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + apply(bx.cx, attrs); + } + PassMode::Indirect { + ref attrs, + extra_attrs: Some(ref extra_attrs), + on_stack: _, + } => { + apply(bx.cx, attrs); + apply(bx.cx, extra_attrs); + } + PassMode::Pair(ref a, ref b) => { + apply(bx.cx, a); + apply(bx.cx, b); + } + PassMode::Cast(cast) => { + apply(bx.cx, &cast.attrs); + } + } + } + + let cconv = self.llvm_cconv(); + if cconv != llvm::CCallConv { + llvm::SetInstructionCallConv(callsite, cconv); + } + + if self.conv == Conv::CCmseNonSecureCall { + // This will probably get ignored on all targets but those supporting the TrustZone-M + // extension (thumbv8m targets). + let cmse_nonsecure_call = llvm::CreateAttrString(bx.cx.llcx, "cmse_nonsecure_call"); + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Function, + &[cmse_nonsecure_call], + ); + } + + // Some intrinsics require that an elementtype attribute (with the pointee type of a + // pointer argument) is added to the callsite. + let element_type_index = unsafe { llvm::LLVMRustGetElementTypeArgIndex(callsite) }; + if element_type_index >= 0 { + let arg_ty = self.args[element_type_index as usize].layout.ty; + let pointee_ty = arg_ty.builtin_deref(true).expect("Must be pointer argument").ty; + let element_type_attr = unsafe { + llvm::LLVMRustCreateElementTypeAttr(bx.llcx, bx.layout_of(pointee_ty).llvm_type(bx)) + }; + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Argument(element_type_index as u32), + &[element_type_attr], + ); + } + } +} + +impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value) { + fn_abi.apply_attrs_callsite(self, callsite) + } + + fn get_param(&mut self, index: usize) -> Self::Value { + llvm::get_param(self.llfn(), index as c_uint) + } +} diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs new file mode 100644 index 00000000000..72961ae888e --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -0,0 +1,157 @@ +use crate::attributes; +use libc::c_uint; +use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; +use rustc_middle::bug; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::{DebugInfo, OomStrategy}; +use rustc_span::symbol::sym; + +use crate::debuginfo; +use crate::llvm::{self, False, True}; +use crate::ModuleLlvm; + +pub(crate) unsafe fn codegen( + tcx: TyCtxt<'_>, + module_llvm: &mut ModuleLlvm, + module_name: &str, + kind: AllocatorKind, + has_alloc_error_handler: bool, +) { + let llcx = &*module_llvm.llcx; + let llmod = module_llvm.llmod(); + let usize = match tcx.sess.target.pointer_width { + 16 => llvm::LLVMInt16TypeInContext(llcx), + 32 => llvm::LLVMInt32TypeInContext(llcx), + 64 => llvm::LLVMInt64TypeInContext(llcx), + tws => bug!("Unsupported target word size for int: {}", tws), + }; + let i8 = llvm::LLVMInt8TypeInContext(llcx); + let i8p = llvm::LLVMPointerType(i8, 0); + let void = llvm::LLVMVoidTypeInContext(llcx); + + for method in ALLOCATOR_METHODS { + let mut args = Vec::with_capacity(method.inputs.len()); + for ty in method.inputs.iter() { + match *ty { + AllocatorTy::Layout => { + args.push(usize); // size + args.push(usize); // align + } + AllocatorTy::Ptr => args.push(i8p), + AllocatorTy::Usize => args.push(usize), + + AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + } + } + let output = match method.output { + AllocatorTy::ResultPtr => Some(i8p), + AllocatorTy::Unit => None, + + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; + let ty = llvm::LLVMFunctionType( + output.unwrap_or(void), + args.as_ptr(), + args.len() as c_uint, + False, + ); + let name = format!("__rust_{}", method.name); + let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); + + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + if tcx.sess.must_emit_unwind_tables() { + let uwtable = attributes::uwtable_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); + } + + let callee = kind.fn_name(method.name); + let callee = + llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); + llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); + + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); + + let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); + llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); + let args = args + .iter() + .enumerate() + .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) + .collect::>(); + let ret = llvm::LLVMRustBuildCall( + llbuilder, + ty, + callee, + args.as_ptr(), + args.len() as c_uint, + None, + ); + llvm::LLVMSetTailCall(ret, True); + if output.is_some() { + llvm::LLVMBuildRet(llbuilder, ret); + } else { + llvm::LLVMBuildRetVoid(llbuilder); + } + llvm::LLVMDisposeBuilder(llbuilder); + } + + // rust alloc error handler + let args = [usize, usize]; // size, align + + let ty = llvm::LLVMFunctionType(void, args.as_ptr(), args.len() as c_uint, False); + let name = "__rust_alloc_error_handler"; + let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); + // -> ! DIFlagNoReturn + let no_return = llvm::AttributeKind::NoReturn.create_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[no_return]); + + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + if tcx.sess.must_emit_unwind_tables() { + let uwtable = attributes::uwtable_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); + } + + let kind = if has_alloc_error_handler { AllocatorKind::Global } else { AllocatorKind::Default }; + let callee = kind.fn_name(sym::oom); + let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); + // -> ! DIFlagNoReturn + attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); + llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); + + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); + + let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); + llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); + let args = args + .iter() + .enumerate() + .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) + .collect::>(); + let ret = + llvm::LLVMRustBuildCall(llbuilder, ty, callee, args.as_ptr(), args.len() as c_uint, None); + llvm::LLVMSetTailCall(ret, True); + llvm::LLVMBuildRetVoid(llbuilder); + llvm::LLVMDisposeBuilder(llbuilder); + + // __rust_alloc_error_handler_should_panic + let name = OomStrategy::SYMBOL; + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); + } + let val = tcx.sess.opts.unstable_opts.oom.should_panic(); + let llval = llvm::LLVMConstInt(i8, val as u64, False); + llvm::LLVMSetInitializer(ll_g, llval); + + if tcx.sess.opts.debuginfo != DebugInfo::None { + let dbg_cx = debuginfo::CodegenUnitDebugContext::new(llmod); + debuginfo::metadata::build_compile_unit_di_node(tcx, module_name, &dbg_cx); + dbg_cx.finalize(tcx.sess); + } +} diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs new file mode 100644 index 00000000000..a53946995ee --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -0,0 +1,1037 @@ +use crate::attributes; +use crate::builder::Builder; +use crate::common::Funclet; +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm_util; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; + +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_codegen_ssa::mir::operand::OperandValue; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::fx::FxHashMap; +use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::{bug, span_bug, ty::Instance}; +use rustc_span::{Pos, Span}; +use rustc_target::abi::*; +use rustc_target::asm::*; + +use libc::{c_char, c_uint}; +use smallvec::SmallVec; +use tracing::debug; + +impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { + fn codegen_inline_asm( + &mut self, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperandRef<'tcx, Self>], + options: InlineAsmOptions, + line_spans: &[Span], + instance: Instance<'_>, + dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + ) { + let asm_arch = self.tcx.sess.asm_arch.unwrap(); + + // Collect the types of output operands + let mut constraints = vec![]; + let mut clobbers = vec![]; + let mut output_types = vec![]; + let mut op_idx = FxHashMap::default(); + let mut clobbered_x87 = false; + for (idx, op) in operands.iter().enumerate() { + match *op { + InlineAsmOperandRef::Out { reg, late, place } => { + let is_target_supported = |reg_class: InlineAsmRegClass| { + for &(_, feature) in reg_class.supported_types(asm_arch) { + if let Some(feature) = feature { + let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id()); + if self.tcx.sess.target_features.contains(&feature) + || codegen_fn_attrs.target_features.contains(&feature) + { + return true; + } + } else { + // Register class is unconditionally supported + return true; + } + } + false + }; + + let mut layout = None; + let ty = if let Some(ref place) = place { + layout = Some(&place.layout); + llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout) + } else if matches!( + reg.reg_class(), + InlineAsmRegClass::X86( + X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg + ) + ) { + // Special handling for x87/mmx registers: we always + // clobber the whole set if one register is marked as + // clobbered. This is due to the way LLVM handles the + // FP stack in inline assembly. + if !clobbered_x87 { + clobbered_x87 = true; + clobbers.push("~{st}".to_string()); + for i in 1..=7 { + clobbers.push(format!("~{{st({})}}", i)); + } + } + continue; + } else if !is_target_supported(reg.reg_class()) + || reg.reg_class().is_clobber_only(asm_arch) + { + // We turn discarded outputs into clobber constraints + // if the target feature needed by the register class is + // disabled. This is necessary otherwise LLVM will try + // to actually allocate a register for the dummy output. + assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_))); + clobbers.push(format!("~{}", reg_to_llvm(reg, None))); + continue; + } else { + // If the output is discarded, we don't really care what + // type is used. We're just using this to tell LLVM to + // reserve the register. + dummy_output_type(self.cx, reg.reg_class()) + }; + output_types.push(ty); + op_idx.insert(idx, constraints.len()); + let prefix = if late { "=" } else { "=&" }; + constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout))); + } + InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { + let layout = if let Some(ref out_place) = out_place { + &out_place.layout + } else { + // LLVM required tied operands to have the same type, + // so we just use the type of the input. + &in_value.layout + }; + let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout); + output_types.push(ty); + op_idx.insert(idx, constraints.len()); + let prefix = if late { "=" } else { "=&" }; + constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout)))); + } + _ => {} + } + } + + // Collect input operands + let mut inputs = vec![]; + for (idx, op) in operands.iter().enumerate() { + match *op { + InlineAsmOperandRef::In { reg, value } => { + let llval = + llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout); + inputs.push(llval); + op_idx.insert(idx, constraints.len()); + constraints.push(reg_to_llvm(reg, Some(&value.layout))); + } + InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => { + let value = llvm_fixup_input( + self, + in_value.immediate(), + reg.reg_class(), + &in_value.layout, + ); + inputs.push(value); + constraints.push(format!("{}", op_idx[&idx])); + } + InlineAsmOperandRef::SymFn { instance } => { + inputs.push(self.cx.get_fn(instance)); + op_idx.insert(idx, constraints.len()); + constraints.push("s".to_string()); + } + InlineAsmOperandRef::SymStatic { def_id } => { + inputs.push(self.cx.get_static(def_id)); + op_idx.insert(idx, constraints.len()); + constraints.push("s".to_string()); + } + _ => {} + } + } + + // Build the template string + let mut template_str = String::new(); + for piece in template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => { + if s.contains('$') { + for c in s.chars() { + if c == '$' { + template_str.push_str("$$"); + } else { + template_str.push(c); + } + } + } else { + template_str.push_str(s) + } + } + InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { + match operands[operand_idx] { + InlineAsmOperandRef::In { reg, .. } + | InlineAsmOperandRef::Out { reg, .. } + | InlineAsmOperandRef::InOut { reg, .. } => { + let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier); + if let Some(modifier) = modifier { + template_str.push_str(&format!( + "${{{}:{}}}", + op_idx[&operand_idx], modifier + )); + } else { + template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx])); + } + } + InlineAsmOperandRef::Const { ref string } => { + // Const operands get injected directly into the template + template_str.push_str(string); + } + InlineAsmOperandRef::SymFn { .. } + | InlineAsmOperandRef::SymStatic { .. } => { + // Only emit the raw symbol name + template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); + } + } + } + } + } + + constraints.append(&mut clobbers); + if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { + match asm_arch { + InlineAsmArch::AArch64 | InlineAsmArch::Arm => { + constraints.push("~{cc}".to_string()); + } + InlineAsmArch::X86 | InlineAsmArch::X86_64 => { + constraints.extend_from_slice(&[ + "~{dirflag}".to_string(), + "~{fpsr}".to_string(), + "~{flags}".to_string(), + ]); + } + InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { + constraints.extend_from_slice(&[ + "~{vtype}".to_string(), + "~{vl}".to_string(), + "~{vxsat}".to_string(), + "~{vxrm}".to_string(), + ]); + } + InlineAsmArch::Avr => { + constraints.push("~{sreg}".to_string()); + } + InlineAsmArch::Nvptx64 => {} + InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} + InlineAsmArch::Hexagon => {} + InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} + InlineAsmArch::S390x => {} + InlineAsmArch::SpirV => {} + InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} + InlineAsmArch::Bpf => {} + InlineAsmArch::Msp430 => { + constraints.push("~{sr}".to_string()); + } + } + } + if !options.contains(InlineAsmOptions::NOMEM) { + // This is actually ignored by LLVM, but it's probably best to keep + // it just in case. LLVM instead uses the ReadOnly/ReadNone + // attributes on the call instruction to optimize. + constraints.push("~{memory}".to_string()); + } + let volatile = !options.contains(InlineAsmOptions::PURE); + let alignstack = !options.contains(InlineAsmOptions::NOSTACK); + let output_type = match &output_types[..] { + [] => self.type_void(), + [ty] => ty, + tys => self.type_struct(tys, false), + }; + let dialect = match asm_arch { + InlineAsmArch::X86 | InlineAsmArch::X86_64 + if !options.contains(InlineAsmOptions::ATT_SYNTAX) => + { + llvm::AsmDialect::Intel + } + _ => llvm::AsmDialect::Att, + }; + let result = inline_asm_call( + self, + &template_str, + &constraints.join(","), + &inputs, + output_type, + volatile, + alignstack, + dialect, + line_spans, + options.contains(InlineAsmOptions::MAY_UNWIND), + dest_catch_funclet, + ) + .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); + + let mut attrs = SmallVec::<[_; 2]>::new(); + if options.contains(InlineAsmOptions::PURE) { + if options.contains(InlineAsmOptions::NOMEM) { + attrs.push(llvm::AttributeKind::ReadNone.create_attr(self.cx.llcx)); + } else if options.contains(InlineAsmOptions::READONLY) { + attrs.push(llvm::AttributeKind::ReadOnly.create_attr(self.cx.llcx)); + } + attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); + } else if options.contains(InlineAsmOptions::NOMEM) { + attrs.push(llvm::AttributeKind::InaccessibleMemOnly.create_attr(self.cx.llcx)); + } else { + // LLVM doesn't have an attribute to represent ReadOnly + SideEffect + } + attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); + + // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` + if let Some((dest, _, _)) = dest_catch_funclet { + self.switch_to_block(dest); + } + + // Write results to outputs + for (idx, op) in operands.iter().enumerate() { + if let InlineAsmOperandRef::Out { reg, place: Some(place), .. } + | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op + { + let value = if output_types.len() == 1 { + result + } else { + self.extract_value(result, op_idx[&idx] as u64) + }; + let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout); + OperandValue::Immediate(value).store(self, place); + } + } + } +} + +impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { + fn codegen_global_asm( + &self, + template: &[InlineAsmTemplatePiece], + operands: &[GlobalAsmOperandRef<'tcx>], + options: InlineAsmOptions, + _line_spans: &[Span], + ) { + let asm_arch = self.tcx.sess.asm_arch.unwrap(); + + // Default to Intel syntax on x86 + let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) + && !options.contains(InlineAsmOptions::ATT_SYNTAX); + + // Build the template string + let mut template_str = String::new(); + if intel_syntax { + template_str.push_str(".intel_syntax\n"); + } + for piece in template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s), + InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => { + match operands[operand_idx] { + GlobalAsmOperandRef::Const { ref string } => { + // Const operands get injected directly into the + // template. Note that we don't need to escape $ + // here unlike normal inline assembly. + template_str.push_str(string); + } + GlobalAsmOperandRef::SymFn { instance } => { + let llval = self.get_fn(instance); + self.add_compiler_used_global(llval); + let symbol = llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8"); + template_str.push_str(&symbol); + } + GlobalAsmOperandRef::SymStatic { def_id } => { + let llval = self + .renamed_statics + .borrow() + .get(&def_id) + .copied() + .unwrap_or_else(|| self.get_static(def_id)); + self.add_compiler_used_global(llval); + let symbol = llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8"); + template_str.push_str(&symbol); + } + } + } + } + } + if intel_syntax { + template_str.push_str("\n.att_syntax\n"); + } + + unsafe { + llvm::LLVMRustAppendModuleInlineAsm( + self.llmod, + template_str.as_ptr().cast(), + template_str.len(), + ); + } + } +} + +pub(crate) fn inline_asm_call<'ll>( + bx: &mut Builder<'_, 'll, '_>, + asm: &str, + cons: &str, + inputs: &[&'ll Value], + output: &'ll llvm::Type, + volatile: bool, + alignstack: bool, + dia: llvm::AsmDialect, + line_spans: &[Span], + unwind: bool, + dest_catch_funclet: Option<( + &'ll llvm::BasicBlock, + &'ll llvm::BasicBlock, + Option<&Funclet<'ll>>, + )>, +) -> Option<&'ll Value> { + let volatile = if volatile { llvm::True } else { llvm::False }; + let alignstack = if alignstack { llvm::True } else { llvm::False }; + let can_throw = if unwind { llvm::True } else { llvm::False }; + + let argtys = inputs + .iter() + .map(|v| { + debug!("Asm Input Type: {:?}", *v); + bx.cx.val_ty(*v) + }) + .collect::>(); + + debug!("Asm Output Type: {:?}", output); + let fty = bx.cx.type_func(&argtys, output); + unsafe { + // Ask LLVM to verify that the constraints are well-formed. + let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); + debug!("constraint verification result: {:?}", constraints_ok); + if constraints_ok { + if unwind && llvm_util::get_version() < (13, 0, 0) { + bx.cx.sess().span_fatal( + line_spans[0], + "unwinding from inline assembly is only supported on llvm >= 13.", + ); + } + + let v = llvm::LLVMRustInlineAsm( + fty, + asm.as_ptr().cast(), + asm.len(), + cons.as_ptr().cast(), + cons.len(), + volatile, + alignstack, + dia, + can_throw, + ); + + let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { + bx.invoke(fty, v, inputs, dest, catch, funclet) + } else { + bx.call(fty, v, inputs, None) + }; + + // Store mark in a metadata node so we can map LLVM errors + // back to source locations. See #17552. + let key = "srcloc"; + let kind = llvm::LLVMGetMDKindIDInContext( + bx.llcx, + key.as_ptr() as *const c_char, + key.len() as c_uint, + ); + + // srcloc contains one integer for each line of assembly code. + // Unfortunately this isn't enough to encode a full span so instead + // we just encode the start position of each line. + // FIXME: Figure out a way to pass the entire line spans. + let mut srcloc = vec![]; + if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 { + // LLVM inserts an extra line to add the ".intel_syntax", so add + // a dummy srcloc entry for it. + // + // Don't do this if we only have 1 line span since that may be + // due to the asm template string coming from a macro. LLVM will + // default to the first srcloc for lines that don't have an + // associated srcloc. + srcloc.push(bx.const_i32(0)); + } + srcloc.extend(line_spans.iter().map(|span| bx.const_i32(span.lo().to_u32() as i32))); + let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32); + llvm::LLVMSetMetadata(call, kind, md); + + Some(call) + } else { + // LLVM has detected an issue with our constraints, bail out + None + } + } +} + +/// If the register is an xmm/ymm/zmm register then return its index. +fn xmm_reg_index(reg: InlineAsmReg) -> Option { + match reg { + InlineAsmReg::X86(reg) + if reg as u32 >= X86InlineAsmReg::xmm0 as u32 + && reg as u32 <= X86InlineAsmReg::xmm15 as u32 => + { + Some(reg as u32 - X86InlineAsmReg::xmm0 as u32) + } + InlineAsmReg::X86(reg) + if reg as u32 >= X86InlineAsmReg::ymm0 as u32 + && reg as u32 <= X86InlineAsmReg::ymm15 as u32 => + { + Some(reg as u32 - X86InlineAsmReg::ymm0 as u32) + } + InlineAsmReg::X86(reg) + if reg as u32 >= X86InlineAsmReg::zmm0 as u32 + && reg as u32 <= X86InlineAsmReg::zmm31 as u32 => + { + Some(reg as u32 - X86InlineAsmReg::zmm0 as u32) + } + _ => None, + } +} + +/// If the register is an AArch64 vector register then return its index. +fn a64_vreg_index(reg: InlineAsmReg) -> Option { + match reg { + InlineAsmReg::AArch64(reg) + if reg as u32 >= AArch64InlineAsmReg::v0 as u32 + && reg as u32 <= AArch64InlineAsmReg::v31 as u32 => + { + Some(reg as u32 - AArch64InlineAsmReg::v0 as u32) + } + _ => None, + } +} + +/// Converts a register class to an LLVM constraint code. +fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String { + match reg { + // For vector registers LLVM wants the register name to match the type size. + InlineAsmRegOrRegClass::Reg(reg) => { + if let Some(idx) = xmm_reg_index(reg) { + let class = if let Some(layout) = layout { + match layout.size.bytes() { + 64 => 'z', + 32 => 'y', + _ => 'x', + } + } else { + // We use f32 as the type for discarded outputs + 'x' + }; + format!("{{{}mm{}}}", class, idx) + } else if let Some(idx) = a64_vreg_index(reg) { + let class = if let Some(layout) = layout { + match layout.size.bytes() { + 16 => 'q', + 8 => 'd', + 4 => 's', + 2 => 'h', + 1 => 'd', // We fixup i8 to i8x8 + _ => unreachable!(), + } + } else { + // We use i64x2 as the type for discarded outputs + 'q' + }; + format!("{{{}{}}}", class, idx) + } else if reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { + // LLVM doesn't recognize x30 + "{lr}".to_string() + } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) { + // LLVM doesn't recognize r14 + "{lr}".to_string() + } else { + format!("{{{}}}", reg.name()) + } + } + InlineAsmRegOrRegClass::RegClass(reg) => match reg { + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t", + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x", + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", + InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r", + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) + | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f", + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q", + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q", + InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) + | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x", + InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", + InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", + InlineAsmRegClass::X86( + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, + ) => unreachable!("clobber-only"), + InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w", + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d", + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r", + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w", + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e", + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", + InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { + bug!("LLVM backend does not support SPIR-V") + } + InlineAsmRegClass::Err => unreachable!(), + } + .to_string(), + } +} + +/// Converts a modifier into LLVM's equivalent modifier. +fn modifier_to_llvm( + arch: InlineAsmArch, + reg: InlineAsmRegClass, + modifier: Option, +) -> Option { + match reg { + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) + | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { + if modifier == Some('v') { None } else { modifier } + } + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None, + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None, + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'), + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { + if modifier.is_none() { + Some('q') + } else { + modifier + } + } + InlineAsmRegClass::Hexagon(_) => None, + InlineAsmRegClass::Mips(_) => None, + InlineAsmRegClass::Nvptx(_) => None, + InlineAsmRegClass::PowerPC(_) => None, + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) + | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None, + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) + | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier { + None if arch == InlineAsmArch::X86_64 => Some('q'), + None => Some('k'), + Some('l') => Some('b'), + Some('h') => Some('h'), + Some('x') => Some('w'), + Some('e') => Some('k'), + Some('r') => Some('q'), + _ => unreachable!(), + }, + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None, + InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg) + | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg) + | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) { + (X86InlineAsmRegClass::xmm_reg, None) => Some('x'), + (X86InlineAsmRegClass::ymm_reg, None) => Some('t'), + (X86InlineAsmRegClass::zmm_reg, None) => Some('g'), + (_, Some('x')) => Some('x'), + (_, Some('y')) => Some('t'), + (_, Some('z')) => Some('g'), + _ => unreachable!(), + }, + InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, + InlineAsmRegClass::X86( + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, + ) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None, + InlineAsmRegClass::Bpf(_) => None, + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) + | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) + | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier { + Some('h') => Some('B'), + Some('l') => Some('A'), + _ => None, + }, + InlineAsmRegClass::Avr(_) => None, + InlineAsmRegClass::S390x(_) => None, + InlineAsmRegClass::Msp430(_) => None, + InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { + bug!("LLVM backend does not support SPIR-V") + } + InlineAsmRegClass::Err => unreachable!(), + } +} + +/// Type to use for outputs that are discarded. It doesn't really matter what +/// the type is, as long as it is valid for the constraint code. +fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type { + match reg { + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) + | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { + cx.type_vector(cx.type_i64(), 2) + } + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(), + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { + cx.type_vector(cx.type_i64(), 2) + } + InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) + | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) + | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(), + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(), + InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) + | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) + | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), + InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), + InlineAsmRegClass::X86( + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, + ) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(), + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(), + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(), + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(), + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(), + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(), + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), + InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), + InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { + bug!("LLVM backend does not support SPIR-V") + } + InlineAsmRegClass::Err => unreachable!(), + } +} + +/// Helper function to get the LLVM type for a Scalar. Pointers are returned as +/// the equivalent integer type. +fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type { + match scalar.primitive() { + Primitive::Int(Integer::I8, _) => cx.type_i8(), + Primitive::Int(Integer::I16, _) => cx.type_i16(), + Primitive::Int(Integer::I32, _) => cx.type_i32(), + Primitive::Int(Integer::I64, _) => cx.type_i64(), + Primitive::F32 => cx.type_f32(), + Primitive::F64 => cx.type_f64(), + Primitive::Pointer => cx.type_isize(), + _ => unreachable!(), + } +} + +/// Fix up an input value to work around LLVM bugs. +fn llvm_fixup_input<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + mut value: &'ll Value, + reg: InlineAsmRegClass, + layout: &TyAndLayout<'tcx>, +) -> &'ll Value { + match (reg, layout.abi) { + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + if let Primitive::Int(Integer::I8, _) = s.primitive() { + let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); + bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) + } else { + value + } + } + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { + let elem_ty = llvm_asm_scalar_type(bx.cx, s); + let count = 16 / layout.size.bytes(); + let vec_ty = bx.cx.type_vector(elem_ty, count); + if let Primitive::Pointer = s.primitive() { + value = bx.ptrtoint(value, bx.cx.type_isize()); + } + bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) + } + ( + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { + let elem_ty = llvm_asm_scalar_type(bx.cx, element); + let vec_ty = bx.cx.type_vector(elem_ty, count); + let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); + bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) + } + (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + if s.primitive() == Primitive::F64 => + { + bx.bitcast(value, bx.cx.type_i64()) + } + ( + InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), + Abi::Vector { .. }, + ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)), + ( + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I32, _) = s.primitive() { + bx.bitcast(value, bx.cx.type_f32()) + } else { + value + } + } + ( + InlineAsmRegClass::Arm( + ArmInlineAsmRegClass::dreg + | ArmInlineAsmRegClass::dreg_low8 + | ArmInlineAsmRegClass::dreg_low16, + ), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I64, _) = s.primitive() { + bx.bitcast(value, bx.cx.type_f64()) + } else { + value + } + } + (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + match s.primitive() { + // MIPS only supports register-length arithmetics. + Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), + Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()), + Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()), + _ => value, + } + } + _ => value, + } +} + +/// Fix up an output value to work around LLVM bugs. +fn llvm_fixup_output<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + mut value: &'ll Value, + reg: InlineAsmRegClass, + layout: &TyAndLayout<'tcx>, +) -> &'ll Value { + match (reg, layout.abi) { + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + if let Primitive::Int(Integer::I8, _) = s.primitive() { + bx.extract_element(value, bx.const_i32(0)) + } else { + value + } + } + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { + value = bx.extract_element(value, bx.const_i32(0)); + if let Primitive::Pointer = s.primitive() { + value = bx.inttoptr(value, layout.llvm_type(bx.cx)); + } + value + } + ( + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { + let elem_ty = llvm_asm_scalar_type(bx.cx, element); + let vec_ty = bx.cx.type_vector(elem_ty, count * 2); + let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); + bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) + } + (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + if s.primitive() == Primitive::F64 => + { + bx.bitcast(value, bx.cx.type_f64()) + } + ( + InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), + Abi::Vector { .. }, + ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)), + ( + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I32, _) = s.primitive() { + bx.bitcast(value, bx.cx.type_i32()) + } else { + value + } + } + ( + InlineAsmRegClass::Arm( + ArmInlineAsmRegClass::dreg + | ArmInlineAsmRegClass::dreg_low8 + | ArmInlineAsmRegClass::dreg_low16, + ), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I64, _) = s.primitive() { + bx.bitcast(value, bx.cx.type_i64()) + } else { + value + } + } + (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + match s.primitive() { + // MIPS only supports register-length arithmetics. + Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), + Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()), + Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()), + Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()), + _ => value, + } + } + _ => value, + } +} + +/// Output type to use for llvm_fixup_output. +fn llvm_fixup_output_type<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + reg: InlineAsmRegClass, + layout: &TyAndLayout<'tcx>, +) -> &'ll Type { + match (reg, layout.abi) { + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + if let Primitive::Int(Integer::I8, _) = s.primitive() { + cx.type_vector(cx.type_i8(), 8) + } else { + layout.llvm_type(cx) + } + } + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { + let elem_ty = llvm_asm_scalar_type(cx, s); + let count = 16 / layout.size.bytes(); + cx.type_vector(elem_ty, count) + } + ( + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { + let elem_ty = llvm_asm_scalar_type(cx, element); + cx.type_vector(elem_ty, count * 2) + } + (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + if s.primitive() == Primitive::F64 => + { + cx.type_i64() + } + ( + InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), + Abi::Vector { .. }, + ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8), + ( + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I32, _) = s.primitive() { + cx.type_f32() + } else { + layout.llvm_type(cx) + } + } + ( + InlineAsmRegClass::Arm( + ArmInlineAsmRegClass::dreg + | ArmInlineAsmRegClass::dreg_low8 + | ArmInlineAsmRegClass::dreg_low16, + ), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I64, _) = s.primitive() { + cx.type_f64() + } else { + layout.llvm_type(cx) + } + } + (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + match s.primitive() { + // MIPS only supports register-length arithmetics. + Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), + Primitive::F32 => cx.type_i32(), + Primitive::F64 => cx.type_i64(), + _ => layout.llvm_type(cx), + } + } + _ => layout.llvm_type(cx), + } +} diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs new file mode 100644 index 00000000000..aabbe8ac276 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -0,0 +1,449 @@ +//! Set and unset common attributes on LLVM values. + +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::small_str::SmallStr; +use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_session::config::OptLevel; +use rustc_span::symbol::sym; +use rustc_target::spec::abi::Abi; +use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; +use smallvec::SmallVec; + +use crate::attributes; +use crate::llvm::AttributePlace::Function; +use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace}; +use crate::llvm_util; +pub use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr}; + +use crate::context::CodegenCx; +use crate::value::Value; + +pub fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { + if !attrs.is_empty() { + llvm::AddFunctionAttributes(llfn, idx, attrs); + } +} + +pub fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attribute]) { + if !attrs.is_empty() { + llvm::AddCallSiteAttributes(callsite, idx, attrs); + } +} + +/// Get LLVM attribute for the provided inline heuristic. +#[inline] +fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { + match inline { + InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), + InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), + InlineAttr::Never => { + if cx.sess().target.arch != "amdgpu" { + Some(AttributeKind::NoInline.create_attr(cx.llcx)) + } else { + None + } + } + InlineAttr::None => None, + } +} + +/// Get LLVM sanitize attributes. +#[inline] +pub fn sanitize_attrs<'ll>( + cx: &CodegenCx<'ll, '_>, + no_sanitize: SanitizerSet, +) -> SmallVec<[&'ll Attribute; 4]> { + let mut attrs = SmallVec::new(); + let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; + if enabled.contains(SanitizerSet::ADDRESS) { + attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); + } + if enabled.contains(SanitizerSet::MEMORY) { + attrs.push(llvm::AttributeKind::SanitizeMemory.create_attr(cx.llcx)); + } + if enabled.contains(SanitizerSet::THREAD) { + attrs.push(llvm::AttributeKind::SanitizeThread.create_attr(cx.llcx)); + } + if enabled.contains(SanitizerSet::HWADDRESS) { + attrs.push(llvm::AttributeKind::SanitizeHWAddress.create_attr(cx.llcx)); + } + if enabled.contains(SanitizerSet::SHADOWCALLSTACK) { + attrs.push(llvm::AttributeKind::ShadowCallStack.create_attr(cx.llcx)); + } + if enabled.contains(SanitizerSet::MEMTAG) { + // Check to make sure the mte target feature is actually enabled. + let features = cx.tcx.global_backend_features(()); + let mte_feature = + features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); + if let None | Some("-mte") = mte_feature { + cx.tcx.sess.err("`-Zsanitizer=memtag` requires `-Ctarget-feature=+mte`"); + } + + attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); + } + attrs +} + +/// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. +#[inline] +pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { + // NOTE: We should determine if we even need async unwind tables, as they + // take have more overhead and if we can use sync unwind tables we + // probably should. + llvm::CreateUWTableAttr(llcx, true) +} + +pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + let mut fp = cx.sess().target.frame_pointer; + // "mcount" function relies on stack pointer. + // See . + if cx.sess().instrument_mcount() || matches!(cx.sess().opts.cg.force_frame_pointers, Some(true)) + { + fp = FramePointer::Always; + } + let attr_value = match fp { + FramePointer::Always => "all", + FramePointer::NonLeaf => "non-leaf", + FramePointer::MayOmit => return None, + }; + Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value)) +} + +/// Tell LLVM what instrument function to insert. +#[inline] +fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + if cx.sess().instrument_mcount() { + // Similar to `clang -pg` behavior. Handled by the + // `post-inline-ee-instrument` LLVM pass. + + // The function name varies on platforms. + // See test/CodeGen/mcount.c in clang. + let mcount_name = cx.sess().target.mcount.as_ref(); + + Some(llvm::CreateAttrStringValue( + cx.llcx, + "instrument-function-entry-inlined", + &mcount_name, + )) + } else { + None + } +} + +fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + // Currently stack probes seem somewhat incompatible with the address + // sanitizer and thread sanitizer. With asan we're already protected from + // stack overflow anyway so we don't really need stack probes regardless. + if cx + .sess() + .opts + .unstable_opts + .sanitizer + .intersects(SanitizerSet::ADDRESS | SanitizerSet::THREAD) + { + return None; + } + + // probestack doesn't play nice either with `-C profile-generate`. + if cx.sess().opts.cg.profile_generate.enabled() { + return None; + } + + // probestack doesn't play nice either with gcov profiling. + if cx.sess().opts.unstable_opts.profile { + return None; + } + + let attr_value = match cx.sess().target.stack_probes { + StackProbeType::None => return None, + // Request LLVM to generate the probes inline. If the given LLVM version does not support + // this, no probe is generated at all (even if the attribute is specified). + StackProbeType::Inline => "inline-asm", + // Flag our internal `__rust_probestack` function as the stack probe symbol. + // This is defined in the `compiler-builtins` crate for each architecture. + StackProbeType::Call => "__rust_probestack", + // Pick from the two above based on the LLVM version. + StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { + if llvm_util::get_version() < min_llvm_version_for_inline { + "__rust_probestack" + } else { + "inline-asm" + } + } + }; + Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value)) +} + +fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + let sspattr = match cx.sess().stack_protector() { + StackProtector::None => return None, + StackProtector::All => AttributeKind::StackProtectReq, + StackProtector::Strong => AttributeKind::StackProtectStrong, + StackProtector::Basic => AttributeKind::StackProtect, + }; + + Some(sspattr.create_attr(cx.llcx)) +} + +pub fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { + let target_cpu = llvm_util::target_cpu(cx.tcx.sess); + llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) +} + +pub fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + llvm_util::tune_cpu(cx.tcx.sess) + .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) +} + +/// Get the `NonLazyBind` LLVM attribute, +/// if the codegen options allow skipping the PLT. +pub fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + // Don't generate calls through PLT if it's not necessary + if !cx.sess().needs_plt() { + Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) + } else { + None + } +} + +/// Get the default optimizations attrs for a function. +#[inline] +pub(crate) fn default_optimisation_attrs<'ll>( + cx: &CodegenCx<'ll, '_>, +) -> SmallVec<[&'ll Attribute; 2]> { + let mut attrs = SmallVec::new(); + match cx.sess().opts.optimize { + OptLevel::Size => { + attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); + } + OptLevel::SizeMin => { + attrs.push(llvm::AttributeKind::MinSize.create_attr(cx.llcx)); + attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); + } + _ => {} + } + attrs +} + +fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { + llvm::CreateAttrStringValue(llcx, "alloc-family", "__rust_alloc") +} + +/// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) +/// attributes. +pub fn from_fn_attrs<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + llfn: &'ll Value, + instance: ty::Instance<'tcx>, +) { + let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); + + let mut to_add = SmallVec::<[_; 16]>::new(); + + match codegen_fn_attrs.optimize { + OptimizeAttr::None => { + to_add.extend(default_optimisation_attrs(cx)); + } + OptimizeAttr::Size => { + to_add.push(llvm::AttributeKind::MinSize.create_attr(cx.llcx)); + to_add.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); + } + OptimizeAttr::Speed => {} + } + + let inline = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { + InlineAttr::Never + } else if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { + InlineAttr::Hint + } else { + codegen_fn_attrs.inline + }; + to_add.extend(inline_attr(cx, inline)); + + // The `uwtable` attribute according to LLVM is: + // + // This attribute indicates that the ABI being targeted requires that an + // unwind table entry be produced for this function even if we can show + // that no exceptions passes by it. This is normally the case for the + // ELF x86-64 abi, but it can be disabled for some compilation units. + // + // Typically when we're compiling with `-C panic=abort` (which implies this + // `no_landing_pads` check) we don't need `uwtable` because we can't + // generate any exceptions! On Windows, however, exceptions include other + // events such as illegal instructions, segfaults, etc. This means that on + // Windows we end up still needing the `uwtable` attribute even if the `-C + // panic=abort` flag is passed. + // + // You can also find more info on why Windows always requires uwtables here: + // https://bugzilla.mozilla.org/show_bug.cgi?id=1302078 + if cx.sess().must_emit_unwind_tables() { + to_add.push(uwtable_attr(cx.llcx)); + } + + if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { + to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); + } + + // FIXME: none of these three functions interact with source level attributes. + to_add.extend(frame_pointer_type_attr(cx)); + to_add.extend(instrument_function_attr(cx)); + to_add.extend(probestack_attr(cx)); + to_add.extend(stackprotector_attr(cx)); + + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { + to_add.push(AttributeKind::Cold.create_attr(cx.llcx)); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_RETURNS_TWICE) { + to_add.push(AttributeKind::ReturnsTwice.create_attr(cx.llcx)); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_PURE) { + to_add.push(AttributeKind::ReadOnly.create_attr(cx.llcx)); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_CONST) { + to_add.push(AttributeKind::ReadNone.create_attr(cx.llcx)); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { + to_add.push(AttributeKind::Naked.create_attr(cx.llcx)); + // HACK(jubilee): "indirect branch tracking" works by attaching prologues to functions. + // And it is a module-level attribute, so the alternative is pulling naked functions into new LLVM modules. + // Otherwise LLVM's "naked" functions come with endbr prefixes per https://github.com/rust-lang/rust/issues/98768 + to_add.push(AttributeKind::NoCfCheck.create_attr(cx.llcx)); + // Need this for AArch64. + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false")); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) + || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) + { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); + let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { + flags |= AllocKindFlags::Uninitialized; + } else { + flags |= AllocKindFlags::Zeroed; + } + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); + } + // apply to return place instead of function (unlike all other attributes applied in this function) + let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::REALLOCATOR) { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr( + cx.llcx, + AllocKindFlags::Realloc | AllocKindFlags::Aligned, + )); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); + } + let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::DEALLOCATOR) { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + } + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { + to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry")); + } + if let Some(align) = codegen_fn_attrs.alignment { + llvm::set_alignment(llfn, align as usize); + } + to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); + + // Always annotate functions with the target-cpu they are compiled for. + // Without this, ThinLTO won't inline Rust functions into Clang generated + // functions (because Clang annotates functions this way too). + to_add.push(target_cpu_attr(cx)); + // tune-cpu is only conveyed through the attribute for our purpose. + // The target doesn't care; the subtarget reads our attribute. + to_add.extend(tune_cpu_attr(cx)); + + let function_features = + codegen_fn_attrs.target_features.iter().map(|f| f.as_str()).collect::>(); + + if let Some(f) = llvm_util::check_tied_features( + cx.tcx.sess, + &function_features.iter().map(|f| (*f, true)).collect(), + ) { + let span = cx + .tcx + .get_attr(instance.def_id(), sym::target_feature) + .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); + let msg = format!( + "the target features {} must all be either enabled or disabled together", + f.join(", ") + ); + let mut err = cx.tcx.sess.struct_span_err(span, &msg); + err.help("add the missing features in a `target_feature` attribute"); + err.emit(); + return; + } + + let mut function_features = function_features + .iter() + .flat_map(|feat| { + llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{}", f)) + }) + .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { + InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(), + InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(), + })) + .collect::>(); + + if cx.tcx.sess.target.is_like_wasm { + // If this function is an import from the environment but the wasm + // import has a specific module/name, apply them here. + if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", &module)); + + let name = + codegen_fn_attrs.link_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); + let name = name.as_str(); + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); + } + + // The `"wasm"` abi on wasm targets automatically enables the + // `+multivalue` feature because the purpose of the wasm abi is to match + // the WebAssembly specification, which has this feature. This won't be + // needed when LLVM enables this `multivalue` feature by default. + if !cx.tcx.is_closure(instance.def_id()) { + let abi = cx.tcx.fn_sig(instance.def_id()).abi(); + if abi == Abi::Wasm { + function_features.push("+multivalue".to_string()); + } + } + } + + let global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str()); + let function_features = function_features.iter().map(|s| s.as_str()); + let target_features = + global_features.chain(function_features).intersperse(",").collect::>(); + if !target_features.is_empty() { + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)); + } + + attributes::apply_to_llfn(llfn, Function, &to_add); +} + +fn wasm_import_module(tcx: TyCtxt<'_>, id: DefId) -> Option<&String> { + tcx.wasm_import_module_map(id.krate).get(&id) +} diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs new file mode 100644 index 00000000000..27039cda253 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -0,0 +1,383 @@ +//! A helper class for dealing with static archives + +use std::env; +use std::ffi::{CStr, CString, OsString}; +use std::io; +use std::mem; +use std::path::{Path, PathBuf}; +use std::ptr; +use std::str; + +use crate::llvm::archive_ro::{ArchiveRO, Child}; +use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; +use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder}; +use rustc_session::cstore::{DllCallingConvention, DllImport}; +use rustc_session::Session; + +/// Helper for adding many files to an archive. +#[must_use = "must call build() to finish building the archive"] +pub struct LlvmArchiveBuilder<'a> { + sess: &'a Session, + additions: Vec, +} + +enum Addition { + File { path: PathBuf, name_in_archive: String }, + Archive { path: PathBuf, archive: ArchiveRO, skip: Box bool> }, +} + +impl Addition { + fn path(&self) -> &Path { + match self { + Addition::File { path, .. } | Addition::Archive { path, .. } => path, + } + } +} + +fn is_relevant_child(c: &Child<'_>) -> bool { + match c.name() { + Some(name) => !name.contains("SYMDEF"), + None => false, + } +} + +/// Map machine type strings to values of LLVM's MachineTypes enum. +fn llvm_machine_type(cpu: &str) -> LLVMMachineType { + match cpu { + "x86_64" => LLVMMachineType::AMD64, + "x86" => LLVMMachineType::I386, + "aarch64" => LLVMMachineType::ARM64, + "arm" => LLVMMachineType::ARM, + _ => panic!("unsupported cpu type {}", cpu), + } +} + +impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { + fn add_archive( + &mut self, + archive: &Path, + skip: Box bool + 'static>, + ) -> io::Result<()> { + let archive_ro = match ArchiveRO::open(archive) { + Ok(ar) => ar, + Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), + }; + if self.additions.iter().any(|ar| ar.path() == archive) { + return Ok(()); + } + self.additions.push(Addition::Archive { + path: archive.to_path_buf(), + archive: archive_ro, + skip: Box::new(skip), + }); + Ok(()) + } + + /// Adds an arbitrary file to this archive + fn add_file(&mut self, file: &Path) { + let name = file.file_name().unwrap().to_str().unwrap(); + self.additions + .push(Addition::File { path: file.to_path_buf(), name_in_archive: name.to_owned() }); + } + + /// Combine the provided files, rlibs, and native libraries into a single + /// `Archive`. + fn build(mut self: Box, output: &Path) -> bool { + match self.build_with_llvm(output) { + Ok(any_members) => any_members, + Err(e) => self.sess.fatal(&format!("failed to build archive: {}", e)), + } + } +} + +pub struct LlvmArchiveBuilderBuilder; + +impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { + fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box + 'a> { + Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) + } + + fn create_dll_import_lib( + &self, + sess: &Session, + lib_name: &str, + dll_imports: &[DllImport], + tmpdir: &Path, + ) -> PathBuf { + let output_path = { + let mut output_path: PathBuf = tmpdir.to_path_buf(); + output_path.push(format!("{}_imports", lib_name)); + output_path.with_extension("lib") + }; + + let target = &sess.target; + let mingw_gnu_toolchain = target.vendor == "pc" + && target.os == "windows" + && target.env == "gnu" + && target.abi.is_empty(); + + let import_name_and_ordinal_vector: Vec<(String, Option)> = dll_imports + .iter() + .map(|import: &DllImport| { + if sess.target.arch == "x86" { + ( + LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), + import.ordinal, + ) + } else { + (import.name.to_string(), import.ordinal) + } + }) + .collect(); + + if mingw_gnu_toolchain { + // The binutils linker used on -windows-gnu targets cannot read the import + // libraries generated by LLVM: in our attempts, the linker produced an .EXE + // that loaded but crashed with an AV upon calling one of the imported + // functions. Therefore, use binutils to create the import library instead, + // by writing a .DEF file to the temp dir and calling binutils's dlltool. + let def_file_path = tmpdir.join(format!("{}_imports", lib_name)).with_extension("def"); + + let def_file_content = format!( + "EXPORTS\n{}", + import_name_and_ordinal_vector + .into_iter() + .map(|(name, ordinal)| { + match ordinal { + Some(n) => format!("{} @{} NONAME", name, n), + None => name, + } + }) + .collect::>() + .join("\n") + ); + + match std::fs::write(&def_file_path, def_file_content) { + Ok(_) => {} + Err(e) => { + sess.fatal(&format!("Error writing .DEF file: {}", e)); + } + }; + + let dlltool = find_binutils_dlltool(sess); + let result = std::process::Command::new(dlltool) + .args([ + "-d", + def_file_path.to_str().unwrap(), + "-D", + lib_name, + "-l", + output_path.to_str().unwrap(), + ]) + .output(); + + match result { + Err(e) => { + sess.fatal(&format!("Error calling dlltool: {}", e)); + } + Ok(output) if !output.status.success() => sess.fatal(&format!( + "Dlltool could not create import library: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )), + _ => {} + } + } else { + // we've checked for \0 characters in the library name already + let dll_name_z = CString::new(lib_name).unwrap(); + + let output_path_z = rustc_fs_util::path_to_c_string(&output_path); + + tracing::trace!("invoking LLVMRustWriteImportLibrary"); + tracing::trace!(" dll_name {:#?}", dll_name_z); + tracing::trace!(" output_path {}", output_path.display()); + tracing::trace!( + " import names: {}", + dll_imports + .iter() + .map(|import| import.name.to_string()) + .collect::>() + .join(", "), + ); + + // All import names are Rust identifiers and therefore cannot contain \0 characters. + // FIXME: when support for #[link_name] is implemented, ensure that the import names + // still don't contain any \0 characters. Also need to check that the names don't + // contain substrings like " @" or "NONAME" that are keywords or otherwise reserved + // in definition files. + let cstring_import_name_and_ordinal_vector: Vec<(CString, Option)> = + import_name_and_ordinal_vector + .into_iter() + .map(|(name, ordinal)| (CString::new(name).unwrap(), ordinal)) + .collect(); + + let ffi_exports: Vec = cstring_import_name_and_ordinal_vector + .iter() + .map(|(name_z, ordinal)| LLVMRustCOFFShortExport::new(name_z.as_ptr(), *ordinal)) + .collect(); + let result = unsafe { + crate::llvm::LLVMRustWriteImportLibrary( + dll_name_z.as_ptr(), + output_path_z.as_ptr(), + ffi_exports.as_ptr(), + ffi_exports.len(), + llvm_machine_type(&sess.target.arch) as u16, + !sess.target.is_like_msvc, + ) + }; + + if result == crate::llvm::LLVMRustResult::Failure { + sess.fatal(&format!( + "Error creating import library for {}: {}", + lib_name, + llvm::last_error().unwrap_or("unknown LLVM error".to_string()) + )); + } + }; + + output_path + } +} + +impl<'a> LlvmArchiveBuilder<'a> { + fn build_with_llvm(&mut self, output: &Path) -> io::Result { + let kind = &*self.sess.target.archive_format; + let kind = kind.parse::().map_err(|_| kind).unwrap_or_else(|kind| { + self.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) + }); + + let mut additions = mem::take(&mut self.additions); + let mut strings = Vec::new(); + let mut members = Vec::new(); + + let dst = CString::new(output.to_str().unwrap())?; + + unsafe { + for addition in &mut additions { + match addition { + Addition::File { path, name_in_archive } => { + let path = CString::new(path.to_str().unwrap())?; + let name = CString::new(name_in_archive.clone())?; + members.push(llvm::LLVMRustArchiveMemberNew( + path.as_ptr(), + name.as_ptr(), + None, + )); + strings.push(path); + strings.push(name); + } + Addition::Archive { archive, skip, .. } => { + for child in archive.iter() { + let child = child.map_err(string_to_io_error)?; + if !is_relevant_child(&child) { + continue; + } + let child_name = child.name().unwrap(); + if skip(child_name) { + continue; + } + + // It appears that LLVM's archive writer is a little + // buggy if the name we pass down isn't just the + // filename component, so chop that off here and + // pass it in. + // + // See LLVM bug 25877 for more info. + let child_name = + Path::new(child_name).file_name().unwrap().to_str().unwrap(); + let name = CString::new(child_name)?; + let m = llvm::LLVMRustArchiveMemberNew( + ptr::null(), + name.as_ptr(), + Some(child.raw), + ); + members.push(m); + strings.push(name); + } + } + } + } + + let r = llvm::LLVMRustWriteArchive( + dst.as_ptr(), + members.len() as libc::size_t, + members.as_ptr() as *const &_, + true, + kind, + ); + let ret = if r.into_result().is_err() { + let err = llvm::LLVMRustGetLastError(); + let msg = if err.is_null() { + "failed to write archive".into() + } else { + String::from_utf8_lossy(CStr::from_ptr(err).to_bytes()) + }; + Err(io::Error::new(io::ErrorKind::Other, msg)) + } else { + Ok(!members.is_empty()) + }; + for member in members { + llvm::LLVMRustArchiveMemberFree(member); + } + ret + } + } + + fn i686_decorated_name(import: &DllImport, mingw: bool) -> String { + let name = import.name; + let prefix = if mingw { "" } else { "_" }; + + match import.calling_convention { + DllCallingConvention::C => format!("{}{}", prefix, name), + DllCallingConvention::Stdcall(arg_list_size) => { + format!("{}{}@{}", prefix, name, arg_list_size) + } + DllCallingConvention::Fastcall(arg_list_size) => format!("@{}@{}", name, arg_list_size), + DllCallingConvention::Vectorcall(arg_list_size) => { + format!("{}@@{}", name, arg_list_size) + } + } + } +} + +fn string_to_io_error(s: String) -> io::Error { + io::Error::new(io::ErrorKind::Other, format!("bad archive: {}", s)) +} + +fn find_binutils_dlltool(sess: &Session) -> OsString { + assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); + if let Some(dlltool_path) = &sess.opts.unstable_opts.dlltool { + return dlltool_path.clone().into_os_string(); + } + + let mut tool_name: OsString = if sess.host.arch != sess.target.arch { + // We are cross-compiling, so we need the tool with the prefix matching our target + if sess.target.arch == "x86" { + "i686-w64-mingw32-dlltool" + } else { + "x86_64-w64-mingw32-dlltool" + } + } else { + // We are not cross-compiling, so we just want `dlltool` + "dlltool" + } + .into(); + + if sess.host.options.is_like_windows { + // If we're compiling on Windows, add the .exe suffix + tool_name.push(".exe"); + } + + // NOTE: it's not clear how useful it is to explicitly search PATH. + for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { + let full_path = dir.join(&tool_name); + if full_path.is_file() { + return full_path.into_os_string(); + } + } + + // The user didn't specify the location of the dlltool binary, and we weren't able + // to find the appropriate one on the PATH. Just return the name of the tool + // and let the invocation fail with a hopefully useful error message. + tool_name +} diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs new file mode 100644 index 00000000000..3731c6bcfe7 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -0,0 +1,936 @@ +use crate::back::write::{ + self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers, +}; +use crate::llvm::archive_ro::ArchiveRO; +use crate::llvm::{self, build_string, False, True}; +use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; +use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; +use rustc_codegen_ssa::back::symbol_export; +use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig}; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; +use rustc_data_structures::fx::FxHashMap; +use rustc_errors::{FatalError, Handler}; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::bug; +use rustc_middle::dep_graph::WorkProduct; +use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; +use rustc_session::cgu_reuse_tracker::CguReuse; +use rustc_session::config::{self, CrateType, Lto}; +use tracing::{debug, info}; + +use std::ffi::{CStr, CString}; +use std::fs::File; +use std::io; +use std::iter; +use std::path::Path; +use std::ptr; +use std::slice; +use std::sync::Arc; + +/// We keep track of the computed LTO cache keys from the previous +/// session to determine which CGUs we can reuse. +pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; + +pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { + match crate_type { + CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => true, + CrateType::Dylib | CrateType::Rlib | CrateType::ProcMacro => false, + } +} + +fn prepare_lto( + cgcx: &CodegenContext, + diag_handler: &Handler, +) -> Result<(Vec, Vec<(SerializedModule, CString)>), FatalError> { + let export_threshold = match cgcx.lto { + // We're just doing LTO for our one crate + Lto::ThinLocal => SymbolExportLevel::Rust, + + // We're doing LTO for the entire crate graph + Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types), + + Lto::No => panic!("didn't request LTO but we're doing LTO"), + }; + + let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| { + if info.level.is_below_threshold(export_threshold) || info.used { + Some(CString::new(name.as_str()).unwrap()) + } else { + None + } + }; + let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); + let mut symbols_below_threshold = { + let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); + exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::>() + }; + info!("{} symbols to preserve in this crate", symbols_below_threshold.len()); + + // If we're performing LTO for the entire crate graph, then for each of our + // upstream dependencies, find the corresponding rlib and load the bitcode + // from the archive. + // + // We save off all the bytecode and LLVM module ids for later processing + // with either fat or thin LTO + let mut upstream_modules = Vec::new(); + if cgcx.lto != Lto::ThinLocal { + if cgcx.opts.cg.prefer_dynamic { + diag_handler + .struct_err("cannot prefer dynamic linking when performing LTO") + .note( + "only 'staticlib', 'bin', and 'cdylib' outputs are \ + supported with LTO", + ) + .emit(); + return Err(FatalError); + } + + // Make sure we actually can run LTO + for crate_type in cgcx.crate_types.iter() { + if !crate_type_allows_lto(*crate_type) { + let e = diag_handler.fatal( + "lto can only be run for executables, cdylibs and \ + static library outputs", + ); + return Err(e); + } + } + + for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() { + let exported_symbols = + cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); + { + let _timer = + cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); + symbols_below_threshold + .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); + } + + let archive = ArchiveRO::open(path).expect("wanted an rlib"); + let obj_files = archive + .iter() + .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c)))) + .filter(|&(name, _)| looks_like_rust_object_file(name)); + for (name, child) in obj_files { + info!("adding bitcode from {}", name); + match get_bitcode_slice_from_object_data(child.data()) { + Ok(data) => { + let module = SerializedModule::FromRlib(data.to_vec()); + upstream_modules.push((module, CString::new(name).unwrap())); + } + Err(msg) => return Err(diag_handler.fatal(&msg)), + } + } + } + } + + Ok((symbols_below_threshold, upstream_modules)) +} + +fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> { + let mut len = 0; + let data = + unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) }; + if !data.is_null() { + assert!(len != 0); + let bc = unsafe { slice::from_raw_parts(data, len) }; + + // `bc` must be a sub-slice of `obj`. + assert!(obj.as_ptr() <= bc.as_ptr()); + assert!(bc[bc.len()..bc.len()].as_ptr() <= obj[obj.len()..obj.len()].as_ptr()); + + Ok(bc) + } else { + assert!(len == 0); + let msg = llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()); + Err(format!("failed to get bitcode from object file for LTO ({})", msg)) + } +} + +/// Performs fat LTO by merging all modules into a single one and returning it +/// for further optimization. +pub(crate) fn run_fat( + cgcx: &CodegenContext, + modules: Vec>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, +) -> Result, FatalError> { + let diag_handler = cgcx.create_diag_handler(); + let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?; + let symbols_below_threshold = + symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); + fat_lto( + cgcx, + &diag_handler, + modules, + cached_modules, + upstream_modules, + &symbols_below_threshold, + ) +} + +/// Performs thin LTO by performing necessary global analysis and returning two +/// lists, one of the modules that need optimization and another for modules that +/// can simply be copied over from the incr. comp. cache. +pub(crate) fn run_thin( + cgcx: &CodegenContext, + modules: Vec<(String, ThinBuffer)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, +) -> Result<(Vec>, Vec), FatalError> { + let diag_handler = cgcx.create_diag_handler(); + let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?; + let symbols_below_threshold = + symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); + if cgcx.opts.cg.linker_plugin_lto.enabled() { + unreachable!( + "We should never reach this case if the LTO step \ + is deferred to the linker" + ); + } + thin_lto( + cgcx, + &diag_handler, + modules, + upstream_modules, + cached_modules, + &symbols_below_threshold, + ) +} + +pub(crate) fn prepare_thin(module: ModuleCodegen) -> (String, ThinBuffer) { + let name = module.name.clone(); + let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); + (name, buffer) +} + +fn fat_lto( + cgcx: &CodegenContext, + diag_handler: &Handler, + modules: Vec>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + mut serialized_modules: Vec<(SerializedModule, CString)>, + symbols_below_threshold: &[*const libc::c_char], +) -> Result, FatalError> { + let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module"); + info!("going for a fat lto"); + + // Sort out all our lists of incoming modules into two lists. + // + // * `serialized_modules` (also and argument to this function) contains all + // modules that are serialized in-memory. + // * `in_memory` contains modules which are already parsed and in-memory, + // such as from multi-CGU builds. + // + // All of `cached_modules` (cached from previous incremental builds) can + // immediately go onto the `serialized_modules` modules list and then we can + // split the `modules` array into these two lists. + let mut in_memory = Vec::new(); + serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| { + info!("pushing cached module {:?}", wp.cgu_name); + (buffer, CString::new(wp.cgu_name).unwrap()) + })); + for module in modules { + match module { + FatLTOInput::InMemory(m) => in_memory.push(m), + FatLTOInput::Serialized { name, buffer } => { + info!("pushing serialized module {:?}", name); + let buffer = SerializedModule::Local(buffer); + serialized_modules.push((buffer, CString::new(name).unwrap())); + } + } + } + + // Find the "costliest" module and merge everything into that codegen unit. + // All the other modules will be serialized and reparsed into the new + // context, so this hopefully avoids serializing and parsing the largest + // codegen unit. + // + // Additionally use a regular module as the base here to ensure that various + // file copy operations in the backend work correctly. The only other kind + // of module here should be an allocator one, and if your crate is smaller + // than the allocator module then the size doesn't really matter anyway. + let costliest_module = in_memory + .iter() + .enumerate() + .filter(|&(_, module)| module.kind == ModuleKind::Regular) + .map(|(i, module)| { + let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) }; + (cost, i) + }) + .max(); + + // If we found a costliest module, we're good to go. Otherwise all our + // inputs were serialized which could happen in the case, for example, that + // all our inputs were incrementally reread from the cache and we're just + // re-executing the LTO passes. If that's the case deserialize the first + // module and create a linker with it. + let module: ModuleCodegen = match costliest_module { + Some((_cost, i)) => in_memory.remove(i), + None => { + assert!(!serialized_modules.is_empty(), "must have at least one serialized module"); + let (buffer, name) = serialized_modules.remove(0); + info!("no in-memory regular modules to choose from, parsing {:?}", name); + ModuleCodegen { + module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), diag_handler)?, + name: name.into_string().unwrap(), + kind: ModuleKind::Regular, + } + } + }; + let mut serialized_bitcode = Vec::new(); + { + let (llcx, llmod) = { + let llvm = &module.module_llvm; + (&llvm.llcx, llvm.llmod()) + }; + info!("using {:?} as a base module", module.name); + + // The linking steps below may produce errors and diagnostics within LLVM + // which we'd like to handle and print, so set up our diagnostic handlers + // (which get unregistered when they go out of scope below). + let _handler = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + + // For all other modules we codegened we'll need to link them into our own + // bitcode. All modules were codegened in their own LLVM context, however, + // and we want to move everything to the same LLVM context. Currently the + // way we know of to do that is to serialize them to a string and them parse + // them later. Not great but hey, that's why it's "fat" LTO, right? + for module in in_memory { + let buffer = ModuleBuffer::new(module.module_llvm.llmod()); + let llmod_id = CString::new(&module.name[..]).unwrap(); + serialized_modules.push((SerializedModule::Local(buffer), llmod_id)); + } + // Sort the modules to ensure we produce deterministic results. + serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1)); + + // For all serialized bitcode files we parse them and link them in as we did + // above, this is all mostly handled in C++. Like above, though, we don't + // know much about the memory management here so we err on the side of being + // save and persist everything with the original module. + let mut linker = Linker::new(llmod); + for (bc_decoded, name) in serialized_modules { + let _timer = cgcx + .prof + .generic_activity_with_arg_recorder("LLVM_fat_lto_link_module", |recorder| { + recorder.record_arg(format!("{:?}", name)) + }); + info!("linking {:?}", name); + let data = bc_decoded.data(); + linker.add(data).map_err(|()| { + let msg = format!("failed to load bitcode of module {:?}", name); + write::llvm_err(diag_handler, &msg) + })?; + serialized_bitcode.push(bc_decoded); + } + drop(linker); + save_temp_bitcode(cgcx, &module, "lto.input"); + + // Internalize everything below threshold to help strip out more modules and such. + unsafe { + let ptr = symbols_below_threshold.as_ptr(); + llvm::LLVMRustRunRestrictionPass( + llmod, + ptr as *const *const libc::c_char, + symbols_below_threshold.len() as libc::size_t, + ); + save_temp_bitcode(cgcx, &module, "lto.after-restriction"); + } + } + + Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode }) +} + +pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); + +impl<'a> Linker<'a> { + pub(crate) fn new(llmod: &'a llvm::Module) -> Self { + unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) } + } + + pub(crate) fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { + unsafe { + if llvm::LLVMRustLinkerAdd( + self.0, + bytecode.as_ptr() as *const libc::c_char, + bytecode.len(), + ) { + Ok(()) + } else { + Err(()) + } + } + } +} + +impl Drop for Linker<'_> { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustLinkerFree(&mut *(self.0 as *mut _)); + } + } +} + +/// Prepare "thin" LTO to get run on these modules. +/// +/// The general structure of ThinLTO is quite different from the structure of +/// "fat" LTO above. With "fat" LTO all LLVM modules in question are merged into +/// one giant LLVM module, and then we run more optimization passes over this +/// big module after internalizing most symbols. Thin LTO, on the other hand, +/// avoid this large bottleneck through more targeted optimization. +/// +/// At a high level Thin LTO looks like: +/// +/// 1. Prepare a "summary" of each LLVM module in question which describes +/// the values inside, cost of the values, etc. +/// 2. Merge the summaries of all modules in question into one "index" +/// 3. Perform some global analysis on this index +/// 4. For each module, use the index and analysis calculated previously to +/// perform local transformations on the module, for example inlining +/// small functions from other modules. +/// 5. Run thin-specific optimization passes over each module, and then code +/// generate everything at the end. +/// +/// The summary for each module is intended to be quite cheap, and the global +/// index is relatively quite cheap to create as well. As a result, the goal of +/// ThinLTO is to reduce the bottleneck on LTO and enable LTO to be used in more +/// situations. For example one cheap optimization is that we can parallelize +/// all codegen modules, easily making use of all the cores on a machine. +/// +/// With all that in mind, the function here is designed at specifically just +/// calculating the *index* for ThinLTO. This index will then be shared amongst +/// all of the `LtoModuleCodegen` units returned below and destroyed once +/// they all go out of scope. +fn thin_lto( + cgcx: &CodegenContext, + diag_handler: &Handler, + modules: Vec<(String, ThinBuffer)>, + serialized_modules: Vec<(SerializedModule, CString)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + symbols_below_threshold: &[*const libc::c_char], +) -> Result<(Vec>, Vec), FatalError> { + let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis"); + unsafe { + info!("going for that thin, thin LTO"); + + let green_modules: FxHashMap<_, _> = + cached_modules.iter().map(|&(_, ref wp)| (wp.cgu_name.clone(), wp.clone())).collect(); + + let full_scope_len = modules.len() + serialized_modules.len() + cached_modules.len(); + let mut thin_buffers = Vec::with_capacity(modules.len()); + let mut module_names = Vec::with_capacity(full_scope_len); + let mut thin_modules = Vec::with_capacity(full_scope_len); + + for (i, (name, buffer)) in modules.into_iter().enumerate() { + info!("local module: {} - {}", i, name); + let cname = CString::new(name.clone()).unwrap(); + thin_modules.push(llvm::ThinLTOModule { + identifier: cname.as_ptr(), + data: buffer.data().as_ptr(), + len: buffer.data().len(), + }); + thin_buffers.push(buffer); + module_names.push(cname); + } + + // FIXME: All upstream crates are deserialized internally in the + // function below to extract their summary and modules. Note that + // unlike the loop above we *must* decode and/or read something + // here as these are all just serialized files on disk. An + // improvement, however, to make here would be to store the + // module summary separately from the actual module itself. Right + // now this is store in one large bitcode file, and the entire + // file is deflate-compressed. We could try to bypass some of the + // decompression by storing the index uncompressed and only + // lazily decompressing the bytecode if necessary. + // + // Note that truly taking advantage of this optimization will + // likely be further down the road. We'd have to implement + // incremental ThinLTO first where we could actually avoid + // looking at upstream modules entirely sometimes (the contents, + // we must always unconditionally look at the index). + let mut serialized = Vec::with_capacity(serialized_modules.len() + cached_modules.len()); + + let cached_modules = + cached_modules.into_iter().map(|(sm, wp)| (sm, CString::new(wp.cgu_name).unwrap())); + + for (module, name) in serialized_modules.into_iter().chain(cached_modules) { + info!("upstream or cached module {:?}", name); + thin_modules.push(llvm::ThinLTOModule { + identifier: name.as_ptr(), + data: module.data().as_ptr(), + len: module.data().len(), + }); + serialized.push(module); + module_names.push(name); + } + + // Sanity check + assert_eq!(thin_modules.len(), module_names.len()); + + // Delegate to the C++ bindings to create some data here. Once this is a + // tried-and-true interface we may wish to try to upstream some of this + // to LLVM itself, right now we reimplement a lot of what they do + // upstream... + let data = llvm::LLVMRustCreateThinLTOData( + thin_modules.as_ptr(), + thin_modules.len() as u32, + symbols_below_threshold.as_ptr(), + symbols_below_threshold.len() as u32, + ) + .ok_or_else(|| write::llvm_err(diag_handler, "failed to prepare thin LTO context"))?; + + let data = ThinData(data); + + info!("thin LTO data created"); + + let (key_map_path, prev_key_map, curr_key_map) = if let Some(ref incr_comp_session_dir) = + cgcx.incr_comp_session_dir + { + let path = incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME); + // If the previous file was deleted, or we get an IO error + // reading the file, then we'll just use `None` as the + // prev_key_map, which will force the code to be recompiled. + let prev = + if path.exists() { ThinLTOKeysMap::load_from_file(&path).ok() } else { None }; + let curr = ThinLTOKeysMap::from_thin_lto_modules(&data, &thin_modules, &module_names); + (Some(path), prev, curr) + } else { + // If we don't compile incrementally, we don't need to load the + // import data from LLVM. + assert!(green_modules.is_empty()); + let curr = ThinLTOKeysMap::default(); + (None, None, curr) + }; + info!("thin LTO cache key map loaded"); + info!("prev_key_map: {:#?}", prev_key_map); + info!("curr_key_map: {:#?}", curr_key_map); + + // Throw our data in an `Arc` as we'll be sharing it across threads. We + // also put all memory referenced by the C++ data (buffers, ids, etc) + // into the arc as well. After this we'll create a thin module + // codegen per module in this data. + let shared = Arc::new(ThinShared { + data, + thin_buffers, + serialized_modules: serialized, + module_names, + }); + + let mut copy_jobs = vec![]; + let mut opt_jobs = vec![]; + + info!("checking which modules can be-reused and which have to be re-optimized."); + for (module_index, module_name) in shared.module_names.iter().enumerate() { + let module_name = module_name_to_str(module_name); + if let (Some(prev_key_map), true) = + (prev_key_map.as_ref(), green_modules.contains_key(module_name)) + { + assert!(cgcx.incr_comp_session_dir.is_some()); + + // If a module exists in both the current and the previous session, + // and has the same LTO cache key in both sessions, then we can re-use it + if prev_key_map.keys.get(module_name) == curr_key_map.keys.get(module_name) { + let work_product = green_modules[module_name].clone(); + copy_jobs.push(work_product); + info!(" - {}: re-used", module_name); + assert!(cgcx.incr_comp_session_dir.is_some()); + cgcx.cgu_reuse_tracker.set_actual_reuse(module_name, CguReuse::PostLto); + continue; + } + } + + info!(" - {}: re-compiled", module_name); + opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { + shared: shared.clone(), + idx: module_index, + })); + } + + // Save the current ThinLTO import information for the next compilation + // session, overwriting the previous serialized data (if any). + if let Some(path) = key_map_path { + if let Err(err) = curr_key_map.save_to_file(&path) { + let msg = format!("Error while writing ThinLTO key data: {}", err); + return Err(write::llvm_err(diag_handler, &msg)); + } + } + + Ok((opt_jobs, copy_jobs)) + } +} + +pub(crate) fn run_pass_manager( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: &mut ModuleCodegen, + thin: bool, +) -> Result<(), FatalError> { + let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &*module.name); + let config = cgcx.config(module.kind); + + // Now we have one massive module inside of llmod. Time to run the + // LTO-specific optimization passes that LLVM provides. + // + // This code is based off the code found in llvm's LTO code generator: + // llvm/lib/LTO/LTOCodeGenerator.cpp + debug!("running the pass manager"); + unsafe { + if !llvm::LLVMRustHasModuleFlag( + module.module_llvm.llmod(), + "LTOPostLink".as_ptr().cast(), + 11, + ) { + llvm::LLVMRustAddModuleFlag( + module.module_llvm.llmod(), + llvm::LLVMModFlagBehavior::Error, + "LTOPostLink\0".as_ptr().cast(), + 1, + ); + } + if llvm_util::should_use_new_llvm_pass_manager( + &config.new_llvm_pass_manager, + &cgcx.target_arch, + ) { + let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; + let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); + write::optimize_with_new_llvm_pass_manager( + cgcx, + diag_handler, + module, + config, + opt_level, + opt_stage, + )?; + debug!("lto done"); + return Ok(()); + } + + let pm = llvm::LLVMCreatePassManager(); + llvm::LLVMAddAnalysisPasses(module.module_llvm.tm, pm); + + if config.verify_llvm_ir { + let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); + llvm::LLVMRustAddPass(pm, pass.unwrap()); + } + + let opt_level = config + .opt_level + .map(|x| to_llvm_opt_settings(x).0) + .unwrap_or(llvm::CodeGenOptLevel::None); + with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| { + if thin { + llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm); + } else { + llvm::LLVMRustPassManagerBuilderPopulateLTOPassManager( + b, pm, /* Internalize = */ False, /* RunInliner = */ True, + ); + } + }); + + // We always generate bitcode through ThinLTOBuffers, + // which do not support anonymous globals + if config.bitcode_needed() { + let pass = llvm::LLVMRustFindAndCreatePass("name-anon-globals\0".as_ptr().cast()); + llvm::LLVMRustAddPass(pm, pass.unwrap()); + } + + if config.verify_llvm_ir { + let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); + llvm::LLVMRustAddPass(pm, pass.unwrap()); + } + + llvm::LLVMRunPassManager(pm, module.module_llvm.llmod()); + + llvm::LLVMDisposePassManager(pm); + } + debug!("lto done"); + Ok(()) +} + +pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); + +unsafe impl Send for ModuleBuffer {} +unsafe impl Sync for ModuleBuffer {} + +impl ModuleBuffer { + pub fn new(m: &llvm::Module) -> ModuleBuffer { + ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) }) + } +} + +impl ModuleBufferMethods for ModuleBuffer { + fn data(&self) -> &[u8] { + unsafe { + let ptr = llvm::LLVMRustModuleBufferPtr(self.0); + let len = llvm::LLVMRustModuleBufferLen(self.0); + slice::from_raw_parts(ptr, len) + } + } +} + +impl Drop for ModuleBuffer { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustModuleBufferFree(&mut *(self.0 as *mut _)); + } + } +} + +pub struct ThinData(&'static mut llvm::ThinLTOData); + +unsafe impl Send for ThinData {} +unsafe impl Sync for ThinData {} + +impl Drop for ThinData { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustFreeThinLTOData(&mut *(self.0 as *mut _)); + } + } +} + +pub struct ThinBuffer(&'static mut llvm::ThinLTOBuffer); + +unsafe impl Send for ThinBuffer {} +unsafe impl Sync for ThinBuffer {} + +impl ThinBuffer { + pub fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { + unsafe { + let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); + ThinBuffer(buffer) + } + } +} + +impl ThinBufferMethods for ThinBuffer { + fn data(&self) -> &[u8] { + unsafe { + let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; + let len = llvm::LLVMRustThinLTOBufferLen(self.0); + slice::from_raw_parts(ptr, len) + } + } +} + +impl Drop for ThinBuffer { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _)); + } + } +} + +pub unsafe fn optimize_thin_module( + thin_module: ThinModule, + cgcx: &CodegenContext, +) -> Result, FatalError> { + let diag_handler = cgcx.create_diag_handler(); + + let module_name = &thin_module.shared.module_names[thin_module.idx]; + let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap()); + let tm = + (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, &e))?; + + // Right now the implementation we've got only works over serialized + // modules, so we create a fresh new LLVM context and parse the module + // into that context. One day, however, we may do this for upstream + // crates but for locally codegened modules we may be able to reuse + // that LLVM Context and Module. + let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); + let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _; + let mut module = ModuleCodegen { + module_llvm: ModuleLlvm { llmod_raw, llcx, tm }, + name: thin_module.name().to_string(), + kind: ModuleKind::Regular, + }; + { + let target = &*module.module_llvm.tm; + let llmod = module.module_llvm.llmod(); + save_temp_bitcode(cgcx, &module, "thin-lto-input"); + + // Before we do much else find the "main" `DICompileUnit` that we'll be + // using below. If we find more than one though then rustc has changed + // in a way we're not ready for, so generate an ICE by returning + // an error. + let mut cu1 = ptr::null_mut(); + let mut cu2 = ptr::null_mut(); + llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); + if !cu2.is_null() { + let msg = "multiple source DICompileUnits found"; + return Err(write::llvm_err(&diag_handler, msg)); + } + + // Up next comes the per-module local analyses that we do for Thin LTO. + // Each of these functions is basically copied from the LLVM + // implementation and then tailored to suit this implementation. Ideally + // each of these would be supported by upstream LLVM but that's perhaps + // a patch for another day! + // + // You can find some more comments about these functions in the LLVM + // bindings we've got (currently `PassWrapper.cpp`) + { + let _timer = + cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name()); + if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)); + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-rename"); + } + + { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name()); + if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)); + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); + } + + { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name()); + if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)); + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); + } + + { + let _timer = + cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name()); + if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)); + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); + } + + // Ok now this is a bit unfortunate. This is also something you won't + // find upstream in LLVM's ThinLTO passes! This is a hack for now to + // work around bugs in LLVM. + // + // First discovered in #45511 it was found that as part of ThinLTO + // importing passes LLVM will import `DICompileUnit` metadata + // information across modules. This means that we'll be working with one + // LLVM module that has multiple `DICompileUnit` instances in it (a + // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of + // bugs in LLVM's backend which generates invalid DWARF in a situation + // like this: + // + // https://bugs.llvm.org/show_bug.cgi?id=35212 + // https://bugs.llvm.org/show_bug.cgi?id=35562 + // + // While the first bug there is fixed the second ended up causing #46346 + // which was basically a resurgence of #45511 after LLVM's bug 35212 was + // fixed. + // + // This function below is a huge hack around this problem. The function + // below is defined in `PassWrapper.cpp` and will basically "merge" + // all `DICompileUnit` instances in a module. Basically it'll take all + // the objects, rewrite all pointers of `DISubprogram` to point to the + // first `DICompileUnit`, and then delete all the other units. + // + // This is probably mangling to the debug info slightly (but hopefully + // not too much) but for now at least gets LLVM to emit valid DWARF (or + // so it appears). Hopefully we can remove this once upstream bugs are + // fixed in LLVM. + { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); + llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); + save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); + } + + // Alright now that we've done everything related to the ThinLTO + // analysis it's time to run some optimizations! Here we use the same + // `run_pass_manager` as the "fat" LTO above except that we tell it to + // populate a thin-specific pass manager, which presumably LLVM treats a + // little differently. + { + info!("running thin lto passes over {}", module.name); + run_pass_manager(cgcx, &diag_handler, &mut module, true)?; + save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); + } + } + Ok(module) +} + +/// Maps LLVM module identifiers to their corresponding LLVM LTO cache keys +#[derive(Debug, Default)] +pub struct ThinLTOKeysMap { + // key = llvm name of importing module, value = LLVM cache key + keys: FxHashMap, +} + +impl ThinLTOKeysMap { + fn save_to_file(&self, path: &Path) -> io::Result<()> { + use std::io::Write; + let file = File::create(path)?; + let mut writer = io::BufWriter::new(file); + for (module, key) in &self.keys { + writeln!(writer, "{} {}", module, key)?; + } + Ok(()) + } + + fn load_from_file(path: &Path) -> io::Result { + use std::io::BufRead; + let mut keys = FxHashMap::default(); + let file = File::open(path)?; + for line in io::BufReader::new(file).lines() { + let line = line?; + let mut split = line.split(' '); + let module = split.next().unwrap(); + let key = split.next().unwrap(); + assert_eq!(split.next(), None, "Expected two space-separated values, found {:?}", line); + keys.insert(module.to_string(), key.to_string()); + } + Ok(Self { keys }) + } + + fn from_thin_lto_modules( + data: &ThinData, + modules: &[llvm::ThinLTOModule], + names: &[CString], + ) -> Self { + let keys = iter::zip(modules, names) + .map(|(module, name)| { + let key = build_string(|rust_str| unsafe { + llvm::LLVMRustComputeLTOCacheKey(rust_str, module.identifier, data.0); + }) + .expect("Invalid ThinLTO module key"); + (name.clone().into_string().unwrap(), key) + }) + .collect(); + Self { keys } + } +} + +fn module_name_to_str(c_str: &CStr) -> &str { + c_str.to_str().unwrap_or_else(|e| { + bug!("Encountered non-utf8 LLVM module name `{}`: {}", c_str.to_string_lossy(), e) + }) +} + +pub fn parse_module<'a>( + cx: &'a llvm::Context, + name: &CStr, + data: &[u8], + diag_handler: &Handler, +) -> Result<&'a llvm::Module, FatalError> { + unsafe { + llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else( + || { + let msg = "failed to parse bitcode for LTO module"; + write::llvm_err(diag_handler, msg) + }, + ) + } +} diff --git a/compiler/rustc_codegen_llvm/src/back/profiling.rs b/compiler/rustc_codegen_llvm/src/back/profiling.rs new file mode 100644 index 00000000000..2741f7d848e --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/profiling.rs @@ -0,0 +1,58 @@ +use measureme::{event_id::SEPARATOR_BYTE, EventId, StringComponent, StringId}; +use rustc_data_structures::profiling::{SelfProfiler, TimingGuard}; +use std::ffi::{c_void, CStr}; +use std::os::raw::c_char; +use std::sync::Arc; + +fn llvm_args_to_string_id(profiler: &SelfProfiler, pass_name: &str, ir_name: &str) -> EventId { + let pass_name = profiler.get_or_alloc_cached_string(pass_name); + let mut components = vec![StringComponent::Ref(pass_name)]; + // handle that LazyCallGraph::SCC is a comma separated list within parentheses + let parentheses: &[_] = &['(', ')']; + let trimmed = ir_name.trim_matches(parentheses); + for part in trimmed.split(", ") { + let demangled_ir_name = rustc_demangle::demangle(part).to_string(); + let ir_name = profiler.get_or_alloc_cached_string(demangled_ir_name); + components.push(StringComponent::Value(SEPARATOR_BYTE)); + components.push(StringComponent::Ref(ir_name)); + } + EventId::from_label(profiler.alloc_string(components.as_slice())) +} + +pub struct LlvmSelfProfiler<'a> { + profiler: Arc, + stack: Vec>, + llvm_pass_event_kind: StringId, +} + +impl<'a> LlvmSelfProfiler<'a> { + pub fn new(profiler: Arc) -> Self { + let llvm_pass_event_kind = profiler.alloc_string("LLVM Pass"); + Self { profiler, stack: Vec::default(), llvm_pass_event_kind } + } + + fn before_pass_callback(&'a mut self, pass_name: &str, ir_name: &str) { + let event_id = llvm_args_to_string_id(&self.profiler, pass_name, ir_name); + + self.stack.push(TimingGuard::start(&self.profiler, self.llvm_pass_event_kind, event_id)); + } + fn after_pass_callback(&mut self) { + self.stack.pop(); + } +} + +pub unsafe extern "C" fn selfprofile_before_pass_callback( + llvm_self_profiler: *mut c_void, + pass_name: *const c_char, + ir_name: *const c_char, +) { + let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>); + let pass_name = CStr::from_ptr(pass_name).to_str().expect("valid UTF-8"); + let ir_name = CStr::from_ptr(ir_name).to_str().expect("valid UTF-8"); + llvm_self_profiler.before_pass_callback(pass_name, ir_name); +} + +pub unsafe extern "C" fn selfprofile_after_pass_callback(llvm_self_profiler: *mut c_void) { + let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>); + llvm_self_profiler.after_pass_callback(); +} diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs new file mode 100644 index 00000000000..534d32e8a90 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -0,0 +1,1212 @@ +use crate::back::lto::ThinBuffer; +use crate::back::profiling::{ + selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler, +}; +use crate::base; +use crate::common; +use crate::consts; +use crate::llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; +use crate::llvm_util; +use crate::type_::Type; +use crate::LlvmCodegenBackend; +use crate::ModuleLlvm; +use rustc_codegen_ssa::back::link::ensure_removed; +use rustc_codegen_ssa::back::write::{ + BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig, + TargetMachineFactoryFn, +}; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; +use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_errors::{FatalError, Handler, Level}; +use rustc_fs_util::{link_or_copy, path_to_c_string}; +use rustc_middle::bug; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath}; +use rustc_session::Session; +use rustc_span::symbol::sym; +use rustc_span::InnerSpan; +use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; +use tracing::debug; + +use libc::{c_char, c_int, c_uint, c_void, size_t}; +use std::ffi::CString; +use std::fs; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::slice; +use std::str; +use std::sync::Arc; + +pub fn llvm_err(handler: &rustc_errors::Handler, msg: &str) -> FatalError { + match llvm::last_error() { + Some(err) => handler.fatal(&format!("{}: {}", msg, err)), + None => handler.fatal(msg), + } +} + +pub fn write_output_file<'ll>( + handler: &rustc_errors::Handler, + target: &'ll llvm::TargetMachine, + pm: &llvm::PassManager<'ll>, + m: &'ll llvm::Module, + output: &Path, + dwo_output: Option<&Path>, + file_type: llvm::FileType, + self_profiler_ref: &SelfProfilerRef, +) -> Result<(), FatalError> { + debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); + unsafe { + let output_c = path_to_c_string(output); + let dwo_output_c; + let dwo_output_ptr = if let Some(dwo_output) = dwo_output { + dwo_output_c = path_to_c_string(dwo_output); + dwo_output_c.as_ptr() + } else { + std::ptr::null() + }; + let result = llvm::LLVMRustWriteOutputFile( + target, + pm, + m, + output_c.as_ptr(), + dwo_output_ptr, + file_type, + ); + + // Record artifact sizes for self-profiling + if result == llvm::LLVMRustResult::Success { + let artifact_kind = match file_type { + llvm::FileType::ObjectFile => "object_file", + llvm::FileType::AssemblyFile => "assembly_file", + }; + record_artifact_size(self_profiler_ref, artifact_kind, output); + if let Some(dwo_file) = dwo_output { + record_artifact_size(self_profiler_ref, "dwo_file", dwo_file); + } + } + + result.into_result().map_err(|()| { + let msg = format!("could not write output to {}", output.display()); + llvm_err(handler, &msg) + }) + } +} + +pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm::TargetMachine { + let config = TargetMachineFactoryConfig { split_dwarf_file: None }; + // Can't use query system here quite yet because this function is invoked before the query + // system/tcx is set up. + let features = llvm_util::global_llvm_features(sess, false); + target_machine_factory(sess, config::OptLevel::No, &features)(config) + .unwrap_or_else(|err| llvm_err(sess.diagnostic(), &err).raise()) +} + +pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { + let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { + tcx.output_filenames(()).split_dwarf_path( + tcx.sess.split_debuginfo(), + tcx.sess.opts.unstable_opts.split_dwarf_kind, + Some(mod_name), + ) + } else { + None + }; + let config = TargetMachineFactoryConfig { split_dwarf_file }; + target_machine_factory( + &tcx.sess, + tcx.backend_optimization_level(()), + tcx.global_backend_features(()), + )(config) + .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) +} + +pub fn to_llvm_opt_settings( + cfg: config::OptLevel, +) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { + use self::config::OptLevel::*; + match cfg { + No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone), + Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone), + Default => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone), + Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone), + Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault), + SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive), + } +} + +fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel { + use config::OptLevel::*; + match cfg { + No => llvm::PassBuilderOptLevel::O0, + Less => llvm::PassBuilderOptLevel::O1, + Default => llvm::PassBuilderOptLevel::O2, + Aggressive => llvm::PassBuilderOptLevel::O3, + Size => llvm::PassBuilderOptLevel::Os, + SizeMin => llvm::PassBuilderOptLevel::Oz, + } +} + +fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel { + match relocation_model { + RelocModel::Static => llvm::RelocModel::Static, + // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra attribute. + RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC, + RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic, + RelocModel::Ropi => llvm::RelocModel::ROPI, + RelocModel::Rwpi => llvm::RelocModel::RWPI, + RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI, + } +} + +pub(crate) fn to_llvm_code_model(code_model: Option) -> llvm::CodeModel { + match code_model { + Some(CodeModel::Tiny) => llvm::CodeModel::Tiny, + Some(CodeModel::Small) => llvm::CodeModel::Small, + Some(CodeModel::Kernel) => llvm::CodeModel::Kernel, + Some(CodeModel::Medium) => llvm::CodeModel::Medium, + Some(CodeModel::Large) => llvm::CodeModel::Large, + None => llvm::CodeModel::None, + } +} + +pub fn target_machine_factory( + sess: &Session, + optlvl: config::OptLevel, + target_features: &[String], +) -> TargetMachineFactoryFn { + let reloc_model = to_llvm_relocation_model(sess.relocation_model()); + + let (opt_level, _) = to_llvm_opt_settings(optlvl); + let use_softfp = sess.opts.cg.soft_float; + + let ffunction_sections = + sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections); + let fdata_sections = ffunction_sections; + let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names; + + let code_model = to_llvm_code_model(sess.code_model()); + + let mut singlethread = sess.target.singlethread; + + // On the wasm target once the `atomics` feature is enabled that means that + // we're no longer single-threaded, or otherwise we don't want LLVM to + // lower atomic operations to single-threaded operations. + if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) { + singlethread = false; + } + + let triple = SmallCStr::new(&sess.target.llvm_target); + let cpu = SmallCStr::new(llvm_util::target_cpu(sess)); + let features = CString::new(target_features.join(",")).unwrap(); + let abi = SmallCStr::new(&sess.target.llvm_abiname); + let trap_unreachable = + sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); + let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; + + let asm_comments = sess.asm_comments(); + let relax_elf_relocations = + sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); + + let use_init_array = + !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section); + + let path_mapping = sess.source_map().path_mapping().clone(); + + Arc::new(move |config: TargetMachineFactoryConfig| { + let split_dwarf_file = + path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; + let split_dwarf_file = CString::new(split_dwarf_file.to_str().unwrap()).unwrap(); + + let tm = unsafe { + llvm::LLVMRustCreateTargetMachine( + triple.as_ptr(), + cpu.as_ptr(), + features.as_ptr(), + abi.as_ptr(), + code_model, + reloc_model, + opt_level, + use_softfp, + ffunction_sections, + fdata_sections, + funique_section_names, + trap_unreachable, + singlethread, + asm_comments, + emit_stack_size_section, + relax_elf_relocations, + use_init_array, + split_dwarf_file.as_ptr(), + ) + }; + + tm.ok_or_else(|| { + format!("Could not create LLVM TargetMachine for triple: {}", triple.to_str().unwrap()) + }) + }) +} + +pub(crate) fn save_temp_bitcode( + cgcx: &CodegenContext, + module: &ModuleCodegen, + name: &str, +) { + if !cgcx.save_temps { + return; + } + unsafe { + let ext = format!("{}.bc", name); + let cgu = Some(&module.name[..]); + let path = cgcx.output_filenames.temp_path_ext(&ext, cgu); + let cstr = path_to_c_string(&path); + let llmod = module.module_llvm.llmod(); + llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); + } +} + +pub struct DiagnosticHandlers<'a> { + data: *mut (&'a CodegenContext, &'a Handler), + llcx: &'a llvm::Context, + old_handler: Option<&'a llvm::DiagnosticHandler>, +} + +impl<'a> DiagnosticHandlers<'a> { + pub fn new( + cgcx: &'a CodegenContext, + handler: &'a Handler, + llcx: &'a llvm::Context, + ) -> Self { + let remark_passes_all: bool; + let remark_passes: Vec; + match &cgcx.remark { + Passes::All => { + remark_passes_all = true; + remark_passes = Vec::new(); + } + Passes::Some(passes) => { + remark_passes_all = false; + remark_passes = + passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect(); + } + }; + let remark_passes: Vec<*const c_char> = + remark_passes.iter().map(|name: &CString| name.as_ptr()).collect(); + let data = Box::into_raw(Box::new((cgcx, handler))); + unsafe { + let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx); + llvm::LLVMRustContextConfigureDiagnosticHandler( + llcx, + diagnostic_handler, + data.cast(), + remark_passes_all, + remark_passes.as_ptr(), + remark_passes.len(), + ); + llvm::LLVMRustSetInlineAsmDiagnosticHandler(llcx, inline_asm_handler, data.cast()); + DiagnosticHandlers { data, llcx, old_handler } + } + } +} + +impl<'a> Drop for DiagnosticHandlers<'a> { + fn drop(&mut self) { + use std::ptr::null_mut; + unsafe { + llvm::LLVMRustSetInlineAsmDiagnosticHandler(self.llcx, inline_asm_handler, null_mut()); + llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler); + drop(Box::from_raw(self.data)); + } + } +} + +fn report_inline_asm( + cgcx: &CodegenContext, + msg: String, + level: llvm::DiagnosticLevel, + mut cookie: c_uint, + source: Option<(String, Vec)>, +) { + // In LTO build we may get srcloc values from other crates which are invalid + // since they use a different source map. To be safe we just suppress these + // in LTO builds. + if matches!(cgcx.lto, Lto::Fat | Lto::Thin) { + cookie = 0; + } + let level = match level { + llvm::DiagnosticLevel::Error => Level::Error { lint: false }, + llvm::DiagnosticLevel::Warning => Level::Warning(None), + llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, + }; + cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); +} + +unsafe extern "C" fn inline_asm_handler(diag: &SMDiagnostic, user: *const c_void, cookie: c_uint) { + if user.is_null() { + return; + } + let (cgcx, _) = *(user as *const (&CodegenContext, &Handler)); + + let smdiag = llvm::diagnostic::SrcMgrDiagnostic::unpack(diag); + report_inline_asm(cgcx, smdiag.message, smdiag.level, cookie, smdiag.source); +} + +unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { + if user.is_null() { + return; + } + let (cgcx, diag_handler) = *(user as *const (&CodegenContext, &Handler)); + + match llvm::diagnostic::Diagnostic::unpack(info) { + llvm::diagnostic::InlineAsm(inline) => { + report_inline_asm(cgcx, inline.message, inline.level, inline.cookie, inline.source); + } + + llvm::diagnostic::Optimization(opt) => { + let enabled = match cgcx.remark { + Passes::All => true, + Passes::Some(ref v) => v.iter().any(|s| *s == opt.pass_name), + }; + + if enabled { + diag_handler.note_without_error(&format!( + "{}:{}:{}: {}: {}", + opt.filename, opt.line, opt.column, opt.pass_name, opt.message, + )); + } + } + llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => { + let msg = llvm::build_string(|s| { + llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) + }) + .expect("non-UTF8 diagnostic"); + diag_handler.warn(&msg); + } + llvm::diagnostic::Unsupported(diagnostic_ref) => { + let msg = llvm::build_string(|s| { + llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) + }) + .expect("non-UTF8 diagnostic"); + diag_handler.err(&msg); + } + llvm::diagnostic::UnknownDiagnostic(..) => {} + } +} + +fn get_pgo_gen_path(config: &ModuleConfig) -> Option { + match config.pgo_gen { + SwitchWithOptPath::Enabled(ref opt_dir_path) => { + let path = if let Some(dir_path) = opt_dir_path { + dir_path.join("default_%m.profraw") + } else { + PathBuf::from("default_%m.profraw") + }; + + Some(CString::new(format!("{}", path.display())).unwrap()) + } + SwitchWithOptPath::Disabled => None, + } +} + +fn get_pgo_use_path(config: &ModuleConfig) -> Option { + config + .pgo_use + .as_ref() + .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) +} + +fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option { + config + .pgo_sample_use + .as_ref() + .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) +} + +pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: &ModuleCodegen, + config: &ModuleConfig, + opt_level: config::OptLevel, + opt_stage: llvm::OptStage, +) -> Result<(), FatalError> { + let unroll_loops = + opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin; + let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed(); + let pgo_gen_path = get_pgo_gen_path(config); + let pgo_use_path = get_pgo_use_path(config); + let pgo_sample_use_path = get_pgo_sample_use_path(config); + let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; + // Sanitizer instrumentation is only inserted during the pre-link optimization stage. + let sanitizer_options = if !is_lto { + Some(llvm::SanitizerOptions { + sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS), + sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS), + sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY), + sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY), + sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int, + sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD), + sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS), + sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS), + }) + } else { + None + }; + + let mut llvm_profiler = if cgcx.prof.llvm_recording_enabled() { + Some(LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())) + } else { + None + }; + + let llvm_selfprofiler = + llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); + + let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() }; + + let llvm_plugins = config.llvm_plugins.join(","); + + // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. + // We would have to add upstream support for this first, before we can support + // config.inline_threshold and our more aggressive default thresholds. + let result = llvm::LLVMRustOptimizeWithNewPassManager( + module.module_llvm.llmod(), + &*module.module_llvm.tm, + to_pass_builder_opt_level(opt_level), + opt_stage, + config.no_prepopulate_passes, + config.verify_llvm_ir, + using_thin_buffers, + config.merge_functions, + unroll_loops, + config.vectorize_slp, + config.vectorize_loop, + config.no_builtins, + config.emit_lifetime_markers, + sanitizer_options.as_ref(), + pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + config.instrument_coverage, + config.instrument_gcov, + pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + config.debug_info_for_profiling, + llvm_selfprofiler, + selfprofile_before_pass_callback, + selfprofile_after_pass_callback, + extra_passes.as_ptr().cast(), + extra_passes.len(), + llvm_plugins.as_ptr().cast(), + llvm_plugins.len(), + ); + result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes")) +} + +// Unsafe due to LLVM calls. +pub(crate) unsafe fn optimize( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: &ModuleCodegen, + config: &ModuleConfig, +) -> Result<(), FatalError> { + let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); + + let llmod = module.module_llvm.llmod(); + let llcx = &*module.module_llvm.llcx; + let tm = &*module.module_llvm.tm; + let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + + let module_name = module.name.clone(); + let module_name = Some(&module_name[..]); + + if let Some(false) = config.new_llvm_pass_manager && llvm_util::get_version() >= (15, 0, 0) { + diag_handler.warn( + "ignoring `-Z new-llvm-pass-manager=no`, which is no longer supported with LLVM 15", + ); + } + + if config.emit_no_opt_bc { + let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); + let out = path_to_c_string(&out); + llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); + } + + if let Some(opt_level) = config.opt_level { + if llvm_util::should_use_new_llvm_pass_manager( + &config.new_llvm_pass_manager, + &cgcx.target_arch, + ) { + let opt_stage = match cgcx.lto { + Lto::Fat => llvm::OptStage::PreLinkFatLTO, + Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, + _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, + _ => llvm::OptStage::PreLinkNoLTO, + }; + return optimize_with_new_llvm_pass_manager( + cgcx, + diag_handler, + module, + config, + opt_level, + opt_stage, + ); + } + + if cgcx.prof.llvm_recording_enabled() { + diag_handler + .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`"); + } + + // Create the two optimizing pass managers. These mirror what clang + // does, and are by populated by LLVM's default PassManagerBuilder. + // Each manager has a different set of passes, but they also share + // some common passes. + let fpm = llvm::LLVMCreateFunctionPassManagerForModule(llmod); + let mpm = llvm::LLVMCreatePassManager(); + + { + let find_pass = |pass_name: &str| { + let pass_name = SmallCStr::new(pass_name); + llvm::LLVMRustFindAndCreatePass(pass_name.as_ptr()) + }; + + if config.verify_llvm_ir { + // Verification should run as the very first pass. + llvm::LLVMRustAddPass(fpm, find_pass("verify").unwrap()); + } + + let mut extra_passes = Vec::new(); + let mut have_name_anon_globals_pass = false; + + for pass_name in &config.passes { + if pass_name == "lint" { + // Linting should also be performed early, directly on the generated IR. + llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap()); + continue; + } + + if let Some(pass) = find_pass(pass_name) { + extra_passes.push(pass); + } else { + diag_handler.warn(&format!("unknown pass `{}`, ignoring", pass_name)); + } + + if pass_name == "name-anon-globals" { + have_name_anon_globals_pass = true; + } + } + + // Instrumentation must be inserted before optimization, + // otherwise LLVM may optimize some functions away which + // breaks llvm-cov. + // + // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp. + if config.instrument_gcov { + llvm::LLVMRustAddPass(mpm, find_pass("insert-gcov-profiling").unwrap()); + } + if config.instrument_coverage { + llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap()); + } + if config.debug_info_for_profiling { + llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap()); + } + + add_sanitizer_passes(config, &mut extra_passes); + + // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need + // to make sure we run LLVM's NameAnonGlobals pass when emitting bitcode; otherwise + // we'll get errors in LLVM. + let using_thin_buffers = config.bitcode_needed(); + if !config.no_prepopulate_passes { + llvm::LLVMAddAnalysisPasses(tm, fpm); + llvm::LLVMAddAnalysisPasses(tm, mpm); + let opt_level = to_llvm_opt_settings(opt_level).0; + let prepare_for_thin_lto = cgcx.lto == Lto::Thin + || cgcx.lto == Lto::ThinLocal + || (cgcx.lto != Lto::Fat && cgcx.opts.cg.linker_plugin_lto.enabled()); + with_llvm_pmb(llmod, config, opt_level, prepare_for_thin_lto, &mut |b| { + llvm::LLVMRustAddLastExtensionPasses( + b, + extra_passes.as_ptr(), + extra_passes.len() as size_t, + ); + llvm::LLVMRustPassManagerBuilderPopulateFunctionPassManager(b, fpm); + llvm::LLVMRustPassManagerBuilderPopulateModulePassManager(b, mpm); + }); + + have_name_anon_globals_pass = have_name_anon_globals_pass || prepare_for_thin_lto; + if using_thin_buffers && !prepare_for_thin_lto { + llvm::LLVMRustAddPass(mpm, find_pass("name-anon-globals").unwrap()); + have_name_anon_globals_pass = true; + } + } else { + // If we don't use the standard pipeline, directly populate the MPM + // with the extra passes. + for pass in extra_passes { + llvm::LLVMRustAddPass(mpm, pass); + } + } + + if using_thin_buffers && !have_name_anon_globals_pass { + // As described above, this will probably cause an error in LLVM + if config.no_prepopulate_passes { + diag_handler.err( + "The current compilation is going to use thin LTO buffers \ + without running LLVM's NameAnonGlobals pass. \ + This will likely cause errors in LLVM. Consider adding \ + -C passes=name-anon-globals to the compiler command line.", + ); + } else { + bug!( + "We are using thin LTO buffers without running the NameAnonGlobals pass. \ + This will likely cause errors in LLVM and should never happen." + ); + } + } + } + + diag_handler.abort_if_errors(); + + // Finally, run the actual optimization passes + { + let _timer = cgcx.prof.extra_verbose_generic_activity( + "LLVM_module_optimize_function_passes", + &*module.name, + ); + llvm::LLVMRustRunFunctionPassManager(fpm, llmod); + } + { + let _timer = cgcx.prof.extra_verbose_generic_activity( + "LLVM_module_optimize_module_passes", + &*module.name, + ); + llvm::LLVMRunPassManager(mpm, llmod); + } + + // Deallocate managers that we're now done with + llvm::LLVMDisposePassManager(fpm); + llvm::LLVMDisposePassManager(mpm); + } + Ok(()) +} + +unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static mut llvm::Pass>) { + if config.sanitizer.contains(SanitizerSet::ADDRESS) { + let recover = config.sanitizer_recover.contains(SanitizerSet::ADDRESS); + passes.push(llvm::LLVMRustCreateAddressSanitizerFunctionPass(recover)); + passes.push(llvm::LLVMRustCreateModuleAddressSanitizerPass(recover)); + } + if config.sanitizer.contains(SanitizerSet::MEMORY) { + let track_origins = config.sanitizer_memory_track_origins as c_int; + let recover = config.sanitizer_recover.contains(SanitizerSet::MEMORY); + passes.push(llvm::LLVMRustCreateMemorySanitizerPass(track_origins, recover)); + } + if config.sanitizer.contains(SanitizerSet::THREAD) { + passes.push(llvm::LLVMRustCreateThreadSanitizerPass()); + } + if config.sanitizer.contains(SanitizerSet::HWADDRESS) { + let recover = config.sanitizer_recover.contains(SanitizerSet::HWADDRESS); + passes.push(llvm::LLVMRustCreateHWAddressSanitizerPass(recover)); + } +} + +pub(crate) fn link( + cgcx: &CodegenContext, + diag_handler: &Handler, + mut modules: Vec>, +) -> Result, FatalError> { + use super::lto::{Linker, ModuleBuffer}; + // Sort the modules by name to ensure to ensure deterministic behavior. + modules.sort_by(|a, b| a.name.cmp(&b.name)); + let (first, elements) = + modules.split_first().expect("Bug! modules must contain at least one module."); + + let mut linker = Linker::new(first.module_llvm.llmod()); + for module in elements { + let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); + let buffer = ModuleBuffer::new(module.module_llvm.llmod()); + linker.add(buffer.data()).map_err(|()| { + let msg = format!("failed to serialize module {:?}", module.name); + llvm_err(diag_handler, &msg) + })?; + } + drop(linker); + Ok(modules.remove(0)) +} + +pub(crate) unsafe fn codegen( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: ModuleCodegen, + config: &ModuleConfig, +) -> Result { + let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name); + { + let llmod = module.module_llvm.llmod(); + let llcx = &*module.module_llvm.llcx; + let tm = &*module.module_llvm.tm; + let module_name = module.name.clone(); + let module_name = Some(&module_name[..]); + let handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + + if cgcx.msvc_imps_needed { + create_msvc_imps(cgcx, llcx, llmod); + } + + // A codegen-specific pass manager is used to generate object + // files for an LLVM module. + // + // Apparently each of these pass managers is a one-shot kind of + // thing, so we create a new one for each type of output. The + // pass manager passed to the closure should be ensured to not + // escape the closure itself, and the manager should only be + // used once. + unsafe fn with_codegen<'ll, F, R>( + tm: &'ll llvm::TargetMachine, + llmod: &'ll llvm::Module, + no_builtins: bool, + f: F, + ) -> R + where + F: FnOnce(&'ll mut PassManager<'ll>) -> R, + { + let cpm = llvm::LLVMCreatePassManager(); + llvm::LLVMAddAnalysisPasses(tm, cpm); + llvm::LLVMRustAddLibraryInfo(cpm, llmod, no_builtins); + f(cpm) + } + + // Two things to note: + // - If object files are just LLVM bitcode we write bitcode, copy it to + // the .o file, and delete the bitcode if it wasn't otherwise + // requested. + // - If we don't have the integrated assembler then we need to emit + // asm from LLVM and use `gcc` to create the object file. + + let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name); + let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, module_name); + + if config.bitcode_needed() { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name); + let thin = ThinBuffer::new(llmod, config.emit_thin_lto); + let data = thin.data(); + + if let Some(bitcode_filename) = bc_out.file_name() { + cgcx.prof.artifact_size( + "llvm_bitcode", + bitcode_filename.to_string_lossy(), + data.len() as u64, + ); + } + + if config.emit_bc || config.emit_obj == EmitObj::Bitcode { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name); + if let Err(e) = fs::write(&bc_out, data) { + let msg = format!("failed to write bytecode to {}: {}", bc_out.display(), e); + diag_handler.err(&msg); + } + } + + if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name); + embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data); + } + } + + if config.emit_ir { + let _timer = + cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name); + let out = cgcx.output_filenames.temp_path(OutputType::LlvmAssembly, module_name); + let out_c = path_to_c_string(&out); + + extern "C" fn demangle_callback( + input_ptr: *const c_char, + input_len: size_t, + output_ptr: *mut c_char, + output_len: size_t, + ) -> size_t { + let input = + unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) }; + + let Ok(input) = str::from_utf8(input) else { return 0 }; + + let output = unsafe { + slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize) + }; + let mut cursor = io::Cursor::new(output); + + let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 }; + + if write!(cursor, "{:#}", demangled).is_err() { + // Possible only if provided buffer is not big enough + return 0; + } + + cursor.position() as size_t + } + + let result = llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback); + + if result == llvm::LLVMRustResult::Success { + record_artifact_size(&cgcx.prof, "llvm_ir", &out); + } + + result.into_result().map_err(|()| { + let msg = format!("failed to write LLVM IR to {}", out.display()); + llvm_err(diag_handler, &msg) + })?; + } + + if config.emit_asm { + let _timer = + cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name); + let path = cgcx.output_filenames.temp_path(OutputType::Assembly, module_name); + + // We can't use the same module for asm and object code output, + // because that triggers various errors like invalid IR or broken + // binaries. So we must clone the module to produce the asm output + // if we are also producing object code. + let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj { + llvm::LLVMCloneModule(llmod) + } else { + llmod + }; + with_codegen(tm, llmod, config.no_builtins, |cpm| { + write_output_file( + diag_handler, + tm, + cpm, + llmod, + &path, + None, + llvm::FileType::AssemblyFile, + &cgcx.prof, + ) + })?; + } + + match config.emit_obj { + EmitObj::ObjectCode(_) => { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name); + + let dwo_out = cgcx.output_filenames.temp_path_dwo(module_name); + let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) { + // Don't change how DWARF is emitted when disabled. + (SplitDebuginfo::Off, _) => None, + // Don't provide a DWARF object path if split debuginfo is enabled but this is + // a platform that doesn't support Split DWARF. + _ if !cgcx.target_can_use_split_dwarf => None, + // Don't provide a DWARF object path in single mode, sections will be written + // into the object as normal but ignored by linker. + (_, SplitDwarfKind::Single) => None, + // Emit (a subset of the) DWARF into a separate dwarf object file in split + // mode. + (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()), + }; + + with_codegen(tm, llmod, config.no_builtins, |cpm| { + write_output_file( + diag_handler, + tm, + cpm, + llmod, + &obj_out, + dwo_out, + llvm::FileType::ObjectFile, + &cgcx.prof, + ) + })?; + } + + EmitObj::Bitcode => { + debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out); + if let Err(e) = link_or_copy(&bc_out, &obj_out) { + diag_handler.err(&format!("failed to copy bitcode to object file: {}", e)); + } + + if !config.emit_bc { + debug!("removing_bitcode {:?}", bc_out); + ensure_removed(diag_handler, &bc_out); + } + } + + EmitObj::None => {} + } + + drop(handlers); + } + + Ok(module.into_compiled_module( + config.emit_obj != EmitObj::None, + cgcx.target_can_use_split_dwarf + && cgcx.split_debuginfo != SplitDebuginfo::Off + && cgcx.split_dwarf_kind == SplitDwarfKind::Split, + config.emit_bc, + &cgcx.output_filenames, + )) +} + +fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec { + let mut asm = format!(".section {},\"{}\"\n", section_name, section_flags).into_bytes(); + asm.extend_from_slice(b".ascii \""); + asm.reserve(data.len()); + for &byte in data { + if byte == b'\\' || byte == b'"' { + asm.push(b'\\'); + asm.push(byte); + } else if byte < 0x20 || byte >= 0x80 { + // Avoid non UTF-8 inline assembly. Use octal escape sequence, because it is fixed + // width, while hex escapes will consume following characters. + asm.push(b'\\'); + asm.push(b'0' + ((byte >> 6) & 0x7)); + asm.push(b'0' + ((byte >> 3) & 0x7)); + asm.push(b'0' + ((byte >> 0) & 0x7)); + } else { + asm.push(byte); + } + } + asm.extend_from_slice(b"\"\n"); + asm +} + +/// Embed the bitcode of an LLVM module in the LLVM module itself. +/// +/// This is done primarily for iOS where it appears to be standard to compile C +/// code at least with `-fembed-bitcode` which creates two sections in the +/// executable: +/// +/// * __LLVM,__bitcode +/// * __LLVM,__cmdline +/// +/// It appears *both* of these sections are necessary to get the linker to +/// recognize what's going on. A suitable cmdline value is taken from the +/// target spec. +/// +/// Furthermore debug/O1 builds don't actually embed bitcode but rather just +/// embed an empty section. +/// +/// Basically all of this is us attempting to follow in the footsteps of clang +/// on iOS. See #35968 for lots more info. +unsafe fn embed_bitcode( + cgcx: &CodegenContext, + llcx: &llvm::Context, + llmod: &llvm::Module, + cmdline: &str, + bitcode: &[u8], +) { + // We're adding custom sections to the output object file, but we definitely + // do not want these custom sections to make their way into the final linked + // executable. The purpose of these custom sections is for tooling + // surrounding object files to work with the LLVM IR, if necessary. For + // example rustc's own LTO will look for LLVM IR inside of the object file + // in these sections by default. + // + // To handle this is a bit different depending on the object file format + // used by the backend, broken down into a few different categories: + // + // * Mach-O - this is for macOS. Inspecting the source code for the native + // linker here shows that the `.llvmbc` and `.llvmcmd` sections are + // automatically skipped by the linker. In that case there's nothing extra + // that we need to do here. + // + // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and + // `.llvmcmd` sections, so there's nothing extra we need to do. + // + // * COFF - if we don't do anything the linker will by default copy all + // these sections to the output artifact, not what we want! To subvert + // this we want to flag the sections we inserted here as + // `IMAGE_SCN_LNK_REMOVE`. + // + // * ELF - this is very similar to COFF above. One difference is that these + // sections are removed from the output linked artifact when + // `--gc-sections` is passed, which we pass by default. If that flag isn't + // passed though then these sections will show up in the final output. + // Additionally the flag that we need to set here is `SHF_EXCLUDE`. + // + // Unfortunately, LLVM provides no way to set custom section flags. For ELF + // and COFF we emit the sections using module level inline assembly for that + // reason (see issue #90326 for historical background). + let is_apple = cgcx.opts.target_triple.triple().contains("-ios") + || cgcx.opts.target_triple.triple().contains("-darwin") + || cgcx.opts.target_triple.triple().contains("-tvos") + || cgcx.opts.target_triple.triple().contains("-watchos"); + if is_apple + || cgcx.opts.target_triple.triple().starts_with("wasm") + || cgcx.opts.target_triple.triple().starts_with("asmjs") + { + // We don't need custom section flags, create LLVM globals. + let llconst = common::bytes_in_context(llcx, bitcode); + let llglobal = llvm::LLVMAddGlobal( + llmod, + common::val_ty(llconst), + "rustc.embedded.module\0".as_ptr().cast(), + ); + llvm::LLVMSetInitializer(llglobal, llconst); + + let section = if is_apple { "__LLVM,__bitcode\0" } else { ".llvmbc\0" }; + llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); + llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::LLVMSetGlobalConstant(llglobal, llvm::True); + + let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); + let llglobal = llvm::LLVMAddGlobal( + llmod, + common::val_ty(llconst), + "rustc.embedded.cmdline\0".as_ptr().cast(), + ); + llvm::LLVMSetInitializer(llglobal, llconst); + let section = if is_apple { "__LLVM,__cmdline\0" } else { ".llvmcmd\0" }; + llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); + llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); + } else { + // We need custom section flags, so emit module-level inline assembly. + let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; + let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); + llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); + llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + } +} + +pub unsafe fn with_llvm_pmb( + llmod: &llvm::Module, + config: &ModuleConfig, + opt_level: llvm::CodeGenOptLevel, + prepare_for_thin_lto: bool, + f: &mut dyn FnMut(&llvm::PassManagerBuilder), +) { + use std::ptr; + + // Create the PassManagerBuilder for LLVM. We configure it with + // reasonable defaults and prepare it to actually populate the pass + // manager. + let builder = llvm::LLVMRustPassManagerBuilderCreate(); + let opt_size = config.opt_size.map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); + let inline_threshold = config.inline_threshold; + let pgo_gen_path = get_pgo_gen_path(config); + let pgo_use_path = get_pgo_use_path(config); + let pgo_sample_use_path = get_pgo_sample_use_path(config); + + llvm::LLVMRustConfigurePassManagerBuilder( + builder, + opt_level, + config.merge_functions, + config.vectorize_slp, + config.vectorize_loop, + prepare_for_thin_lto, + pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + opt_size as c_int, + ); + + llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins); + + // Here we match what clang does (kinda). For O0 we only inline + // always-inline functions (but don't add lifetime intrinsics), at O1 we + // inline with lifetime intrinsics, and O2+ we add an inliner with a + // thresholds copied from clang. + match (opt_level, opt_size, inline_threshold) { + (.., Some(t)) => { + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, t); + } + (llvm::CodeGenOptLevel::Aggressive, ..) => { + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 275); + } + (_, llvm::CodeGenOptSizeDefault, _) => { + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 75); + } + (_, llvm::CodeGenOptSizeAggressive, _) => { + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 25); + } + (llvm::CodeGenOptLevel::None, ..) => { + llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); + } + (llvm::CodeGenOptLevel::Less, ..) => { + llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); + } + (llvm::CodeGenOptLevel::Default, ..) => { + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 225); + } + } + + f(builder); + llvm::LLVMRustPassManagerBuilderDispose(builder); +} + +// Create a `__imp_ = &symbol` global for every public static `symbol`. +// This is required to satisfy `dllimport` references to static data in .rlibs +// when using MSVC linker. We do this only for data, as linker can fix up +// code references on its own. +// See #26591, #27438 +fn create_msvc_imps( + cgcx: &CodegenContext, + llcx: &llvm::Context, + llmod: &llvm::Module, +) { + if !cgcx.msvc_imps_needed { + return; + } + // The x86 ABI seems to require that leading underscores are added to symbol + // names, so we need an extra underscore on x86. There's also a leading + // '\x01' here which disables LLVM's symbol mangling (e.g., no extra + // underscores added in front). + let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" }; + + unsafe { + let i8p_ty = Type::i8p_llcx(llcx); + let globals = base::iter_globals(llmod) + .filter(|&val| { + llvm::LLVMRustGetLinkage(val) == llvm::Linkage::ExternalLinkage + && llvm::LLVMIsDeclaration(val) == 0 + }) + .filter_map(|val| { + // Exclude some symbols that we know are not Rust symbols. + let name = llvm::get_value_name(val); + if ignored(name) { None } else { Some((val, name)) } + }) + .map(move |(val, name)| { + let mut imp_name = prefix.as_bytes().to_vec(); + imp_name.extend(name); + let imp_name = CString::new(imp_name).unwrap(); + (imp_name, val) + }) + .collect::>(); + + for (imp_name, val) in globals { + let imp = llvm::LLVMAddGlobal(llmod, i8p_ty, imp_name.as_ptr().cast()); + llvm::LLVMSetInitializer(imp, consts::ptrcast(val, i8p_ty)); + llvm::LLVMRustSetLinkage(imp, llvm::Linkage::ExternalLinkage); + } + } + + // Use this function to exclude certain symbols from `__imp` generation. + fn ignored(symbol_name: &[u8]) -> bool { + // These are symbols generated by LLVM's profiling instrumentation + symbol_name.starts_with(b"__llvm_profile_") + } +} + +fn record_artifact_size( + self_profiler_ref: &SelfProfilerRef, + artifact_kind: &'static str, + path: &Path, +) { + // Don't stat the file if we are not going to record its size. + if !self_profiler_ref.enabled() { + return; + } + + if let Some(artifact_name) = path.file_name() { + let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size); + } +} diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs new file mode 100644 index 00000000000..86f92dc0239 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -0,0 +1,173 @@ +//! Codegen the MIR to the LLVM IR. +//! +//! Hopefully useful general knowledge about codegen: +//! +//! * There's no way to find out the [`Ty`] type of a [`Value`]. Doing so +//! would be "trying to get the eggs out of an omelette" (credit: +//! pcwalton). You can, instead, find out its [`llvm::Type`] by calling [`val_ty`], +//! but one [`llvm::Type`] corresponds to many [`Ty`]s; for instance, `tup(int, int, +//! int)` and `rec(x=int, y=int, z=int)` will have the same [`llvm::Type`]. +//! +//! [`Ty`]: rustc_middle::ty::Ty +//! [`val_ty`]: crate::common::val_ty + +use super::ModuleLlvm; + +use crate::attributes; +use crate::builder::Builder; +use crate::context::CodegenCx; +use crate::llvm; +use crate::value::Value; + +use rustc_codegen_ssa::base::maybe_create_entry_wrapper; +use rustc_codegen_ssa::mono_item::MonoItemExt; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::{ModuleCodegen, ModuleKind}; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_middle::dep_graph; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; +use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::DebugInfo; +use rustc_span::symbol::Symbol; +use rustc_target::spec::SanitizerSet; + +use std::time::Instant; + +pub struct ValueIter<'ll> { + cur: Option<&'ll Value>, + step: unsafe extern "C" fn(&'ll Value) -> Option<&'ll Value>, +} + +impl<'ll> Iterator for ValueIter<'ll> { + type Item = &'ll Value; + + fn next(&mut self) -> Option<&'ll Value> { + let old = self.cur; + if let Some(old) = old { + self.cur = unsafe { (self.step)(old) }; + } + old + } +} + +pub fn iter_globals(llmod: &llvm::Module) -> ValueIter<'_> { + unsafe { ValueIter { cur: llvm::LLVMGetFirstGlobal(llmod), step: llvm::LLVMGetNextGlobal } } +} + +pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen, u64) { + let start_time = Instant::now(); + + let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx); + let (module, _) = tcx.dep_graph.with_task( + dep_node, + tcx, + cgu_name, + module_codegen, + Some(dep_graph::hash_result), + ); + let time_to_codegen = start_time.elapsed(); + + // We assume that the cost to run LLVM on a CGU is proportional to + // the time we needed for codegenning it. + let cost = time_to_codegen.as_nanos() as u64; + + fn module_codegen(tcx: TyCtxt<'_>, cgu_name: Symbol) -> ModuleCodegen { + let cgu = tcx.codegen_unit(cgu_name); + let _prof_timer = + tcx.prof.generic_activity_with_arg_recorder("codegen_module", |recorder| { + recorder.record_arg(cgu_name.to_string()); + recorder.record_arg(cgu.size_estimate().to_string()); + }); + // Instantiate monomorphizations without filling out definitions yet... + let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str()); + { + let cx = CodegenCx::new(tcx, cgu, &llvm_module); + let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); + for &(mono_item, (linkage, visibility)) in &mono_items { + mono_item.predefine::>(&cx, linkage, visibility); + } + + // ... and now that we have everything pre-defined, fill out those definitions. + for &(mono_item, _) in &mono_items { + mono_item.define::>(&cx); + } + + // If this codegen unit contains the main function, also create the + // wrapper here + if let Some(entry) = maybe_create_entry_wrapper::>(&cx) { + let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty()); + attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); + } + + // Finalize code coverage by injecting the coverage map. Note, the coverage map will + // also be added to the `llvm.compiler.used` variable, created next. + if cx.sess().instrument_coverage() { + cx.coverageinfo_finalize(); + } + + // Create the llvm.used and llvm.compiler.used variables. + if !cx.used_statics().borrow().is_empty() { + cx.create_used_variable() + } + if !cx.compiler_used_statics().borrow().is_empty() { + cx.create_compiler_used_variable() + } + + // Run replace-all-uses-with for statics that need it. This must + // happen after the llvm.used variables are created. + for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { + unsafe { + let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); + llvm::LLVMReplaceAllUsesWith(old_g, bitcast); + llvm::LLVMDeleteGlobal(old_g); + } + } + + // Finalize debuginfo + if cx.sess().opts.debuginfo != DebugInfo::None { + cx.debuginfo_finalize(); + } + } + + ModuleCodegen { + name: cgu_name.to_string(), + module_llvm: llvm_module, + kind: ModuleKind::Regular, + } + } + + (module, cost) +} + +pub fn set_link_section(llval: &Value, attrs: &CodegenFnAttrs) { + let Some(sect) = attrs.link_section else { return }; + unsafe { + let buf = SmallCStr::new(sect.as_str()); + llvm::LLVMSetSection(llval, buf.as_ptr()); + } +} + +pub fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage { + match linkage { + Linkage::External => llvm::Linkage::ExternalLinkage, + Linkage::AvailableExternally => llvm::Linkage::AvailableExternallyLinkage, + Linkage::LinkOnceAny => llvm::Linkage::LinkOnceAnyLinkage, + Linkage::LinkOnceODR => llvm::Linkage::LinkOnceODRLinkage, + Linkage::WeakAny => llvm::Linkage::WeakAnyLinkage, + Linkage::WeakODR => llvm::Linkage::WeakODRLinkage, + Linkage::Appending => llvm::Linkage::AppendingLinkage, + Linkage::Internal => llvm::Linkage::InternalLinkage, + Linkage::Private => llvm::Linkage::PrivateLinkage, + Linkage::ExternalWeak => llvm::Linkage::ExternalWeakLinkage, + Linkage::Common => llvm::Linkage::CommonLinkage, + } +} + +pub fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { + match linkage { + Visibility::Default => llvm::Visibility::Default, + Visibility::Hidden => llvm::Visibility::Hidden, + Visibility::Protected => llvm::Visibility::Protected, + } +} diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs new file mode 100644 index 00000000000..d3096c73a8a --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -0,0 +1,1508 @@ +use crate::attributes; +use crate::common::Funclet; +use crate::context::CodegenCx; +use crate::llvm::{self, BasicBlock, False}; +use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope}; +use crate::llvm_util; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; +use cstr::cstr; +use libc::{c_char, c_uint}; +use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, TypeKind}; +use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; +use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::MemFlags; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::def_id::DefId; +use rustc_middle::ty::layout::{ + FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, +}; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_span::Span; +use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; +use rustc_target::spec::{HasTargetSpec, Target}; +use std::borrow::Cow; +use std::ffi::CStr; +use std::iter; +use std::ops::Deref; +use std::ptr; +use tracing::{debug, instrument}; + +// All Builders must have an llfn associated with them +#[must_use] +pub struct Builder<'a, 'll, 'tcx> { + pub llbuilder: &'ll mut llvm::Builder<'ll>, + pub cx: &'a CodegenCx<'ll, 'tcx>, +} + +impl Drop for Builder<'_, '_, '_> { + fn drop(&mut self) { + unsafe { + llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _)); + } + } +} + +// FIXME(eddyb) use a checked constructor when they become `const fn`. +const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") }; + +/// Empty string, to be used where LLVM expects an instruction name, indicating +/// that the instruction is to be left unnamed (i.e. numbered, in textual IR). +// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer. +const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr(); + +impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> { + type Value = as BackendTypes>::Value; + type Function = as BackendTypes>::Function; + type BasicBlock = as BackendTypes>::BasicBlock; + type Type = as BackendTypes>::Type; + type Funclet = as BackendTypes>::Funclet; + + type DIScope = as BackendTypes>::DIScope; + type DILocation = as BackendTypes>::DILocation; + type DIVariable = as BackendTypes>::DIVariable; +} + +impl abi::HasDataLayout for Builder<'_, '_, '_> { + fn data_layout(&self) -> &abi::TargetDataLayout { + self.cx.data_layout() + } +} + +impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> { + #[inline] + fn tcx(&self) -> TyCtxt<'tcx> { + self.cx.tcx + } +} + +impl<'tcx> ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> { + fn param_env(&self) -> ty::ParamEnv<'tcx> { + self.cx.param_env() + } +} + +impl HasTargetSpec for Builder<'_, '_, '_> { + #[inline] + fn target_spec(&self) -> &Target { + self.cx.target_spec() + } +} + +impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { + type LayoutOfResult = TyAndLayout<'tcx>; + + #[inline] + fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { + self.cx.handle_layout_err(err, span, ty) + } +} + +impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { + type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; + + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + self.cx.handle_fn_abi_err(err, span, fn_abi_request) + } +} + +impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> { + type Target = CodegenCx<'ll, 'tcx>; + + #[inline] + fn deref(&self) -> &Self::Target { + self.cx + } +} + +impl<'ll, 'tcx> HasCodegen<'tcx> for Builder<'_, 'll, 'tcx> { + type CodegenCx = CodegenCx<'ll, 'tcx>; +} + +macro_rules! builder_methods_for_value_instructions { + ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => { + $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value { + unsafe { + llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED) + } + })+ + } +} + +impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { + fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self { + let bx = Builder::with_cx(cx); + unsafe { + llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb); + } + bx + } + + fn cx(&self) -> &CodegenCx<'ll, 'tcx> { + self.cx + } + + fn llbb(&self) -> &'ll BasicBlock { + unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) } + } + + fn set_span(&mut self, _span: Span) {} + + fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock { + unsafe { + let name = SmallCStr::new(name); + llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr()) + } + } + + fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock { + Self::append_block(self.cx, self.llfn(), name) + } + + fn switch_to_block(&mut self, llbb: Self::BasicBlock) { + *self = Self::build(self.cx, llbb) + } + + fn ret_void(&mut self) { + unsafe { + llvm::LLVMBuildRetVoid(self.llbuilder); + } + } + + fn ret(&mut self, v: &'ll Value) { + unsafe { + llvm::LLVMBuildRet(self.llbuilder, v); + } + } + + fn br(&mut self, dest: &'ll BasicBlock) { + unsafe { + llvm::LLVMBuildBr(self.llbuilder, dest); + } + } + + fn cond_br( + &mut self, + cond: &'ll Value, + then_llbb: &'ll BasicBlock, + else_llbb: &'ll BasicBlock, + ) { + unsafe { + llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb); + } + } + + fn switch( + &mut self, + v: &'ll Value, + else_llbb: &'ll BasicBlock, + cases: impl ExactSizeIterator, + ) { + let switch = + unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) }; + for (on_val, dest) in cases { + let on_val = self.const_uint_big(self.val_ty(v), on_val); + unsafe { llvm::LLVMAddCase(switch, on_val, dest) } + } + } + + fn invoke( + &mut self, + llty: &'ll Type, + llfn: &'ll Value, + args: &[&'ll Value], + then: &'ll BasicBlock, + catch: &'ll BasicBlock, + funclet: Option<&Funclet<'ll>>, + ) -> &'ll Value { + debug!("invoke {:?} with args ({:?})", llfn, args); + + let args = self.check_call("invoke", llty, llfn, args); + let bundle = funclet.map(|funclet| funclet.bundle()); + let bundle = bundle.as_ref().map(|b| &*b.raw); + + unsafe { + llvm::LLVMRustBuildInvoke( + self.llbuilder, + llty, + llfn, + args.as_ptr(), + args.len() as c_uint, + then, + catch, + bundle, + UNNAMED, + ) + } + } + + fn unreachable(&mut self) { + unsafe { + llvm::LLVMBuildUnreachable(self.llbuilder); + } + } + + builder_methods_for_value_instructions! { + add(a, b) => LLVMBuildAdd, + fadd(a, b) => LLVMBuildFAdd, + sub(a, b) => LLVMBuildSub, + fsub(a, b) => LLVMBuildFSub, + mul(a, b) => LLVMBuildMul, + fmul(a, b) => LLVMBuildFMul, + udiv(a, b) => LLVMBuildUDiv, + exactudiv(a, b) => LLVMBuildExactUDiv, + sdiv(a, b) => LLVMBuildSDiv, + exactsdiv(a, b) => LLVMBuildExactSDiv, + fdiv(a, b) => LLVMBuildFDiv, + urem(a, b) => LLVMBuildURem, + srem(a, b) => LLVMBuildSRem, + frem(a, b) => LLVMBuildFRem, + shl(a, b) => LLVMBuildShl, + lshr(a, b) => LLVMBuildLShr, + ashr(a, b) => LLVMBuildAShr, + and(a, b) => LLVMBuildAnd, + or(a, b) => LLVMBuildOr, + xor(a, b) => LLVMBuildXor, + neg(x) => LLVMBuildNeg, + fneg(x) => LLVMBuildFNeg, + not(x) => LLVMBuildNot, + unchecked_sadd(x, y) => LLVMBuildNSWAdd, + unchecked_uadd(x, y) => LLVMBuildNUWAdd, + unchecked_ssub(x, y) => LLVMBuildNSWSub, + unchecked_usub(x, y) => LLVMBuildNUWSub, + unchecked_smul(x, y) => LLVMBuildNSWMul, + unchecked_umul(x, y) => LLVMBuildNUWMul, + } + + fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + + fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + + fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + + fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + + fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + + fn checked_binop( + &mut self, + oop: OverflowOp, + ty: Ty<'_>, + lhs: Self::Value, + rhs: Self::Value, + ) -> (Self::Value, Self::Value) { + use rustc_middle::ty::{Int, Uint}; + use rustc_middle::ty::{IntTy::*, UintTy::*}; + + let new_kind = match ty.kind() { + Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)), + Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)), + t @ (Uint(_) | Int(_)) => t.clone(), + _ => panic!("tried to get overflow intrinsic for op applied to non-int type"), + }; + + let name = match oop { + OverflowOp::Add => match new_kind { + Int(I8) => "llvm.sadd.with.overflow.i8", + Int(I16) => "llvm.sadd.with.overflow.i16", + Int(I32) => "llvm.sadd.with.overflow.i32", + Int(I64) => "llvm.sadd.with.overflow.i64", + Int(I128) => "llvm.sadd.with.overflow.i128", + + Uint(U8) => "llvm.uadd.with.overflow.i8", + Uint(U16) => "llvm.uadd.with.overflow.i16", + Uint(U32) => "llvm.uadd.with.overflow.i32", + Uint(U64) => "llvm.uadd.with.overflow.i64", + Uint(U128) => "llvm.uadd.with.overflow.i128", + + _ => unreachable!(), + }, + OverflowOp::Sub => match new_kind { + Int(I8) => "llvm.ssub.with.overflow.i8", + Int(I16) => "llvm.ssub.with.overflow.i16", + Int(I32) => "llvm.ssub.with.overflow.i32", + Int(I64) => "llvm.ssub.with.overflow.i64", + Int(I128) => "llvm.ssub.with.overflow.i128", + + Uint(U8) => "llvm.usub.with.overflow.i8", + Uint(U16) => "llvm.usub.with.overflow.i16", + Uint(U32) => "llvm.usub.with.overflow.i32", + Uint(U64) => "llvm.usub.with.overflow.i64", + Uint(U128) => "llvm.usub.with.overflow.i128", + + _ => unreachable!(), + }, + OverflowOp::Mul => match new_kind { + Int(I8) => "llvm.smul.with.overflow.i8", + Int(I16) => "llvm.smul.with.overflow.i16", + Int(I32) => "llvm.smul.with.overflow.i32", + Int(I64) => "llvm.smul.with.overflow.i64", + Int(I128) => "llvm.smul.with.overflow.i128", + + Uint(U8) => "llvm.umul.with.overflow.i8", + Uint(U16) => "llvm.umul.with.overflow.i16", + Uint(U32) => "llvm.umul.with.overflow.i32", + Uint(U64) => "llvm.umul.with.overflow.i64", + Uint(U128) => "llvm.umul.with.overflow.i128", + + _ => unreachable!(), + }, + }; + + let res = self.call_intrinsic(name, &[lhs, rhs]); + (self.extract_value(res, 0), self.extract_value(res, 1)) + } + + fn from_immediate(&mut self, val: Self::Value) -> Self::Value { + if self.cx().val_ty(val) == self.cx().type_i1() { + self.zext(val, self.cx().type_i8()) + } else { + val + } + } + fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value { + if scalar.is_bool() { + return self.trunc(val, self.cx().type_i1()); + } + val + } + + fn alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { + let mut bx = Builder::with_cx(self.cx); + bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) }); + bx.dynamic_alloca(ty, align) + } + + fn dynamic_alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { + unsafe { + let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED); + llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); + alloca + } + } + + fn array_alloca(&mut self, ty: &'ll Type, len: &'ll Value, align: Align) -> &'ll Value { + unsafe { + let alloca = llvm::LLVMBuildArrayAlloca(self.llbuilder, ty, len, UNNAMED); + llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); + alloca + } + } + + fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value { + unsafe { + let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); + llvm::LLVMSetAlignment(load, align.bytes() as c_uint); + load + } + } + + fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { + unsafe { + let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); + llvm::LLVMSetVolatile(load, llvm::True); + load + } + } + + fn atomic_load( + &mut self, + ty: &'ll Type, + ptr: &'ll Value, + order: rustc_codegen_ssa::common::AtomicOrdering, + size: Size, + ) -> &'ll Value { + unsafe { + let load = llvm::LLVMRustBuildAtomicLoad( + self.llbuilder, + ty, + ptr, + UNNAMED, + AtomicOrdering::from_generic(order), + ); + // LLVM requires the alignment of atomic loads to be at least the size of the type. + llvm::LLVMSetAlignment(load, size.bytes() as c_uint); + load + } + } + + #[instrument(level = "trace", skip(self))] + fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> { + assert_eq!(place.llextra.is_some(), place.layout.is_unsized()); + + if place.layout.is_zst() { + return OperandRef::new_zst(self, place.layout); + } + + #[instrument(level = "trace", skip(bx))] + fn scalar_load_metadata<'a, 'll, 'tcx>( + bx: &mut Builder<'a, 'll, 'tcx>, + load: &'ll Value, + scalar: abi::Scalar, + layout: TyAndLayout<'tcx>, + offset: Size, + ) { + if !scalar.is_always_valid(bx) { + bx.noundef_metadata(load); + } + + match scalar.primitive() { + abi::Int(..) => { + if !scalar.is_always_valid(bx) { + bx.range_metadata(load, scalar.valid_range(bx)); + } + } + abi::Pointer => { + if !scalar.valid_range(bx).contains(0) { + bx.nonnull_metadata(load); + } + + if let Some(pointee) = layout.pointee_info_at(bx, offset) { + if let Some(_) = pointee.safe { + bx.align_metadata(load, pointee.align); + } + } + } + abi::F32 | abi::F64 => {} + } + } + + let val = if let Some(llextra) = place.llextra { + OperandValue::Ref(place.llval, Some(llextra), place.align) + } else if place.layout.is_llvm_immediate() { + let mut const_llval = None; + let llty = place.layout.llvm_type(self); + unsafe { + if let Some(global) = llvm::LLVMIsAGlobalVariable(place.llval) { + if llvm::LLVMIsGlobalConstant(global) == llvm::True { + if let Some(init) = llvm::LLVMGetInitializer(global) { + if self.val_ty(init) == llty { + const_llval = Some(init); + } + } + } + } + } + let llval = const_llval.unwrap_or_else(|| { + let load = self.load(llty, place.llval, place.align); + if let abi::Abi::Scalar(scalar) = place.layout.abi { + scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO); + } + load + }); + OperandValue::Immediate(self.to_immediate(llval, place.layout)) + } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { + let b_offset = a.size(self).align_to(b.align(self).abi); + let pair_ty = place.layout.llvm_type(self); + + let mut load = |i, scalar: abi::Scalar, layout, align, offset| { + let llptr = self.struct_gep(pair_ty, place.llval, i as u64); + let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); + let load = self.load(llty, llptr, align); + scalar_load_metadata(self, load, scalar, layout, offset); + self.to_immediate_scalar(load, scalar) + }; + + OperandValue::Pair( + load(0, a, place.layout, place.align, Size::ZERO), + load(1, b, place.layout, place.align.restrict_for_offset(b_offset), b_offset), + ) + } else { + OperandValue::Ref(place.llval, None, place.align) + }; + + OperandRef { val, layout: place.layout } + } + + fn write_operand_repeatedly( + mut self, + cg_elem: OperandRef<'tcx, &'ll Value>, + count: u64, + dest: PlaceRef<'tcx, &'ll Value>, + ) -> Self { + let zero = self.const_usize(0); + let count = self.const_usize(count); + let start = dest.project_index(&mut self, zero).llval; + let end = dest.project_index(&mut self, count).llval; + + let header_bb = self.append_sibling_block("repeat_loop_header"); + let body_bb = self.append_sibling_block("repeat_loop_body"); + let next_bb = self.append_sibling_block("repeat_loop_next"); + + self.br(header_bb); + + let mut header_bx = Self::build(self.cx, header_bb); + let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]); + + let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end); + header_bx.cond_br(keep_going, body_bb, next_bb); + + let mut body_bx = Self::build(self.cx, body_bb); + let align = dest.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size); + cg_elem + .val + .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align)); + + let next = body_bx.inbounds_gep( + self.backend_type(cg_elem.layout), + current, + &[self.const_usize(1)], + ); + body_bx.br(header_bb); + header_bx.add_incoming_to_phi(current, next, body_bb); + + Self::build(self.cx, next_bb) + } + + fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) { + if self.sess().target.arch == "amdgpu" { + // amdgpu/LLVM does something weird and thinks an i64 value is + // split into a v2i32, halving the bitwidth LLVM expects, + // tripping an assertion. So, for now, just disable this + // optimization. + return; + } + + unsafe { + let llty = self.cx.val_ty(load); + let v = [ + self.cx.const_uint_big(llty, range.start), + self.cx.const_uint_big(llty, range.end.wrapping_add(1)), + ]; + + llvm::LLVMSetMetadata( + load, + llvm::MD_range as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint), + ); + } + } + + fn nonnull_metadata(&mut self, load: &'ll Value) { + unsafe { + llvm::LLVMSetMetadata( + load, + llvm::MD_nonnull as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), + ); + } + } + + fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { + self.store_with_flags(val, ptr, align, MemFlags::empty()) + } + + fn store_with_flags( + &mut self, + val: &'ll Value, + ptr: &'ll Value, + align: Align, + flags: MemFlags, + ) -> &'ll Value { + debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags); + let ptr = self.check_store(val, ptr); + unsafe { + let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr); + let align = + if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint }; + llvm::LLVMSetAlignment(store, align); + if flags.contains(MemFlags::VOLATILE) { + llvm::LLVMSetVolatile(store, llvm::True); + } + if flags.contains(MemFlags::NONTEMPORAL) { + // According to LLVM [1] building a nontemporal store must + // *always* point to a metadata value of the integer 1. + // + // [1]: https://llvm.org/docs/LangRef.html#store-instruction + let one = self.cx.const_i32(1); + let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1); + llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node); + } + store + } + } + + fn atomic_store( + &mut self, + val: &'ll Value, + ptr: &'ll Value, + order: rustc_codegen_ssa::common::AtomicOrdering, + size: Size, + ) { + debug!("Store {:?} -> {:?}", val, ptr); + let ptr = self.check_store(val, ptr); + unsafe { + let store = llvm::LLVMRustBuildAtomicStore( + self.llbuilder, + val, + ptr, + AtomicOrdering::from_generic(order), + ); + // LLVM requires the alignment of atomic stores to be at least the size of the type. + llvm::LLVMSetAlignment(store, size.bytes() as c_uint); + } + } + + fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value { + unsafe { + llvm::LLVMBuildGEP2( + self.llbuilder, + ty, + ptr, + indices.as_ptr(), + indices.len() as c_uint, + UNNAMED, + ) + } + } + + fn inbounds_gep( + &mut self, + ty: &'ll Type, + ptr: &'ll Value, + indices: &[&'ll Value], + ) -> &'ll Value { + unsafe { + llvm::LLVMBuildInBoundsGEP2( + self.llbuilder, + ty, + ptr, + indices.as_ptr(), + indices.len() as c_uint, + UNNAMED, + ) + } + } + + fn struct_gep(&mut self, ty: &'ll Type, ptr: &'ll Value, idx: u64) -> &'ll Value { + assert_eq!(idx as c_uint as u64, idx); + unsafe { llvm::LLVMBuildStructGEP2(self.llbuilder, ty, ptr, idx as c_uint, UNNAMED) } + } + + /* Casts */ + fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + self.fptoint_sat(false, val, dest_ty) + } + + fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + self.fptoint_sat(true, val, dest_ty) + } + + fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + // On WebAssembly the `fptoui` and `fptosi` instructions currently have + // poor codegen. The reason for this is that the corresponding wasm + // instructions, `i32.trunc_f32_s` for example, will trap when the float + // is out-of-bounds, infinity, or nan. This means that LLVM + // automatically inserts control flow around `fptoui` and `fptosi` + // because the LLVM instruction `fptoui` is defined as producing a + // poison value, not having UB on out-of-bounds values. + // + // This method, however, is only used with non-saturating casts that + // have UB on out-of-bounds values. This means that it's ok if we use + // the raw wasm instruction since out-of-bounds values can do whatever + // we like. To ensure that LLVM picks the right instruction we choose + // the raw wasm intrinsic functions which avoid LLVM inserting all the + // other control flow automatically. + if self.sess().target.is_like_wasm { + let src_ty = self.cx.val_ty(val); + if self.cx.type_kind(src_ty) != TypeKind::Vector { + let float_width = self.cx.float_width(src_ty); + let int_width = self.cx.int_width(dest_ty); + let name = match (int_width, float_width) { + (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"), + (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"), + (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"), + (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"), + _ => None, + }; + if let Some(name) = name { + return self.call_intrinsic(name, &[val]); + } + } + } + unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + // see `fptoui` above for why wasm is different here + if self.sess().target.is_like_wasm { + let src_ty = self.cx.val_ty(val); + if self.cx.type_kind(src_ty) != TypeKind::Vector { + let float_width = self.cx.float_width(src_ty); + let int_width = self.cx.int_width(dest_ty); + let name = match (int_width, float_width) { + (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"), + (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"), + (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"), + (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"), + _ => None, + }; + if let Some(name) = name { + return self.call_intrinsic(name, &[val]); + } + } + } + unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { + unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) } + } + + fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) } + } + + /* Comparisons */ + fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + let op = llvm::IntPredicate::from_generic(op); + unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) } + } + + fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + let op = llvm::RealPredicate::from_generic(op); + unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) } + } + + /* Miscellaneous instructions */ + fn memcpy( + &mut self, + dst: &'ll Value, + dst_align: Align, + src: &'ll Value, + src_align: Align, + size: &'ll Value, + flags: MemFlags, + ) { + assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); + let size = self.intcast(size, self.type_isize(), false); + let is_volatile = flags.contains(MemFlags::VOLATILE); + let dst = self.pointercast(dst, self.type_i8p()); + let src = self.pointercast(src, self.type_i8p()); + unsafe { + llvm::LLVMRustBuildMemCpy( + self.llbuilder, + dst, + dst_align.bytes() as c_uint, + src, + src_align.bytes() as c_uint, + size, + is_volatile, + ); + } + } + + fn memmove( + &mut self, + dst: &'ll Value, + dst_align: Align, + src: &'ll Value, + src_align: Align, + size: &'ll Value, + flags: MemFlags, + ) { + assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported"); + let size = self.intcast(size, self.type_isize(), false); + let is_volatile = flags.contains(MemFlags::VOLATILE); + let dst = self.pointercast(dst, self.type_i8p()); + let src = self.pointercast(src, self.type_i8p()); + unsafe { + llvm::LLVMRustBuildMemMove( + self.llbuilder, + dst, + dst_align.bytes() as c_uint, + src, + src_align.bytes() as c_uint, + size, + is_volatile, + ); + } + } + + fn memset( + &mut self, + ptr: &'ll Value, + fill_byte: &'ll Value, + size: &'ll Value, + align: Align, + flags: MemFlags, + ) { + let is_volatile = flags.contains(MemFlags::VOLATILE); + let ptr = self.pointercast(ptr, self.type_i8p()); + unsafe { + llvm::LLVMRustBuildMemSet( + self.llbuilder, + ptr, + align.bytes() as c_uint, + fill_byte, + size, + is_volatile, + ); + } + } + + fn select( + &mut self, + cond: &'ll Value, + then_val: &'ll Value, + else_val: &'ll Value, + ) -> &'ll Value { + unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) } + } + + fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } + } + + fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) } + } + + fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value { + unsafe { + let elt_ty = self.cx.val_ty(elt); + let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64)); + let vec = self.insert_element(undef, elt, self.cx.const_i32(0)); + let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64); + self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty)) + } + } + + fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value { + assert_eq!(idx as c_uint as u64, idx); + unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) } + } + + fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value { + assert_eq!(idx as c_uint as u64, idx); + unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) } + } + + fn set_personality_fn(&mut self, personality: &'ll Value) { + unsafe { + llvm::LLVMSetPersonalityFn(self.llfn(), personality); + } + } + + fn cleanup_landing_pad(&mut self, ty: &'ll Type, pers_fn: &'ll Value) -> &'ll Value { + let landing_pad = self.landing_pad(ty, pers_fn, 1 /* FIXME should this be 0? */); + unsafe { + llvm::LLVMSetCleanup(landing_pad, llvm::True); + } + landing_pad + } + + fn resume(&mut self, exn: &'ll Value) { + unsafe { + llvm::LLVMBuildResume(self.llbuilder, exn); + } + } + + fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { + let name = cstr!("cleanuppad"); + let ret = unsafe { + llvm::LLVMRustBuildCleanupPad( + self.llbuilder, + parent, + args.len() as c_uint, + args.as_ptr(), + name.as_ptr(), + ) + }; + Funclet::new(ret.expect("LLVM does not have support for cleanuppad")) + } + + fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) { + unsafe { + llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) + .expect("LLVM does not have support for cleanupret"); + } + } + + fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { + let name = cstr!("catchpad"); + let ret = unsafe { + llvm::LLVMRustBuildCatchPad( + self.llbuilder, + parent, + args.len() as c_uint, + args.as_ptr(), + name.as_ptr(), + ) + }; + Funclet::new(ret.expect("LLVM does not have support for catchpad")) + } + + fn catch_switch( + &mut self, + parent: Option<&'ll Value>, + unwind: Option<&'ll BasicBlock>, + handlers: &[&'ll BasicBlock], + ) -> &'ll Value { + let name = cstr!("catchswitch"); + let ret = unsafe { + llvm::LLVMRustBuildCatchSwitch( + self.llbuilder, + parent, + unwind, + handlers.len() as c_uint, + name.as_ptr(), + ) + }; + let ret = ret.expect("LLVM does not have support for catchswitch"); + for handler in handlers { + unsafe { + llvm::LLVMRustAddHandler(ret, handler); + } + } + ret + } + + // Atomic Operations + fn atomic_cmpxchg( + &mut self, + dst: &'ll Value, + cmp: &'ll Value, + src: &'ll Value, + mut order: rustc_codegen_ssa::common::AtomicOrdering, + failure_order: rustc_codegen_ssa::common::AtomicOrdering, + weak: bool, + ) -> &'ll Value { + let weak = if weak { llvm::True } else { llvm::False }; + if llvm_util::get_version() < (13, 0, 0) { + use rustc_codegen_ssa::common::AtomicOrdering::*; + // Older llvm has the pre-C++17 restriction on + // success and failure memory ordering, + // requiring the former to be at least as strong as the latter. + // So, for llvm 12, we upgrade the success ordering to a stronger + // one if necessary. + match (order, failure_order) { + (Relaxed, Acquire) => order = Acquire, + (Release, Acquire) => order = AcquireRelease, + (_, SequentiallyConsistent) => order = SequentiallyConsistent, + _ => {} + } + } + unsafe { + llvm::LLVMRustBuildAtomicCmpXchg( + self.llbuilder, + dst, + cmp, + src, + AtomicOrdering::from_generic(order), + AtomicOrdering::from_generic(failure_order), + weak, + ) + } + } + fn atomic_rmw( + &mut self, + op: rustc_codegen_ssa::common::AtomicRmwBinOp, + dst: &'ll Value, + src: &'ll Value, + order: rustc_codegen_ssa::common::AtomicOrdering, + ) -> &'ll Value { + unsafe { + llvm::LLVMBuildAtomicRMW( + self.llbuilder, + AtomicRmwBinOp::from_generic(op), + dst, + src, + AtomicOrdering::from_generic(order), + False, + ) + } + } + + fn atomic_fence( + &mut self, + order: rustc_codegen_ssa::common::AtomicOrdering, + scope: rustc_codegen_ssa::common::SynchronizationScope, + ) { + unsafe { + llvm::LLVMRustBuildAtomicFence( + self.llbuilder, + AtomicOrdering::from_generic(order), + SynchronizationScope::from_generic(scope), + ); + } + } + + fn set_invariant_load(&mut self, load: &'ll Value) { + unsafe { + llvm::LLVMSetMetadata( + load, + llvm::MD_invariant_load as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), + ); + } + } + + fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) { + self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size); + } + + fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) { + self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size); + } + + fn instrprof_increment( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + num_counters: &'ll Value, + index: &'ll Value, + ) { + debug!( + "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})", + fn_name, hash, num_counters, index + ); + + let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; + let llty = self.cx.type_func( + &[self.cx.type_i8p(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], + self.cx.type_void(), + ); + let args = &[fn_name, hash, num_counters, index]; + let args = self.check_call("call", llty, llfn, args); + + unsafe { + let _ = llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + None, + ); + } + } + + fn call( + &mut self, + llty: &'ll Type, + llfn: &'ll Value, + args: &[&'ll Value], + funclet: Option<&Funclet<'ll>>, + ) -> &'ll Value { + debug!("call {:?} with args ({:?})", llfn, args); + + let args = self.check_call("call", llty, llfn, args); + let bundle = funclet.map(|funclet| funclet.bundle()); + let bundle = bundle.as_ref().map(|b| &*b.raw); + + unsafe { + llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + bundle, + ) + } + } + + fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn do_not_inline(&mut self, llret: &'ll Value) { + let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); + } +} + +impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> { + fn get_static(&mut self, def_id: DefId) -> &'ll Value { + // Forward to the `get_static` method of `CodegenCx` + self.cx().get_static(def_id) + } +} + +impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { + fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self { + // Create a fresh builder from the crate context. + let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) }; + Builder { llbuilder, cx } + } + + pub fn llfn(&self) -> &'ll Value { + unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) } + } + + fn position_at_start(&mut self, llbb: &'ll BasicBlock) { + unsafe { + llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb); + } + } + + fn align_metadata(&mut self, load: &'ll Value, align: Align) { + unsafe { + let v = [self.cx.const_u64(align.bytes())]; + + llvm::LLVMSetMetadata( + load, + llvm::MD_align as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint), + ); + } + } + + fn noundef_metadata(&mut self, load: &'ll Value) { + unsafe { + llvm::LLVMSetMetadata( + load, + llvm::MD_noundef as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), + ); + } + } + + pub fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) } + } + + pub fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) } + } + + pub fn insert_element( + &mut self, + vec: &'ll Value, + elt: &'ll Value, + idx: &'ll Value, + ) -> &'ll Value { + unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) } + } + + pub fn shuffle_vector( + &mut self, + v1: &'ll Value, + v2: &'ll Value, + mask: &'ll Value, + ) -> &'ll Value { + unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) } + } + + pub fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) } + } + pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } + } + pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + pub fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) } + } + pub fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) } + } + pub fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) } + } + pub fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) } + } + pub fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) } + } + pub fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { + llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false) + } + } + pub fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { + llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) + } + } + pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { + let instr = + llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { + let instr = + llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } + } + pub fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) } + } + + pub fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) { + unsafe { + llvm::LLVMAddClause(landing_pad, clause); + } + } + + pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value { + let ret = + unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; + ret.expect("LLVM does not have support for catchret") + } + + fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value { + let dest_ptr_ty = self.cx.val_ty(ptr); + let stored_ty = self.cx.val_ty(val); + let stored_ptr_ty = self.cx.type_ptr_to(stored_ty); + + assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer); + + if dest_ptr_ty == stored_ptr_ty { + ptr + } else { + debug!( + "type mismatch in store. \ + Expected {:?}, got {:?}; inserting bitcast", + dest_ptr_ty, stored_ptr_ty + ); + self.bitcast(ptr, stored_ptr_ty) + } + } + + fn check_call<'b>( + &mut self, + typ: &str, + fn_ty: &'ll Type, + llfn: &'ll Value, + args: &'b [&'ll Value], + ) -> Cow<'b, [&'ll Value]> { + assert!( + self.cx.type_kind(fn_ty) == TypeKind::Function, + "builder::{} not passed a function, but {:?}", + typ, + fn_ty + ); + + let param_tys = self.cx.func_params_types(fn_ty); + + let all_args_match = iter::zip(¶m_tys, args.iter().map(|&v| self.val_ty(v))) + .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty); + + if all_args_match { + return Cow::Borrowed(args); + } + + let casted_args: Vec<_> = iter::zip(param_tys, args) + .enumerate() + .map(|(i, (expected_ty, &actual_val))| { + let actual_ty = self.val_ty(actual_val); + if expected_ty != actual_ty { + debug!( + "type mismatch in function call of {:?}. \ + Expected {:?} for param {}, got {:?}; injecting bitcast", + llfn, expected_ty, i, actual_ty + ); + self.bitcast(actual_val, expected_ty) + } else { + actual_val + } + }) + .collect(); + + Cow::Owned(casted_args) + } + + pub fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } + } + + pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { + let (ty, f) = self.cx.get_intrinsic(intrinsic); + self.call(ty, f, args, None) + } + + fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { + let size = size.bytes(); + if size == 0 { + return; + } + + if !self.cx().sess().emit_lifetime_markers() { + return; + } + + let ptr = self.pointercast(ptr, self.cx.type_i8p()); + self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]); + } + + pub(crate) fn phi( + &mut self, + ty: &'ll Type, + vals: &[&'ll Value], + bbs: &[&'ll BasicBlock], + ) -> &'ll Value { + assert_eq!(vals.len(), bbs.len()); + let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) }; + unsafe { + llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint); + phi + } + } + + fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) { + unsafe { + llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint); + } + } + + fn fptoint_sat_broken_in_llvm(&self) -> bool { + match self.tcx.sess.target.arch.as_ref() { + // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083 + "riscv64" => llvm_util::get_version() < (13, 0, 0), + _ => false, + } + } + + fn fptoint_sat( + &mut self, + signed: bool, + val: &'ll Value, + dest_ty: &'ll Type, + ) -> Option<&'ll Value> { + if !self.fptoint_sat_broken_in_llvm() { + let src_ty = self.cx.val_ty(val); + let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector + { + assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); + ( + self.cx.element_type(src_ty), + self.cx.element_type(dest_ty), + Some(self.cx.vector_length(src_ty)), + ) + } else { + (src_ty, dest_ty, None) + }; + let float_width = self.cx.float_width(float_ty); + let int_width = self.cx.int_width(int_ty); + + let instr = if signed { "fptosi" } else { "fptoui" }; + let name = if let Some(vector_length) = vector_length { + format!( + "llvm.{}.sat.v{}i{}.v{}f{}", + instr, vector_length, int_width, vector_length, float_width + ) + } else { + format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) + }; + let f = + self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); + Some(self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None)) + } else { + None + } + } + + pub(crate) fn landing_pad( + &mut self, + ty: &'ll Type, + pers_fn: &'ll Value, + num_clauses: usize, + ) -> &'ll Value { + // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while, + // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The + // personality lives on the parent function anyway. + self.set_personality_fn(pers_fn); + unsafe { + llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs new file mode 100644 index 00000000000..72155d874a2 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -0,0 +1,194 @@ +//! Handles codegen of callees as well as other call-related +//! things. Callees are a superset of normal rust values and sometimes +//! have different representations. In particular, top-level fn items +//! and methods are represented as just a fn ptr and not a full +//! closure. + +use crate::abi::FnAbiLlvmExt; +use crate::attributes; +use crate::context::CodegenCx; +use crate::llvm; +use crate::value::Value; +use rustc_codegen_ssa::traits::*; +use tracing::debug; + +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; +use rustc_middle::ty::{self, Instance, TypeVisitable}; + +/// Codegens a reference to a fn/method item, monomorphizing and +/// inlining as it goes. +/// +/// # Parameters +/// +/// - `cx`: the crate context +/// - `instance`: the instance to be instantiated +pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value { + let tcx = cx.tcx(); + + debug!("get_fn(instance={:?})", instance); + + assert!(!instance.substs.needs_infer()); + assert!(!instance.substs.has_escaping_bound_vars()); + + if let Some(&llfn) = cx.instances.borrow().get(&instance) { + return llfn; + } + + let sym = tcx.symbol_name(instance).name; + debug!( + "get_fn({:?}: {:?}) => {}", + instance, + instance.ty(cx.tcx(), ty::ParamEnv::reveal_all()), + sym + ); + + let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); + + let llfn = if let Some(llfn) = cx.get_declared_value(sym) { + // Create a fn pointer with the new signature. + let llptrty = fn_abi.ptr_to_llvm_type(cx); + + // This is subtle and surprising, but sometimes we have to bitcast + // the resulting fn pointer. The reason has to do with external + // functions. If you have two crates that both bind the same C + // library, they may not use precisely the same types: for + // example, they will probably each declare their own structs, + // which are distinct types from LLVM's point of view (nominal + // types). + // + // Now, if those two crates are linked into an application, and + // they contain inlined code, you can wind up with a situation + // where both of those functions wind up being loaded into this + // application simultaneously. In that case, the same function + // (from LLVM's point of view) requires two types. But of course + // LLVM won't allow one function to have two types. + // + // What we currently do, therefore, is declare the function with + // one of the two types (whichever happens to come first) and then + // bitcast as needed when the function is referenced to make sure + // it has the type we expect. + // + // This can occur on either a crate-local or crate-external + // reference. It also occurs when testing libcore and in some + // other weird situations. Annoying. + if cx.val_ty(llfn) != llptrty { + debug!("get_fn: casting {:?} to {:?}", llfn, llptrty); + cx.const_ptrcast(llfn, llptrty) + } else { + debug!("get_fn: not casting pointer!"); + llfn + } + } else { + let llfn = cx.declare_fn(sym, fn_abi); + debug!("get_fn: not casting pointer!"); + + attributes::from_fn_attrs(cx, llfn, instance); + + let instance_def_id = instance.def_id(); + + // Apply an appropriate linkage/visibility value to our item that we + // just declared. + // + // This is sort of subtle. Inside our codegen unit we started off + // compilation by predefining all our own `MonoItem` instances. That + // is, everything we're codegenning ourselves is already defined. That + // means that anything we're actually codegenning in this codegen unit + // will have hit the above branch in `get_declared_value`. As a result, + // we're guaranteed here that we're declaring a symbol that won't get + // defined, or in other words we're referencing a value from another + // codegen unit or even another crate. + // + // So because this is a foreign value we blanket apply an external + // linkage directive because it's coming from a different object file. + // The visibility here is where it gets tricky. This symbol could be + // referencing some foreign crate or foreign library (an `extern` + // block) in which case we want to leave the default visibility. We may + // also, though, have multiple codegen units. It could be a + // monomorphization, in which case its expected visibility depends on + // whether we are sharing generics or not. The important thing here is + // that the visibility we apply to the declaration is the same one that + // has been applied to the definition (wherever that definition may be). + unsafe { + llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::ExternalLinkage); + + let is_generic = instance.substs.non_erasable_generics().next().is_some(); + + if is_generic { + // This is a monomorphization. Its expected visibility depends + // on whether we are in share-generics mode. + + if cx.tcx.sess.opts.share_generics() { + // We are in share_generics mode. + + if let Some(instance_def_id) = instance_def_id.as_local() { + // This is a definition from the current crate. If the + // definition is unreachable for downstream crates or + // the current crate does not re-export generics, the + // definition of the instance will have been declared + // as `hidden`. + if cx.tcx.is_unreachable_local_definition(instance_def_id) + || !cx.tcx.local_crate_exports_generics() + { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } else { + // This is a monomorphization of a generic function + // defined in an upstream crate. + if instance.upstream_monomorphization(tcx).is_some() { + // This is instantiated in another crate. It cannot + // be `hidden`. + } else { + // This is a local instantiation of an upstream definition. + // If the current crate does not re-export it + // (because it is a C library or an executable), it + // will have been declared `hidden`. + if !cx.tcx.local_crate_exports_generics() { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } + } + } else { + // When not sharing generics, all instances are in the same + // crate and have hidden visibility + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } else { + // This is a non-generic function + if cx.tcx.is_codegened_item(instance_def_id) { + // This is a function that is instantiated in the local crate + + if instance_def_id.is_local() { + // This is function that is defined in the local crate. + // If it is not reachable, it is hidden. + if !cx.tcx.is_reachable_non_generic(instance_def_id) { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } else { + // This is a function from an upstream crate that has + // been instantiated here. These are always hidden. + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } + } + + // MinGW: For backward compatibility we rely on the linker to decide whether it + // should use dllimport for functions. + if cx.use_dll_storage_attrs + && tcx.is_dllimport_foreign_item(instance_def_id) + && !matches!(tcx.sess.target.env.as_ref(), "gnu" | "uclibc") + { + llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); + } + + if cx.should_assume_dso_local(llfn, true) { + llvm::LLVMRustSetDSOLocal(llfn, true); + } + } + + llfn + }; + + cx.instances.borrow_mut().insert(instance, llfn); + + llfn +} diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs new file mode 100644 index 00000000000..fb4da9a5f33 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -0,0 +1,359 @@ +//! Code that is useful in various codegen modules. + +use crate::consts::{self, const_alloc_to_llvm}; +pub use crate::context::CodegenCx; +use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, OperandBundleDef, True}; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; + +use rustc_ast::Mutability; +use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_codegen_ssa::traits::*; +use rustc_middle::bug; +use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; + +use libc::{c_char, c_uint}; +use tracing::debug; + +/* +* A note on nomenclature of linking: "extern", "foreign", and "upcall". +* +* An "extern" is an LLVM symbol we wind up emitting an undefined external +* reference to. This means "we don't have the thing in this compilation unit, +* please make sure you link it in at runtime". This could be a reference to +* C code found in a C library, or rust code found in a rust crate. +* +* Most "externs" are implicitly declared (automatically) as a result of a +* user declaring an extern _module_ dependency; this causes the rust driver +* to locate an extern crate, scan its compilation metadata, and emit extern +* declarations for any symbols used by the declaring crate. +* +* A "foreign" is an extern that references C (or other non-rust ABI) code. +* There is no metadata to scan for extern references so in these cases either +* a header-digester like bindgen, or manual function prototypes, have to +* serve as declarators. So these are usually given explicitly as prototype +* declarations, in rust code, with ABI attributes on them noting which ABI to +* link via. +* +* An "upcall" is a foreign call generated by the compiler (not corresponding +* to any user-written call in the code) into the runtime library, to perform +* some helper task such as bringing a task to life, allocating memory, etc. +* +*/ + +/// A structure representing an active landing pad for the duration of a basic +/// block. +/// +/// Each `Block` may contain an instance of this, indicating whether the block +/// is part of a landing pad or not. This is used to make decision about whether +/// to emit `invoke` instructions (e.g., in a landing pad we don't continue to +/// use `invoke`) and also about various function call metadata. +/// +/// For GNU exceptions (`landingpad` + `resume` instructions) this structure is +/// just a bunch of `None` instances (not too interesting), but for MSVC +/// exceptions (`cleanuppad` + `cleanupret` instructions) this contains data. +/// When inside of a landing pad, each function call in LLVM IR needs to be +/// annotated with which landing pad it's a part of. This is accomplished via +/// the `OperandBundleDef` value created for MSVC landing pads. +pub struct Funclet<'ll> { + cleanuppad: &'ll Value, + operand: OperandBundleDef<'ll>, +} + +impl<'ll> Funclet<'ll> { + pub fn new(cleanuppad: &'ll Value) -> Self { + Funclet { cleanuppad, operand: OperandBundleDef::new("funclet", &[cleanuppad]) } + } + + pub fn cleanuppad(&self) -> &'ll Value { + self.cleanuppad + } + + pub fn bundle(&self) -> &OperandBundleDef<'ll> { + &self.operand + } +} + +impl<'ll> BackendTypes for CodegenCx<'ll, '_> { + type Value = &'ll Value; + // FIXME(eddyb) replace this with a `Function` "subclass" of `Value`. + type Function = &'ll Value; + + type BasicBlock = &'ll BasicBlock; + type Type = &'ll Type; + type Funclet = Funclet<'ll>; + + type DIScope = &'ll llvm::debuginfo::DIScope; + type DILocation = &'ll llvm::debuginfo::DILocation; + type DIVariable = &'ll llvm::debuginfo::DIVariable; +} + +impl<'ll> CodegenCx<'ll, '_> { + pub fn const_array(&self, ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { + unsafe { llvm::LLVMConstArray(ty, elts.as_ptr(), elts.len() as c_uint) } + } + + pub fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { + unsafe { llvm::LLVMConstVector(elts.as_ptr(), elts.len() as c_uint) } + } + + pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { + bytes_in_context(self.llcx, bytes) + } + + pub fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { + unsafe { + assert_eq!(idx as c_uint as u64, idx); + let r = llvm::LLVMGetAggregateElement(v, idx as c_uint).unwrap(); + + debug!("const_get_elt(v={:?}, idx={}, r={:?})", v, idx, r); + + r + } + } +} + +impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn const_null(&self, t: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMConstNull(t) } + } + + fn const_undef(&self, t: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMGetUndef(t) } + } + + fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value { + unsafe { llvm::LLVMConstInt(t, i as u64, True) } + } + + fn const_uint(&self, t: &'ll Type, i: u64) -> &'ll Value { + unsafe { llvm::LLVMConstInt(t, i, False) } + } + + fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value { + unsafe { + let words = [u as u64, (u >> 64) as u64]; + llvm::LLVMConstIntOfArbitraryPrecision(t, 2, words.as_ptr()) + } + } + + fn const_bool(&self, val: bool) -> &'ll Value { + self.const_uint(self.type_i1(), val as u64) + } + + fn const_i16(&self, i: i16) -> &'ll Value { + self.const_int(self.type_i16(), i as i64) + } + + fn const_i32(&self, i: i32) -> &'ll Value { + self.const_int(self.type_i32(), i as i64) + } + + fn const_u32(&self, i: u32) -> &'ll Value { + self.const_uint(self.type_i32(), i as u64) + } + + fn const_u64(&self, i: u64) -> &'ll Value { + self.const_uint(self.type_i64(), i) + } + + fn const_usize(&self, i: u64) -> &'ll Value { + let bit_size = self.data_layout().pointer_size.bits(); + if bit_size < 64 { + // make sure it doesn't overflow + assert!(i < (1 << bit_size)); + } + + self.const_uint(self.isize_ty, i) + } + + fn const_u8(&self, i: u8) -> &'ll Value { + self.const_uint(self.type_i8(), i as u64) + } + + fn const_real(&self, t: &'ll Type, val: f64) -> &'ll Value { + unsafe { llvm::LLVMConstReal(t, val) } + } + + fn const_str(&self, s: &str) -> (&'ll Value, &'ll Value) { + let str_global = *self + .const_str_cache + .borrow_mut() + .raw_entry_mut() + .from_key(s) + .or_insert_with(|| { + let sc = self.const_bytes(s.as_bytes()); + let sym = self.generate_local_symbol_name("str"); + let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + unsafe { + llvm::LLVMSetInitializer(g, sc); + llvm::LLVMSetGlobalConstant(g, True); + llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); + } + (s.to_owned(), g) + }) + .1; + let len = s.len(); + let cs = consts::ptrcast( + str_global, + self.type_ptr_to(self.layout_of(self.tcx.types.str_).llvm_type(self)), + ); + (cs, self.const_usize(len as u64)) + } + + fn const_struct(&self, elts: &[&'ll Value], packed: bool) -> &'ll Value { + struct_in_context(self.llcx, elts, packed) + } + + fn const_to_opt_uint(&self, v: &'ll Value) -> Option { + try_as_const_integral(v).map(|v| unsafe { llvm::LLVMConstIntGetZExtValue(v) }) + } + + fn const_to_opt_u128(&self, v: &'ll Value, sign_ext: bool) -> Option { + try_as_const_integral(v).and_then(|v| unsafe { + let (mut lo, mut hi) = (0u64, 0u64); + let success = llvm::LLVMRustConstInt128Get(v, sign_ext, &mut hi, &mut lo); + success.then_some(hi_lo_to_u128(lo, hi)) + }) + } + + fn zst_to_backend(&self, _llty: &'ll Type) -> &'ll Value { + self.const_undef(self.type_ix(0)) + } + + fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { + let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() }; + match cv { + Scalar::Int(int) => { + let data = int.assert_bits(layout.size(self)); + let llval = self.const_uint_big(self.type_ix(bitsize), data); + if layout.primitive() == Pointer { + unsafe { llvm::LLVMConstIntToPtr(llval, llty) } + } else { + self.const_bitcast(llval, llty) + } + } + Scalar::Ptr(ptr, _size) => { + let (alloc_id, offset) = ptr.into_parts(); + let (base_addr, base_addr_space) = match self.tcx.global_alloc(alloc_id) { + GlobalAlloc::Memory(alloc) => { + let init = const_alloc_to_llvm(self, alloc); + let alloc = alloc.inner(); + let value = match alloc.mutability { + Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), + _ => self.static_addr_of(init, alloc.align, None), + }; + if !self.sess().fewer_names() { + llvm::set_value_name(value, format!("{:?}", alloc_id).as_bytes()); + } + (value, AddressSpace::DATA) + } + GlobalAlloc::Function(fn_instance) => ( + self.get_fn_addr(fn_instance.polymorphize(self.tcx)), + self.data_layout().instruction_address_space, + ), + GlobalAlloc::VTable(ty, trait_ref) => { + let alloc = self + .tcx + .global_alloc(self.tcx.vtable_allocation((ty, trait_ref))) + .unwrap_memory(); + let init = const_alloc_to_llvm(self, alloc); + let value = self.static_addr_of(init, alloc.inner().align, None); + (value, AddressSpace::DATA) + } + GlobalAlloc::Static(def_id) => { + assert!(self.tcx.is_static(def_id)); + assert!(!self.tcx.is_thread_local_static(def_id)); + (self.get_static(def_id), AddressSpace::DATA) + } + }; + let llval = unsafe { + llvm::LLVMRustConstInBoundsGEP2( + self.type_i8(), + self.const_bitcast(base_addr, self.type_i8p_ext(base_addr_space)), + &self.const_usize(offset.bytes()), + 1, + ) + }; + if layout.primitive() != Pointer { + unsafe { llvm::LLVMConstPtrToInt(llval, llty) } + } else { + self.const_bitcast(llval, llty) + } + } + } + } + + fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value { + const_alloc_to_llvm(self, alloc) + } + + fn from_const_alloc( + &self, + layout: TyAndLayout<'tcx>, + alloc: ConstAllocation<'tcx>, + offset: Size, + ) -> PlaceRef<'tcx, &'ll Value> { + let alloc_align = alloc.inner().align; + assert_eq!(alloc_align, layout.align.abi); + let llty = self.type_ptr_to(layout.llvm_type(self)); + let llval = if layout.size == Size::ZERO { + let llval = self.const_usize(alloc_align.bytes()); + unsafe { llvm::LLVMConstIntToPtr(llval, llty) } + } else { + let init = const_alloc_to_llvm(self, alloc); + let base_addr = self.static_addr_of(init, alloc_align, None); + + let llval = unsafe { + llvm::LLVMRustConstInBoundsGEP2( + self.type_i8(), + self.const_bitcast(base_addr, self.type_i8p()), + &self.const_usize(offset.bytes()), + 1, + ) + }; + self.const_bitcast(llval, llty) + }; + PlaceRef::new_sized(llval, layout) + } + + fn const_ptrcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + consts::ptrcast(val, ty) + } +} + +/// Get the [LLVM type][Type] of a [`Value`]. +pub fn val_ty(v: &Value) -> &Type { + unsafe { llvm::LLVMTypeOf(v) } +} + +pub fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { + unsafe { + let ptr = bytes.as_ptr() as *const c_char; + llvm::LLVMConstStringInContext(llcx, ptr, bytes.len() as c_uint, True) + } +} + +pub fn struct_in_context<'ll>( + llcx: &'ll llvm::Context, + elts: &[&'ll Value], + packed: bool, +) -> &'ll Value { + unsafe { + llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), elts.len() as c_uint, packed as Bool) + } +} + +#[inline] +fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 { + ((hi as u128) << 64) | (lo as u128) +} + +fn try_as_const_integral(v: &Value) -> Option<&ConstantInt> { + unsafe { llvm::LLVMIsAConstantInt(v) } +} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs new file mode 100644 index 00000000000..18467e37082 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -0,0 +1,577 @@ +use crate::base; +use crate::common::CodegenCx; +use crate::debuginfo; +use crate::llvm::{self, True}; +use crate::llvm_util; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; +use cstr::cstr; +use libc::c_uint; +use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; +use rustc_middle::mir::interpret::{ + read_target_uint, Allocation, ConstAllocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer, + Scalar as InterpScalar, +}; +use rustc_middle::mir::mono::MonoItem; +use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::{bug, span_bug}; +use rustc_target::abi::{ + AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange, +}; +use std::ops::Range; +use tracing::debug; + +pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value { + let alloc = alloc.inner(); + let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1); + let dl = cx.data_layout(); + let pointer_size = dl.pointer_size.bytes() as usize; + + // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, + // so `range` must be within the bounds of `alloc` and not contain or overlap a relocation. + fn append_chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>( + llvals: &mut Vec<&'ll Value>, + cx: &'a CodegenCx<'ll, 'b>, + alloc: &'a Allocation, + range: Range, + ) { + let chunks = alloc + .init_mask() + .range_as_init_chunks(Size::from_bytes(range.start), Size::from_bytes(range.end)); + + let chunk_to_llval = move |chunk| match chunk { + InitChunk::Init(range) => { + let range = (range.start.bytes() as usize)..(range.end.bytes() as usize); + let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); + cx.const_bytes(bytes) + } + InitChunk::Uninit(range) => { + let len = range.end.bytes() - range.start.bytes(); + cx.const_undef(cx.type_array(cx.type_i8(), len)) + } + }; + + // Generating partially-uninit consts is limited to small numbers of chunks, + // to avoid the cost of generating large complex const expressions. + // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element, + // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`. + let max = if llvm_util::get_version() < (14, 0, 0) { + // Generating partially-uninit consts inhibits optimizations in LLVM < 14. + // See https://github.com/rust-lang/rust/issues/84565. + 1 + } else { + cx.sess().opts.unstable_opts.uninit_const_chunk_threshold + }; + let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; + + if allow_uninit_chunks { + llvals.extend(chunks.map(chunk_to_llval)); + } else { + // If this allocation contains any uninit bytes, codegen as if it was initialized + // (using some arbitrary value for uninit bytes). + let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); + llvals.push(cx.const_bytes(bytes)); + } + } + + let mut next_offset = 0; + for &(offset, alloc_id) in alloc.relocations().iter() { + let offset = offset.bytes(); + assert_eq!(offset as usize as u64, offset); + let offset = offset as usize; + if offset > next_offset { + // This `inspect` is okay since we have checked that it is not within a relocation, it + // is within the bounds of the allocation, and it doesn't affect interpreter execution + // (we inspect the result after interpreter execution). + append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); + } + let ptr_offset = read_target_uint( + dl.endian, + // This `inspect` is okay since it is within the bounds of the allocation, it doesn't + // affect interpreter execution (we inspect the result after interpreter execution), + // and we properly interpret the relocation as a relocation pointer offset. + alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), + ) + .expect("const_alloc_to_llvm: could not read relocation pointer") + as u64; + + let address_space = match cx.tcx.global_alloc(alloc_id) { + GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space, + GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) | GlobalAlloc::VTable(..) => { + AddressSpace::DATA + } + }; + + llvals.push(cx.scalar_to_backend( + InterpScalar::from_pointer( + Pointer::new(alloc_id, Size::from_bytes(ptr_offset)), + &cx.tcx, + ), + Scalar::Initialized { + value: Primitive::Pointer, + valid_range: WrappingRange::full(dl.pointer_size), + }, + cx.type_i8p_ext(address_space), + )); + next_offset = offset + pointer_size; + } + if alloc.len() >= next_offset { + let range = next_offset..alloc.len(); + // This `inspect` is okay since we have check that it is after all relocations, it is + // within the bounds of the allocation, and it doesn't affect interpreter execution (we + // inspect the result after interpreter execution). + append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); + } + + cx.const_struct(&llvals, true) +} + +pub fn codegen_static_initializer<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + def_id: DefId, +) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { + let alloc = cx.tcx.eval_static_initializer(def_id)?; + Ok((const_alloc_to_llvm(cx, alloc), alloc)) +} + +fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) { + // The target may require greater alignment for globals than the type does. + // Note: GCC and Clang also allow `__attribute__((aligned))` on variables, + // which can force it to be smaller. Rust doesn't support this yet. + if let Some(min) = cx.sess().target.min_global_align { + match Align::from_bits(min) { + Ok(min) => align = align.max(min), + Err(err) => { + cx.sess().err(&format!("invalid minimum global alignment: {}", err)); + } + } + } + unsafe { + llvm::LLVMSetAlignment(gv, align.bytes() as u32); + } +} + +fn check_and_apply_linkage<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + attrs: &CodegenFnAttrs, + ty: Ty<'tcx>, + sym: &str, + span_def_id: DefId, +) -> &'ll Value { + let llty = cx.layout_of(ty).llvm_type(cx); + if let Some(linkage) = attrs.linkage { + debug!("get_static: sym={} linkage={:?}", sym, linkage); + + // If this is a static with a linkage specified, then we need to handle + // it a little specially. The typesystem prevents things like &T and + // extern "C" fn() from being non-null, so we can't just declare a + // static and call it a day. Some linkages (like weak) will make it such + // that the static actually has a null value. + let llty2 = if let ty::RawPtr(ref mt) = ty.kind() { + cx.layout_of(mt.ty).llvm_type(cx) + } else { + cx.sess().span_fatal( + cx.tcx.def_span(span_def_id), + "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", + ) + }; + unsafe { + // Declare a symbol `foo` with the desired linkage. + let g1 = cx.declare_global(sym, llty2); + llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage)); + + // Declare an internal global `extern_with_linkage_foo` which + // is initialized with the address of `foo`. If `foo` is + // discarded during linking (for example, if `foo` has weak + // linkage and there are no definitions), then + // `extern_with_linkage_foo` will instead be initialized to + // zero. + let mut real_name = "_rust_extern_with_linkage_".to_string(); + real_name.push_str(sym); + let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { + cx.sess().span_fatal( + cx.tcx.def_span(span_def_id), + &format!("symbol `{}` is already defined", &sym), + ) + }); + llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage); + llvm::LLVMSetInitializer(g2, g1); + g2 + } + } else { + // Generate an external declaration. + // FIXME(nagisa): investigate whether it can be changed into define_global + cx.declare_global(sym, llty) + } +} + +pub fn ptrcast<'ll>(val: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMConstPointerCast(val, ty) } +} + +impl<'ll> CodegenCx<'ll, '_> { + pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMConstBitCast(val, ty) } + } + + pub(crate) fn static_addr_of_mut( + &self, + cv: &'ll Value, + align: Align, + kind: Option<&str>, + ) -> &'ll Value { + unsafe { + let gv = match kind { + Some(kind) if !self.tcx.sess.fewer_names() => { + let name = self.generate_local_symbol_name(kind); + let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", name); + }); + llvm::LLVMRustSetLinkage(gv, llvm::Linkage::PrivateLinkage); + gv + } + _ => self.define_private_global(self.val_ty(cv)), + }; + llvm::LLVMSetInitializer(gv, cv); + set_global_alignment(self, gv, align); + llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); + gv + } + } + + pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { + let instance = Instance::mono(self.tcx, def_id); + if let Some(&g) = self.instances.borrow().get(&instance) { + return g; + } + + let defined_in_current_codegen_unit = + self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); + assert!( + !defined_in_current_codegen_unit, + "consts::get_static() should always hit the cache for \ + statics defined in the same CGU, but did not for `{:?}`", + def_id + ); + + let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + let sym = self.tcx.symbol_name(instance).name; + let fn_attrs = self.tcx.codegen_fn_attrs(def_id); + + debug!("get_static: sym={} instance={:?} fn_attrs={:?}", sym, instance, fn_attrs); + + let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) { + let llty = self.layout_of(ty).llvm_type(self); + if let Some(g) = self.get_declared_value(sym) { + if self.val_ty(g) != self.type_ptr_to(llty) { + span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); + } + } + + let g = self.declare_global(sym, llty); + + if !self.tcx.is_reachable_non_generic(def_id) { + unsafe { + llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden); + } + } + + g + } else { + check_and_apply_linkage(self, fn_attrs, ty, sym, def_id) + }; + + // Thread-local statics in some other crate need to *always* be linked + // against in a thread-local fashion, so we need to be sure to apply the + // thread-local attribute locally if it was present remotely. If we + // don't do this then linker errors can be generated where the linker + // complains that one object files has a thread local version of the + // symbol and another one doesn't. + if fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { + llvm::set_thread_local_mode(g, self.tls_model); + } + + if !def_id.is_local() { + let needs_dll_storage_attr = self.use_dll_storage_attrs && !self.tcx.is_foreign_item(def_id) && + // ThinLTO can't handle this workaround in all cases, so we don't + // emit the attrs. Instead we make them unnecessary by disallowing + // dynamic linking when linker plugin based LTO is enabled. + !self.tcx.sess.opts.cg.linker_plugin_lto.enabled(); + + // If this assertion triggers, there's something wrong with commandline + // argument validation. + debug_assert!( + !(self.tcx.sess.opts.cg.linker_plugin_lto.enabled() + && self.tcx.sess.target.is_like_windows + && self.tcx.sess.opts.cg.prefer_dynamic) + ); + + if needs_dll_storage_attr { + // This item is external but not foreign, i.e., it originates from an external Rust + // crate. Since we don't know whether this crate will be linked dynamically or + // statically in the final application, we always mark such symbols as 'dllimport'. + // If final linkage happens to be static, we rely on compiler-emitted __imp_ stubs + // to make things work. + // + // However, in some scenarios we defer emission of statics to downstream + // crates, so there are cases where a static with an upstream DefId + // is actually present in the current crate. We can find out via the + // is_codegened_item query. + if !self.tcx.is_codegened_item(def_id) { + unsafe { + llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport); + } + } + } + } + + if self.use_dll_storage_attrs && self.tcx.is_dllimport_foreign_item(def_id) { + // For foreign (native) libs we know the exact storage type to use. + unsafe { + llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport); + } + } + + unsafe { + if self.should_assume_dso_local(g, true) { + llvm::LLVMRustSetDSOLocal(g, true); + } + } + + self.instances.borrow_mut().insert(instance, g); + g + } +} + +impl<'ll> StaticMethods for CodegenCx<'ll, '_> { + fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { + if let Some(&gv) = self.const_globals.borrow().get(&cv) { + unsafe { + // Upgrade the alignment in cases where the same constant is used with different + // alignment requirements + let llalign = align.bytes() as u32; + if llalign > llvm::LLVMGetAlignment(gv) { + llvm::LLVMSetAlignment(gv, llalign); + } + } + return gv; + } + let gv = self.static_addr_of_mut(cv, align, kind); + unsafe { + llvm::LLVMSetGlobalConstant(gv, True); + } + self.const_globals.borrow_mut().insert(cv, gv); + gv + } + + fn codegen_static(&self, def_id: DefId, is_mutable: bool) { + unsafe { + let attrs = self.tcx.codegen_fn_attrs(def_id); + + let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else { + // Error has already been reported + return; + }; + let alloc = alloc.inner(); + + let g = self.get_static(def_id); + + // boolean SSA values are i1, but they have to be stored in i8 slots, + // otherwise some LLVM optimization passes don't work as expected + let mut val_llty = self.val_ty(v); + let v = if val_llty == self.type_i1() { + val_llty = self.type_i8(); + llvm::LLVMConstZExt(v, val_llty) + } else { + v + }; + + let instance = Instance::mono(self.tcx, def_id); + let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + let llty = self.layout_of(ty).llvm_type(self); + let g = if val_llty == llty { + g + } else { + // If we created the global with the wrong type, + // correct the type. + let name = llvm::get_value_name(g).to_vec(); + llvm::set_value_name(g, b""); + + let linkage = llvm::LLVMRustGetLinkage(g); + let visibility = llvm::LLVMRustGetVisibility(g); + + let new_g = llvm::LLVMRustGetOrInsertGlobal( + self.llmod, + name.as_ptr().cast(), + name.len(), + val_llty, + ); + + llvm::LLVMRustSetLinkage(new_g, linkage); + llvm::LLVMRustSetVisibility(new_g, visibility); + + // The old global has had its name removed but is returned by + // get_static since it is in the instance cache. Provide an + // alternative lookup that points to the new global so that + // global_asm! can compute the correct mangled symbol name + // for the global. + self.renamed_statics.borrow_mut().insert(def_id, new_g); + + // To avoid breaking any invariants, we leave around the old + // global for the moment; we'll replace all references to it + // with the new global later. (See base::codegen_backend.) + self.statics_to_rauw.borrow_mut().push((g, new_g)); + new_g + }; + set_global_alignment(self, g, self.align_of(ty)); + llvm::LLVMSetInitializer(g, v); + + if self.should_assume_dso_local(g, true) { + llvm::LLVMRustSetDSOLocal(g, true); + } + + // As an optimization, all shared statics which do not have interior + // mutability are placed into read-only memory. + if !is_mutable && self.type_is_freeze(ty) { + llvm::LLVMSetGlobalConstant(g, llvm::True); + } + + debuginfo::build_global_var_di_node(self, def_id, g); + + if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { + llvm::set_thread_local_mode(g, self.tls_model); + + // Do not allow LLVM to change the alignment of a TLS on macOS. + // + // By default a global's alignment can be freely increased. + // This allows LLVM to generate more performant instructions + // e.g., using load-aligned into a SIMD register. + // + // However, on macOS 10.10 or below, the dynamic linker does not + // respect any alignment given on the TLS (radar 24221680). + // This will violate the alignment assumption, and causing segfault at runtime. + // + // This bug is very easy to trigger. In `println!` and `panic!`, + // the `LOCAL_STDOUT`/`LOCAL_STDERR` handles are stored in a TLS, + // which the values would be `mem::replace`d on initialization. + // The implementation of `mem::replace` will use SIMD + // whenever the size is 32 bytes or higher. LLVM notices SIMD is used + // and tries to align `LOCAL_STDOUT`/`LOCAL_STDERR` to a 32-byte boundary, + // which macOS's dyld disregarded and causing crashes + // (see issues #51794, #51758, #50867, #48866 and #44056). + // + // To workaround the bug, we trick LLVM into not increasing + // the global's alignment by explicitly assigning a section to it + // (equivalent to automatically generating a `#[link_section]` attribute). + // See the comment in the `GlobalValue::canIncreaseAlignment()` function + // of `lib/IR/Globals.cpp` for why this works. + // + // When the alignment is not increased, the optimized `mem::replace` + // will use load-unaligned instructions instead, and thus avoiding the crash. + // + // We could remove this hack whenever we decide to drop macOS 10.10 support. + if self.tcx.sess.target.is_like_osx { + // The `inspect` method is okay here because we checked relocations, and + // because we are doing this access to inspect the final interpreter state + // (not as part of the interpreter execution). + // + // FIXME: This check requires that the (arbitrary) value of undefined bytes + // happens to be zero. Instead, we should only check the value of defined bytes + // and set all undefined bytes to zero if this allocation is headed for the + // BSS. + let all_bytes_are_zero = alloc.relocations().is_empty() + && alloc + .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()) + .iter() + .all(|&byte| byte == 0); + + let sect_name = if all_bytes_are_zero { + cstr!("__DATA,__thread_bss") + } else { + cstr!("__DATA,__thread_data") + }; + llvm::LLVMSetSection(g, sect_name.as_ptr()); + } + } + + // Wasm statics with custom link sections get special treatment as they + // go into custom sections of the wasm executable. + if self.tcx.sess.target.is_like_wasm { + if let Some(section) = attrs.link_section { + let section = llvm::LLVMMDStringInContext( + self.llcx, + section.as_str().as_ptr().cast(), + section.as_str().len() as c_uint, + ); + assert!(alloc.relocations().is_empty()); + + // The `inspect` method is okay here because we checked relocations, and + // because we are doing this access to inspect the final interpreter state (not + // as part of the interpreter execution). + let bytes = + alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); + let alloc = llvm::LLVMMDStringInContext( + self.llcx, + bytes.as_ptr().cast(), + bytes.len() as c_uint, + ); + let data = [section, alloc]; + let meta = llvm::LLVMMDNodeInContext(self.llcx, data.as_ptr(), 2); + llvm::LLVMAddNamedMetadataOperand( + self.llmod, + "wasm.custom_sections\0".as_ptr().cast(), + meta, + ); + } + } else { + base::set_link_section(g, attrs); + } + + if attrs.flags.contains(CodegenFnAttrFlags::USED) { + // `USED` and `USED_LINKER` can't be used together. + assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)); + + // The semantics of #[used] in Rust only require the symbol to make it into the + // object file. It is explicitly allowed for the linker to strip the symbol if it + // is dead, which means we are allowed use `llvm.compiler.used` instead of + // `llvm.used` here. + // + // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique + // sections with SHF_GNU_RETAIN flag for llvm.used symbols, which may trigger bugs + // in the handling of `.init_array` (the static constructor list) in versions of + // the gold linker (prior to the one released with binutils 2.36). + // + // That said, we only ever emit these when compiling for ELF targets, unless + // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage + // on other targets, in particular MachO targets have *their* static constructor + // lists broken if `llvm.compiler.used` is emitted rather than llvm.used. However, + // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_typeck`, + // so we don't need to take care of it here. + self.add_compiler_used_global(g); + } + if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { + // `USED` and `USED_LINKER` can't be used together. + assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED)); + + self.add_used_global(g); + } + } + } + + /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*. + fn add_used_global(&self, global: &'ll Value) { + let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; + self.used_statics.borrow_mut().push(cast); + } + + /// Add a global value to a list to be stored in the `llvm.compiler.used` variable, + /// an array of i8*. + fn add_compiler_used_global(&self, global: &'ll Value) { + let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; + self.compiler_used_statics.borrow_mut().push(cast); + } +} diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs new file mode 100644 index 00000000000..5857b83f6c9 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -0,0 +1,1014 @@ +use crate::attributes; +use crate::back::write::to_llvm_code_model; +use crate::callee::get_fn; +use crate::coverageinfo; +use crate::debuginfo; +use crate::llvm; +use crate::llvm_util; +use crate::type_::Type; +use crate::value::Value; + +use cstr::cstr; +use rustc_codegen_ssa::base::wants_msvc_seh; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::base_n; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::def_id::DefId; +use rustc_middle::mir::mono::CodegenUnit; +use rustc_middle::ty::layout::{ + FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, + TyAndLayout, +}; +use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug}; +use rustc_session::config::{BranchProtection, CFGuard, CFProtection}; +use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; +use rustc_session::Session; +use rustc_span::source_map::Span; +use rustc_target::abi::{ + call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, +}; +use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; +use smallvec::SmallVec; + +use std::cell::{Cell, RefCell}; +use std::ffi::CStr; +use std::str; + +/// There is one `CodegenCx` per compilation unit. Each one has its own LLVM +/// `llvm::Context` so that several compilation units may be optimized in parallel. +/// All other LLVM data structures in the `CodegenCx` are tied to that `llvm::Context`. +pub struct CodegenCx<'ll, 'tcx> { + pub tcx: TyCtxt<'tcx>, + pub check_overflow: bool, + pub use_dll_storage_attrs: bool, + pub tls_model: llvm::ThreadLocalMode, + + pub llmod: &'ll llvm::Module, + pub llcx: &'ll llvm::Context, + pub codegen_unit: &'tcx CodegenUnit<'tcx>, + + /// Cache instances of monomorphic and polymorphic items + pub instances: RefCell, &'ll Value>>, + /// Cache generated vtables + pub vtables: + RefCell, Option>), &'ll Value>>, + /// Cache of constant strings, + pub const_str_cache: RefCell>, + + /// Reverse-direction for const ptrs cast from globals. + /// + /// Key is a Value holding a `*T`, + /// Val is a Value holding a `*[T]`. + /// + /// Needed because LLVM loses pointer->pointee association + /// when we ptrcast, and we have to ptrcast during codegen + /// of a `[T]` const because we form a slice, a `(*T,usize)` pair, not + /// a pointer to an LLVM array type. Similar for trait objects. + pub const_unsized: RefCell>, + + /// Cache of emitted const globals (value -> global) + pub const_globals: RefCell>, + + /// List of globals for static variables which need to be passed to the + /// LLVM function ReplaceAllUsesWith (RAUW) when codegen is complete. + /// (We have to make sure we don't invalidate any Values referring + /// to constants.) + pub statics_to_rauw: RefCell>, + + /// Statics that will be placed in the llvm.used variable + /// See for details + pub used_statics: RefCell>, + + /// Statics that will be placed in the llvm.compiler.used variable + /// See for details + pub compiler_used_statics: RefCell>, + + /// Mapping of non-scalar types to llvm types and field remapping if needed. + pub type_lowering: RefCell, Option), TypeLowering<'ll>>>, + + /// Mapping of scalar types to llvm types. + pub scalar_lltypes: RefCell, &'ll Type>>, + + pub pointee_infos: RefCell, Size), Option>>, + pub isize_ty: &'ll Type, + + pub coverage_cx: Option>, + pub dbg_cx: Option>, + + eh_personality: Cell>, + eh_catch_typeinfo: Cell>, + pub rust_try_fn: Cell>, + + intrinsics: RefCell>, + + /// A counter that is used for generating local symbol names + local_gen_sym_counter: Cell, + + /// `codegen_static` will sometimes create a second global variable with a + /// different type and clear the symbol name of the original global. + /// `global_asm!` needs to be able to find this new global so that it can + /// compute the correct mangled symbol name to insert into the asm. + pub renamed_statics: RefCell>, +} + +pub struct TypeLowering<'ll> { + /// Associated LLVM type + pub lltype: &'ll Type, + + /// If padding is used the slice maps fields from source order + /// to llvm order. + pub field_remapping: Option>, +} + +fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { + match tls_model { + TlsModel::GeneralDynamic => llvm::ThreadLocalMode::GeneralDynamic, + TlsModel::LocalDynamic => llvm::ThreadLocalMode::LocalDynamic, + TlsModel::InitialExec => llvm::ThreadLocalMode::InitialExec, + TlsModel::LocalExec => llvm::ThreadLocalMode::LocalExec, + } +} + +pub unsafe fn create_module<'ll>( + tcx: TyCtxt<'_>, + llcx: &'ll llvm::Context, + mod_name: &str, +) -> &'ll llvm::Module { + let sess = tcx.sess; + let mod_name = SmallCStr::new(mod_name); + let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx); + + let mut target_data_layout = sess.target.data_layout.to_string(); + let llvm_version = llvm_util::get_version(); + if llvm_version < (13, 0, 0) { + if sess.target.arch == "powerpc64" { + target_data_layout = target_data_layout.replace("-S128", ""); + } + if sess.target.arch == "wasm32" { + target_data_layout = "e-m:e-p:32:32-i64:64-n32:64-S128".to_string(); + } + if sess.target.arch == "wasm64" { + target_data_layout = "e-m:e-p:64:64-i64:64-n32:64-S128".to_string(); + } + } + if llvm_version < (14, 0, 0) { + if sess.target.llvm_target == "i686-pc-windows-msvc" + || sess.target.llvm_target == "i586-pc-windows-msvc" + { + target_data_layout = + "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32" + .to_string(); + } + if sess.target.arch == "wasm32" { + target_data_layout = target_data_layout.replace("-p10:8:8-p20:8:8", ""); + } + } + + // Ensure the data-layout values hardcoded remain the defaults. + if sess.target.is_builtin { + let tm = crate::back::write::create_informational_target_machine(tcx.sess); + llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, tm); + llvm::LLVMRustDisposeTargetMachine(tm); + + let llvm_data_layout = llvm::LLVMGetDataLayoutStr(llmod); + let llvm_data_layout = str::from_utf8(CStr::from_ptr(llvm_data_layout).to_bytes()) + .expect("got a non-UTF8 data-layout from LLVM"); + + // Unfortunately LLVM target specs change over time, and right now we + // don't have proper support to work with any more than one + // `data_layout` than the one that is in the rust-lang/rust repo. If + // this compiler is configured against a custom LLVM, we may have a + // differing data layout, even though we should update our own to use + // that one. + // + // As an interim hack, if CFG_LLVM_ROOT is not an empty string then we + // disable this check entirely as we may be configured with something + // that has a different target layout. + // + // Unsure if this will actually cause breakage when rustc is configured + // as such. + // + // FIXME(#34960) + let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or(""); + let custom_llvm_used = cfg_llvm_root.trim() != ""; + + if !custom_llvm_used && target_data_layout != llvm_data_layout { + bug!( + "data-layout for target `{rustc_target}`, `{rustc_layout}`, \ + differs from LLVM target's `{llvm_target}` default layout, `{llvm_layout}`", + rustc_target = sess.opts.target_triple, + rustc_layout = target_data_layout, + llvm_target = sess.target.llvm_target, + llvm_layout = llvm_data_layout + ); + } + } + + let data_layout = SmallCStr::new(&target_data_layout); + llvm::LLVMSetDataLayout(llmod, data_layout.as_ptr()); + + let llvm_target = SmallCStr::new(&sess.target.llvm_target); + llvm::LLVMRustSetNormalizedTarget(llmod, llvm_target.as_ptr()); + + let reloc_model = sess.relocation_model(); + if matches!(reloc_model, RelocModel::Pic | RelocModel::Pie) { + llvm::LLVMRustSetModulePICLevel(llmod); + // PIE is potentially more effective than PIC, but can only be used in executables. + // If all our outputs are executables, then we can relax PIC to PIE. + if reloc_model == RelocModel::Pie + || sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) + { + llvm::LLVMRustSetModulePIELevel(llmod); + } + } + + // Linking object files with different code models is undefined behavior + // because the compiler would have to generate additional code (to span + // longer jumps) if a larger code model is used with a smaller one. + // + // See https://reviews.llvm.org/D52322 and https://reviews.llvm.org/D52323. + llvm::LLVMRustSetModuleCodeModel(llmod, to_llvm_code_model(sess.code_model())); + + // If skipping the PLT is enabled, we need to add some module metadata + // to ensure intrinsic calls don't use it. + if !sess.needs_plt() { + let avoid_plt = "RtLibUseGOT\0".as_ptr().cast(); + llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); + } + + if sess.is_sanitizer_cfi_enabled() { + // FIXME(rcvalle): Add support for non canonical jump tables. + let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); + // FIXME(rcvalle): Add it with Override behavior flag. + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + canonical_jump_tables, + 1, + ); + } + + // Control Flow Guard is currently only supported by the MSVC linker on Windows. + if sess.target.is_like_msvc { + match sess.opts.cg.control_flow_guard { + CFGuard::Disabled => {} + CFGuard::NoChecks => { + // Set `cfguard=1` module flag to emit metadata only. + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + "cfguard\0".as_ptr() as *const _, + 1, + ) + } + CFGuard::Checks => { + // Set `cfguard=2` module flag to emit metadata and checks. + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + "cfguard\0".as_ptr() as *const _, + 2, + ) + } + } + } + + if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { + if sess.target.arch != "aarch64" { + sess.err("-Zbranch-protection is only supported on aarch64"); + } else { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "branch-target-enforcement\0".as_ptr().cast(), + bti.into(), + ); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "sign-return-address\0".as_ptr().cast(), + pac_ret.is_some().into(), + ); + let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "sign-return-address-all\0".as_ptr().cast(), + pac_opts.leaf.into(), + ); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "sign-return-address-with-bkey\0".as_ptr().cast(), + u32::from(pac_opts.key == PAuthKey::B), + ); + } + } + + // Pass on the control-flow protection flags to LLVM (equivalent to `-fcf-protection` in Clang). + if let CFProtection::Branch | CFProtection::Full = sess.opts.unstable_opts.cf_protection { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + "cf-protection-branch\0".as_ptr().cast(), + 1, + ) + } + if let CFProtection::Return | CFProtection::Full = sess.opts.unstable_opts.cf_protection { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + "cf-protection-return\0".as_ptr().cast(), + 1, + ) + } + + if sess.opts.unstable_opts.virtual_function_elimination { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "Virtual Function Elim\0".as_ptr().cast(), + 1, + ); + } + + llmod +} + +impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { + pub(crate) fn new( + tcx: TyCtxt<'tcx>, + codegen_unit: &'tcx CodegenUnit<'tcx>, + llvm_module: &'ll crate::ModuleLlvm, + ) -> Self { + // An interesting part of Windows which MSVC forces our hand on (and + // apparently MinGW didn't) is the usage of `dllimport` and `dllexport` + // attributes in LLVM IR as well as native dependencies (in C these + // correspond to `__declspec(dllimport)`). + // + // LD (BFD) in MinGW mode can often correctly guess `dllexport` but + // relying on that can result in issues like #50176. + // LLD won't support that and expects symbols with proper attributes. + // Because of that we make MinGW target emit dllexport just like MSVC. + // When it comes to dllimport we use it for constants but for functions + // rely on the linker to do the right thing. Opposed to dllexport this + // task is easy for them (both LD and LLD) and allows us to easily use + // symbols from static libraries in shared libraries. + // + // Whenever a dynamic library is built on Windows it must have its public + // interface specified by functions tagged with `dllexport` or otherwise + // they're not available to be linked against. This poses a few problems + // for the compiler, some of which are somewhat fundamental, but we use + // the `use_dll_storage_attrs` variable below to attach the `dllexport` + // attribute to all LLVM functions that are exported e.g., they're + // already tagged with external linkage). This is suboptimal for a few + // reasons: + // + // * If an object file will never be included in a dynamic library, + // there's no need to attach the dllexport attribute. Most object + // files in Rust are not destined to become part of a dll as binaries + // are statically linked by default. + // * If the compiler is emitting both an rlib and a dylib, the same + // source object file is currently used but with MSVC this may be less + // feasible. The compiler may be able to get around this, but it may + // involve some invasive changes to deal with this. + // + // The flip side of this situation is that whenever you link to a dll and + // you import a function from it, the import should be tagged with + // `dllimport`. At this time, however, the compiler does not emit + // `dllimport` for any declarations other than constants (where it is + // required), which is again suboptimal for even more reasons! + // + // * Calling a function imported from another dll without using + // `dllimport` causes the linker/compiler to have extra overhead (one + // `jmp` instruction on x86) when calling the function. + // * The same object file may be used in different circumstances, so a + // function may be imported from a dll if the object is linked into a + // dll, but it may be just linked against if linked into an rlib. + // * The compiler has no knowledge about whether native functions should + // be tagged dllimport or not. + // + // For now the compiler takes the perf hit (I do not have any numbers to + // this effect) by marking very little as `dllimport` and praying the + // linker will take care of everything. Fixing this problem will likely + // require adding a few attributes to Rust itself (feature gated at the + // start) and then strongly recommending static linkage on Windows! + let use_dll_storage_attrs = tcx.sess.target.is_like_windows; + + let check_overflow = tcx.sess.overflow_checks(); + + let tls_model = to_llvm_tls_model(tcx.sess.tls_model()); + + let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod()); + + let coverage_cx = if tcx.sess.instrument_coverage() { + let covctx = coverageinfo::CrateCoverageContext::new(); + Some(covctx) + } else { + None + }; + + let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None { + let dctx = debuginfo::CodegenUnitDebugContext::new(llmod); + debuginfo::metadata::build_compile_unit_di_node( + tcx, + codegen_unit.name().as_str(), + &dctx, + ); + Some(dctx) + } else { + None + }; + + let isize_ty = Type::ix_llcx(llcx, tcx.data_layout.pointer_size.bits()); + + CodegenCx { + tcx, + check_overflow, + use_dll_storage_attrs, + tls_model, + llmod, + llcx, + codegen_unit, + instances: Default::default(), + vtables: Default::default(), + const_str_cache: Default::default(), + const_unsized: Default::default(), + const_globals: Default::default(), + statics_to_rauw: RefCell::new(Vec::new()), + used_statics: RefCell::new(Vec::new()), + compiler_used_statics: RefCell::new(Vec::new()), + type_lowering: Default::default(), + scalar_lltypes: Default::default(), + pointee_infos: Default::default(), + isize_ty, + coverage_cx, + dbg_cx, + eh_personality: Cell::new(None), + eh_catch_typeinfo: Cell::new(None), + rust_try_fn: Cell::new(None), + intrinsics: Default::default(), + local_gen_sym_counter: Cell::new(0), + renamed_statics: Default::default(), + } + } + + pub(crate) fn statics_to_rauw(&self) -> &RefCell> { + &self.statics_to_rauw + } + + #[inline] + pub fn coverage_context(&self) -> Option<&coverageinfo::CrateCoverageContext<'ll, 'tcx>> { + self.coverage_cx.as_ref() + } + + fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { + let section = cstr!("llvm.metadata"); + let array = self.const_array(self.type_ptr_to(self.type_i8()), values); + + unsafe { + let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); + llvm::LLVMSetInitializer(g, array); + llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); + llvm::LLVMSetSection(g, section.as_ptr()); + } + } +} + +impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn vtables( + &self, + ) -> &RefCell, Option>), &'ll Value>> + { + &self.vtables + } + + fn get_fn(&self, instance: Instance<'tcx>) -> &'ll Value { + get_fn(self, instance) + } + + fn get_fn_addr(&self, instance: Instance<'tcx>) -> &'ll Value { + get_fn(self, instance) + } + + fn eh_personality(&self) -> &'ll Value { + // The exception handling personality function. + // + // If our compilation unit has the `eh_personality` lang item somewhere + // within it, then we just need to codegen that. Otherwise, we're + // building an rlib which will depend on some upstream implementation of + // this function, so we just codegen a generic reference to it. We don't + // specify any of the types for the function, we just make it a symbol + // that LLVM can later use. + // + // Note that MSVC is a little special here in that we don't use the + // `eh_personality` lang item at all. Currently LLVM has support for + // both Dwarf and SEH unwind mechanisms for MSVC targets and uses the + // *name of the personality function* to decide what kind of unwind side + // tables/landing pads to emit. It looks like Dwarf is used by default, + // injecting a dependency on the `_Unwind_Resume` symbol for resuming + // an "exception", but for MSVC we want to force SEH. This means that we + // can't actually have the personality function be our standard + // `rust_eh_personality` function, but rather we wired it up to the + // CRT's custom personality function, which forces LLVM to consider + // landing pads as "landing pads for SEH". + if let Some(llpersonality) = self.eh_personality.get() { + return llpersonality; + } + let tcx = self.tcx; + let llfn = match tcx.lang_items().eh_personality() { + Some(def_id) if !wants_msvc_seh(self.sess()) => self.get_fn_addr( + ty::Instance::resolve( + tcx, + ty::ParamEnv::reveal_all(), + def_id, + tcx.intern_substs(&[]), + ) + .unwrap() + .unwrap(), + ), + _ => { + let name = if wants_msvc_seh(self.sess()) { + "__CxxFrameHandler3" + } else { + "rust_eh_personality" + }; + if let Some(llfn) = self.get_declared_value(name) { + llfn + } else { + let fty = self.type_variadic_func(&[], self.type_i32()); + let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty); + let target_cpu = attributes::target_cpu_attr(self); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]); + llfn + } + } + }; + self.eh_personality.set(Some(llfn)); + llfn + } + + fn sess(&self) -> &Session { + self.tcx.sess + } + + fn check_overflow(&self) -> bool { + self.check_overflow + } + + fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx> { + self.codegen_unit + } + + fn used_statics(&self) -> &RefCell> { + &self.used_statics + } + + fn compiler_used_statics(&self) -> &RefCell> { + &self.compiler_used_statics + } + + fn set_frame_pointer_type(&self, llfn: &'ll Value) { + if let Some(attr) = attributes::frame_pointer_type_attr(self) { + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); + } + } + + fn apply_target_cpu_attr(&self, llfn: &'ll Value) { + let mut attrs = SmallVec::<[_; 2]>::new(); + attrs.push(attributes::target_cpu_attr(self)); + attrs.extend(attributes::tune_cpu_attr(self)); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); + } + + fn create_used_variable(&self) { + self.create_used_variable_impl(cstr!("llvm.used"), &*self.used_statics.borrow()); + } + + fn create_compiler_used_variable(&self) { + self.create_used_variable_impl( + cstr!("llvm.compiler.used"), + &*self.compiler_used_statics.borrow(), + ); + } + + fn declare_c_main(&self, fn_type: Self::Type) -> Option { + if self.get_declared_value("main").is_none() { + Some(self.declare_cfn("main", llvm::UnnamedAddr::Global, fn_type)) + } else { + // If the symbol already exists, it is an error: for example, the user wrote + // #[no_mangle] extern "C" fn main(..) {..} + // instead of #[start] + None + } + } +} + +impl<'ll> CodegenCx<'ll, '_> { + pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { + if let Some(v) = self.intrinsics.borrow().get(key).cloned() { + return v; + } + + self.declare_intrinsic(key).unwrap_or_else(|| bug!("unknown intrinsic '{}'", key)) + } + + fn insert_intrinsic( + &self, + name: &'static str, + args: Option<&[&'ll llvm::Type]>, + ret: &'ll llvm::Type, + ) -> (&'ll llvm::Type, &'ll llvm::Value) { + let fn_ty = if let Some(args) = args { + self.type_func(args, ret) + } else { + self.type_variadic_func(&[], ret) + }; + let f = self.declare_cfn(name, llvm::UnnamedAddr::No, fn_ty); + self.intrinsics.borrow_mut().insert(name, (fn_ty, f)); + (fn_ty, f) + } + + fn declare_intrinsic(&self, key: &str) -> Option<(&'ll Type, &'ll Value)> { + macro_rules! ifn { + ($name:expr, fn() -> $ret:expr) => ( + if key == $name { + return Some(self.insert_intrinsic($name, Some(&[]), $ret)); + } + ); + ($name:expr, fn(...) -> $ret:expr) => ( + if key == $name { + return Some(self.insert_intrinsic($name, None, $ret)); + } + ); + ($name:expr, fn($($arg:expr),*) -> $ret:expr) => ( + if key == $name { + return Some(self.insert_intrinsic($name, Some(&[$($arg),*]), $ret)); + } + ); + } + macro_rules! mk_struct { + ($($field_ty:expr),*) => (self.type_struct( &[$($field_ty),*], false)) + } + + let i8p = self.type_i8p(); + let void = self.type_void(); + let i1 = self.type_i1(); + let t_i8 = self.type_i8(); + let t_i16 = self.type_i16(); + let t_i32 = self.type_i32(); + let t_i64 = self.type_i64(); + let t_i128 = self.type_i128(); + let t_isize = self.type_isize(); + let t_f32 = self.type_f32(); + let t_f64 = self.type_f64(); + let t_metadata = self.type_metadata(); + + ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32); + ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32); + ifn!("llvm.wasm.trunc.unsigned.i64.f32", fn(t_f32) -> t_i64); + ifn!("llvm.wasm.trunc.unsigned.i64.f64", fn(t_f64) -> t_i64); + ifn!("llvm.wasm.trunc.signed.i32.f32", fn(t_f32) -> t_i32); + ifn!("llvm.wasm.trunc.signed.i32.f64", fn(t_f64) -> t_i32); + ifn!("llvm.wasm.trunc.signed.i64.f32", fn(t_f32) -> t_i64); + ifn!("llvm.wasm.trunc.signed.i64.f64", fn(t_f64) -> t_i64); + + ifn!("llvm.fptosi.sat.i8.f32", fn(t_f32) -> t_i8); + ifn!("llvm.fptosi.sat.i16.f32", fn(t_f32) -> t_i16); + ifn!("llvm.fptosi.sat.i32.f32", fn(t_f32) -> t_i32); + ifn!("llvm.fptosi.sat.i64.f32", fn(t_f32) -> t_i64); + ifn!("llvm.fptosi.sat.i128.f32", fn(t_f32) -> t_i128); + ifn!("llvm.fptosi.sat.i8.f64", fn(t_f64) -> t_i8); + ifn!("llvm.fptosi.sat.i16.f64", fn(t_f64) -> t_i16); + ifn!("llvm.fptosi.sat.i32.f64", fn(t_f64) -> t_i32); + ifn!("llvm.fptosi.sat.i64.f64", fn(t_f64) -> t_i64); + ifn!("llvm.fptosi.sat.i128.f64", fn(t_f64) -> t_i128); + + ifn!("llvm.fptoui.sat.i8.f32", fn(t_f32) -> t_i8); + ifn!("llvm.fptoui.sat.i16.f32", fn(t_f32) -> t_i16); + ifn!("llvm.fptoui.sat.i32.f32", fn(t_f32) -> t_i32); + ifn!("llvm.fptoui.sat.i64.f32", fn(t_f32) -> t_i64); + ifn!("llvm.fptoui.sat.i128.f32", fn(t_f32) -> t_i128); + ifn!("llvm.fptoui.sat.i8.f64", fn(t_f64) -> t_i8); + ifn!("llvm.fptoui.sat.i16.f64", fn(t_f64) -> t_i16); + ifn!("llvm.fptoui.sat.i32.f64", fn(t_f64) -> t_i32); + ifn!("llvm.fptoui.sat.i64.f64", fn(t_f64) -> t_i64); + ifn!("llvm.fptoui.sat.i128.f64", fn(t_f64) -> t_i128); + + ifn!("llvm.trap", fn() -> void); + ifn!("llvm.debugtrap", fn() -> void); + ifn!("llvm.frameaddress", fn(t_i32) -> i8p); + + ifn!("llvm.powi.f32", fn(t_f32, t_i32) -> t_f32); + ifn!("llvm.powi.f64", fn(t_f64, t_i32) -> t_f64); + + ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32); + ifn!("llvm.pow.f64", fn(t_f64, t_f64) -> t_f64); + + ifn!("llvm.sqrt.f32", fn(t_f32) -> t_f32); + ifn!("llvm.sqrt.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.sin.f32", fn(t_f32) -> t_f32); + ifn!("llvm.sin.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.cos.f32", fn(t_f32) -> t_f32); + ifn!("llvm.cos.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.exp.f32", fn(t_f32) -> t_f32); + ifn!("llvm.exp.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.exp2.f32", fn(t_f32) -> t_f32); + ifn!("llvm.exp2.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.log.f32", fn(t_f32) -> t_f32); + ifn!("llvm.log.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.log10.f32", fn(t_f32) -> t_f32); + ifn!("llvm.log10.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.log2.f32", fn(t_f32) -> t_f32); + ifn!("llvm.log2.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.fma.f32", fn(t_f32, t_f32, t_f32) -> t_f32); + ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64); + + ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32); + ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32); + ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32); + ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64); + + ifn!("llvm.floor.f32", fn(t_f32) -> t_f32); + ifn!("llvm.floor.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.ceil.f32", fn(t_f32) -> t_f32); + ifn!("llvm.ceil.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.trunc.f32", fn(t_f32) -> t_f32); + ifn!("llvm.trunc.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32); + ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.round.f32", fn(t_f32) -> t_f32); + ifn!("llvm.round.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.rint.f32", fn(t_f32) -> t_f32); + ifn!("llvm.rint.f64", fn(t_f64) -> t_f64); + ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32); + ifn!("llvm.nearbyint.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.ctpop.i8", fn(t_i8) -> t_i8); + ifn!("llvm.ctpop.i16", fn(t_i16) -> t_i16); + ifn!("llvm.ctpop.i32", fn(t_i32) -> t_i32); + ifn!("llvm.ctpop.i64", fn(t_i64) -> t_i64); + ifn!("llvm.ctpop.i128", fn(t_i128) -> t_i128); + + ifn!("llvm.ctlz.i8", fn(t_i8, i1) -> t_i8); + ifn!("llvm.ctlz.i16", fn(t_i16, i1) -> t_i16); + ifn!("llvm.ctlz.i32", fn(t_i32, i1) -> t_i32); + ifn!("llvm.ctlz.i64", fn(t_i64, i1) -> t_i64); + ifn!("llvm.ctlz.i128", fn(t_i128, i1) -> t_i128); + + ifn!("llvm.cttz.i8", fn(t_i8, i1) -> t_i8); + ifn!("llvm.cttz.i16", fn(t_i16, i1) -> t_i16); + ifn!("llvm.cttz.i32", fn(t_i32, i1) -> t_i32); + ifn!("llvm.cttz.i64", fn(t_i64, i1) -> t_i64); + ifn!("llvm.cttz.i128", fn(t_i128, i1) -> t_i128); + + ifn!("llvm.bswap.i16", fn(t_i16) -> t_i16); + ifn!("llvm.bswap.i32", fn(t_i32) -> t_i32); + ifn!("llvm.bswap.i64", fn(t_i64) -> t_i64); + ifn!("llvm.bswap.i128", fn(t_i128) -> t_i128); + + ifn!("llvm.bitreverse.i8", fn(t_i8) -> t_i8); + ifn!("llvm.bitreverse.i16", fn(t_i16) -> t_i16); + ifn!("llvm.bitreverse.i32", fn(t_i32) -> t_i32); + ifn!("llvm.bitreverse.i64", fn(t_i64) -> t_i64); + ifn!("llvm.bitreverse.i128", fn(t_i128) -> t_i128); + + ifn!("llvm.fshl.i8", fn(t_i8, t_i8, t_i8) -> t_i8); + ifn!("llvm.fshl.i16", fn(t_i16, t_i16, t_i16) -> t_i16); + ifn!("llvm.fshl.i32", fn(t_i32, t_i32, t_i32) -> t_i32); + ifn!("llvm.fshl.i64", fn(t_i64, t_i64, t_i64) -> t_i64); + ifn!("llvm.fshl.i128", fn(t_i128, t_i128, t_i128) -> t_i128); + + ifn!("llvm.fshr.i8", fn(t_i8, t_i8, t_i8) -> t_i8); + ifn!("llvm.fshr.i16", fn(t_i16, t_i16, t_i16) -> t_i16); + ifn!("llvm.fshr.i32", fn(t_i32, t_i32, t_i32) -> t_i32); + ifn!("llvm.fshr.i64", fn(t_i64, t_i64, t_i64) -> t_i64); + ifn!("llvm.fshr.i128", fn(t_i128, t_i128, t_i128) -> t_i128); + + ifn!("llvm.sadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.sadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.sadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.sadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.sadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.uadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.uadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.uadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.uadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.uadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.ssub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.ssub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.ssub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.ssub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.ssub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.usub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.usub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.usub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.usub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.usub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.smul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.smul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.smul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.smul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.smul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.umul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.umul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.umul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.umul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.umul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.sadd.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.sadd.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!("llvm.sadd.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!("llvm.sadd.sat.i64", fn(t_i64, t_i64) -> t_i64); + ifn!("llvm.sadd.sat.i128", fn(t_i128, t_i128) -> t_i128); + + ifn!("llvm.uadd.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.uadd.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!("llvm.uadd.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!("llvm.uadd.sat.i64", fn(t_i64, t_i64) -> t_i64); + ifn!("llvm.uadd.sat.i128", fn(t_i128, t_i128) -> t_i128); + + ifn!("llvm.ssub.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.ssub.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!("llvm.ssub.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!("llvm.ssub.sat.i64", fn(t_i64, t_i64) -> t_i64); + ifn!("llvm.ssub.sat.i128", fn(t_i128, t_i128) -> t_i128); + + ifn!("llvm.usub.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.usub.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!("llvm.usub.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64); + ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128); + + ifn!("llvm.lifetime.start.p0i8", fn(t_i64, i8p) -> void); + ifn!("llvm.lifetime.end.p0i8", fn(t_i64, i8p) -> void); + + ifn!("llvm.expect.i1", fn(i1, i1) -> i1); + ifn!("llvm.eh.typeid.for", fn(i8p) -> t_i32); + ifn!("llvm.localescape", fn(...) -> void); + ifn!("llvm.localrecover", fn(i8p, i8p, t_i32) -> i8p); + ifn!("llvm.x86.seh.recoverfp", fn(i8p, i8p) -> i8p); + + ifn!("llvm.assume", fn(i1) -> void); + ifn!("llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void); + + // This isn't an "LLVM intrinsic", but LLVM's optimization passes + // recognize it like one and we assume it exists in `core::slice::cmp` + match self.sess().target.arch.as_ref() { + "avr" | "msp430" => ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i16), + _ => ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i32), + } + + // variadic intrinsics + ifn!("llvm.va_start", fn(i8p) -> void); + ifn!("llvm.va_end", fn(i8p) -> void); + ifn!("llvm.va_copy", fn(i8p, i8p) -> void); + + if self.sess().instrument_coverage() { + ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void); + } + + ifn!("llvm.type.test", fn(i8p, t_metadata) -> i1); + ifn!("llvm.type.checked.load", fn(i8p, t_i32, t_metadata) -> mk_struct! {i8p, i1}); + + if self.sess().opts.debuginfo != DebugInfo::None { + ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); + ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); + } + None + } + + pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value { + if let Some(eh_catch_typeinfo) = self.eh_catch_typeinfo.get() { + return eh_catch_typeinfo; + } + let tcx = self.tcx; + assert!(self.sess().target.os == "emscripten"); + let eh_catch_typeinfo = match tcx.lang_items().eh_catch_typeinfo() { + Some(def_id) => self.get_static(def_id), + _ => { + let ty = self + .type_struct(&[self.type_ptr_to(self.type_isize()), self.type_i8p()], false); + self.declare_global("rust_eh_catch_typeinfo", ty) + } + }; + let eh_catch_typeinfo = self.const_bitcast(eh_catch_typeinfo, self.type_i8p()); + self.eh_catch_typeinfo.set(Some(eh_catch_typeinfo)); + eh_catch_typeinfo + } +} + +impl CodegenCx<'_, '_> { + /// Generates a new symbol name with the given prefix. This symbol name must + /// only be used for definitions with `internal` or `private` linkage. + pub fn generate_local_symbol_name(&self, prefix: &str) -> String { + let idx = self.local_gen_sym_counter.get(); + self.local_gen_sym_counter.set(idx + 1); + // Include a '.' character, so there can be no accidental conflicts with + // user defined names + let mut name = String::with_capacity(prefix.len() + 6); + name.push_str(prefix); + name.push('.'); + base_n::push_str(idx as u128, base_n::ALPHANUMERIC_ONLY, &mut name); + name + } +} + +impl HasDataLayout for CodegenCx<'_, '_> { + #[inline] + fn data_layout(&self) -> &TargetDataLayout { + &self.tcx.data_layout + } +} + +impl HasTargetSpec for CodegenCx<'_, '_> { + #[inline] + fn target_spec(&self) -> &Target { + &self.tcx.sess.target + } +} + +impl<'tcx> ty::layout::HasTyCtxt<'tcx> for CodegenCx<'_, 'tcx> { + #[inline] + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } +} + +impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> { + fn param_env(&self) -> ty::ParamEnv<'tcx> { + ty::ParamEnv::reveal_all() + } +} + +impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { + type LayoutOfResult = TyAndLayout<'tcx>; + + #[inline] + fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { + if let LayoutError::SizeOverflow(_) = err { + self.sess().span_fatal(span, &err.to_string()) + } else { + span_bug!(span, "failed to get layout for `{}`: {}", ty, err) + } + } +} + +impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { + type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; + + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { + self.sess().span_fatal(span, &err.to_string()) + } else { + match fn_abi_request { + FnAbiRequest::OfFnPtr { sig, extra_args } => { + span_bug!( + span, + "`fn_abi_of_fn_ptr({}, {:?})` failed: {}", + sig, + extra_args, + err + ); + } + FnAbiRequest::OfInstance { instance, extra_args } => { + span_bug!( + span, + "`fn_abi_of_instance({}, {:?})` failed: {}", + instance, + extra_args, + err + ); + } + } + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs new file mode 100644 index 00000000000..58f391692c4 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -0,0 +1,334 @@ +use crate::common::CodegenCx; +use crate::coverageinfo; +use crate::llvm; + +use llvm::coverageinfo::CounterMappingRegion; +use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; +use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods}; +use rustc_data_structures::fx::FxIndexSet; +use rustc_hir::def::DefKind; +use rustc_hir::def_id::DefIdSet; +use rustc_llvm::RustString; +use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mir::coverage::CodeRegion; +use rustc_middle::ty::TyCtxt; + +use std::ffi::CString; + +use tracing::debug; + +/// Generates and exports the Coverage Map. +/// +/// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions +/// 5 (LLVM 12, only) and 6 (zero-based encoded as 4 and 5, respectively), as defined at +/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). +/// These versions are supported by the LLVM coverage tools (`llvm-profdata` and `llvm-cov`) +/// bundled with Rust's fork of LLVM. +/// +/// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with +/// the same version. Clang's implementation of Coverage Map generation was referenced when +/// implementing this Rust version, and though the format documentation is very explicit and +/// detailed, some undocumented details in Clang's implementation (that may or may not be important) +/// were also replicated for Rust's Coverage Map. +pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { + let tcx = cx.tcx; + + // Ensure the installed version of LLVM supports at least Coverage Map + // Version 5 (encoded as a zero-based value: 4), which was introduced with + // LLVM 12. + let version = coverageinfo::mapping_version(); + if version < 4 { + tcx.sess.fatal("rustc option `-C instrument-coverage` requires LLVM 12 or higher."); + } + + debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name()); + + // In order to show that unused functions have coverage counts of zero (0), LLVM requires the + // functions exist. Generate synthetic functions with a (required) single counter, and add the + // MIR `Coverage` code regions to the `function_coverage_map`, before calling + // `ctx.take_function_coverage_map()`. + if cx.codegen_unit.is_code_coverage_dead_code_cgu() { + add_unused_functions(cx); + } + + let function_coverage_map = match cx.coverage_context() { + Some(ctx) => ctx.take_function_coverage_map(), + None => return, + }; + + if function_coverage_map.is_empty() { + // This module has no functions with coverage instrumentation + return; + } + + let mut mapgen = CoverageMapGenerator::new(tcx, version); + + // Encode coverage mappings and generate function records + let mut function_data = Vec::new(); + for (instance, function_coverage) in function_coverage_map { + debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); + let mangled_function_name = tcx.symbol_name(instance).to_string(); + let source_hash = function_coverage.source_hash(); + let is_used = function_coverage.is_used(); + let (expressions, counter_regions) = + function_coverage.get_expressions_and_counter_regions(); + + let coverage_mapping_buffer = llvm::build_byte_buffer(|coverage_mapping_buffer| { + mapgen.write_coverage_mapping(expressions, counter_regions, coverage_mapping_buffer); + }); + + if coverage_mapping_buffer.is_empty() { + if function_coverage.is_used() { + bug!( + "A used function should have had coverage mapping data but did not: {}", + mangled_function_name + ); + } else { + debug!("unused function had no coverage mapping data: {}", mangled_function_name); + continue; + } + } + + function_data.push((mangled_function_name, source_hash, is_used, coverage_mapping_buffer)); + } + + // Encode all filenames referenced by counters/expressions in this module + let filenames_buffer = llvm::build_byte_buffer(|filenames_buffer| { + coverageinfo::write_filenames_section_to_buffer(&mapgen.filenames, filenames_buffer); + }); + + let filenames_size = filenames_buffer.len(); + let filenames_val = cx.const_bytes(&filenames_buffer); + let filenames_ref = coverageinfo::hash_bytes(filenames_buffer); + + // Generate the LLVM IR representation of the coverage map and store it in a well-known global + let cov_data_val = mapgen.generate_coverage_map(cx, version, filenames_size, filenames_val); + + for (mangled_function_name, source_hash, is_used, coverage_mapping_buffer) in function_data { + save_function_record( + cx, + mangled_function_name, + source_hash, + filenames_ref, + coverage_mapping_buffer, + is_used, + ); + } + + // Save the coverage data value to LLVM IR + coverageinfo::save_cov_data_to_mod(cx, cov_data_val); +} + +struct CoverageMapGenerator { + filenames: FxIndexSet, +} + +impl CoverageMapGenerator { + fn new(tcx: TyCtxt<'_>, version: u32) -> Self { + let mut filenames = FxIndexSet::default(); + if version >= 5 { + // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) + // requires setting the first filename to the compilation directory. + // Since rustc generates coverage maps with relative paths, the + // compilation directory can be combined with the the relative paths + // to get absolute paths, if needed. + let working_dir = tcx + .sess + .opts + .working_dir + .remapped_path_if_available() + .to_string_lossy() + .to_string(); + let c_filename = + CString::new(working_dir).expect("null error converting filename to C string"); + filenames.insert(c_filename); + } + Self { filenames } + } + + /// Using the `expressions` and `counter_regions` collected for the current function, generate + /// the `mapping_regions` and `virtual_file_mapping`, and capture any new filenames. Then use + /// LLVM APIs to encode the `virtual_file_mapping`, `expressions`, and `mapping_regions` into + /// the given `coverage_mapping` byte buffer, compliant with the LLVM Coverage Mapping format. + fn write_coverage_mapping<'a>( + &mut self, + expressions: Vec, + counter_regions: impl Iterator, + coverage_mapping_buffer: &RustString, + ) { + let mut counter_regions = counter_regions.collect::>(); + if counter_regions.is_empty() { + return; + } + + let mut virtual_file_mapping = Vec::new(); + let mut mapping_regions = Vec::new(); + let mut current_file_name = None; + let mut current_file_id = 0; + + // Convert the list of (Counter, CodeRegion) pairs to an array of `CounterMappingRegion`, sorted + // by filename and position. Capture any new files to compute the `CounterMappingRegion`s + // `file_id` (indexing files referenced by the current function), and construct the + // function-specific `virtual_file_mapping` from `file_id` to its index in the module's + // `filenames` array. + counter_regions.sort_unstable_by_key(|(_counter, region)| *region); + for (counter, region) in counter_regions { + let CodeRegion { file_name, start_line, start_col, end_line, end_col } = *region; + let same_file = current_file_name.as_ref().map_or(false, |p| *p == file_name); + if !same_file { + if current_file_name.is_some() { + current_file_id += 1; + } + current_file_name = Some(file_name); + let c_filename = CString::new(file_name.to_string()) + .expect("null error converting filename to C string"); + debug!(" file_id: {} = '{:?}'", current_file_id, c_filename); + let (filenames_index, _) = self.filenames.insert_full(c_filename); + virtual_file_mapping.push(filenames_index as u32); + } + debug!("Adding counter {:?} to map for {:?}", counter, region); + mapping_regions.push(CounterMappingRegion::code_region( + counter, + current_file_id, + start_line, + start_col, + end_line, + end_col, + )); + } + + // Encode and append the current function's coverage mapping data + coverageinfo::write_mapping_to_buffer( + virtual_file_mapping, + expressions, + mapping_regions, + coverage_mapping_buffer, + ); + } + + /// Construct coverage map header and the array of function records, and combine them into the + /// coverage map. Save the coverage map data into the LLVM IR as a static global using a + /// specific, well-known section and name. + fn generate_coverage_map<'ll>( + self, + cx: &CodegenCx<'ll, '_>, + version: u32, + filenames_size: usize, + filenames_val: &'ll llvm::Value, + ) -> &'ll llvm::Value { + debug!("cov map: filenames_size = {}, 0-based version = {}", filenames_size, version); + + // Create the coverage data header (Note, fields 0 and 2 are now always zero, + // as of `llvm::coverage::CovMapVersion::Version4`.) + let zero_was_n_records_val = cx.const_u32(0); + let filenames_size_val = cx.const_u32(filenames_size as u32); + let zero_was_coverage_size_val = cx.const_u32(0); + let version_val = cx.const_u32(version); + let cov_data_header_val = cx.const_struct( + &[zero_was_n_records_val, filenames_size_val, zero_was_coverage_size_val, version_val], + /*packed=*/ false, + ); + + // Create the complete LLVM coverage data value to add to the LLVM IR + cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false) + } +} + +/// Construct a function record and combine it with the function's coverage mapping data. +/// Save the function record into the LLVM IR as a static global using a +/// specific, well-known section and name. +fn save_function_record( + cx: &CodegenCx<'_, '_>, + mangled_function_name: String, + source_hash: u64, + filenames_ref: u64, + coverage_mapping_buffer: Vec, + is_used: bool, +) { + // Concatenate the encoded coverage mappings + let coverage_mapping_size = coverage_mapping_buffer.len(); + let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer); + + let func_name_hash = coverageinfo::hash_str(&mangled_function_name); + let func_name_hash_val = cx.const_u64(func_name_hash); + let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); + let source_hash_val = cx.const_u64(source_hash); + let filenames_ref_val = cx.const_u64(filenames_ref); + let func_record_val = cx.const_struct( + &[ + func_name_hash_val, + coverage_mapping_size_val, + source_hash_val, + filenames_ref_val, + coverage_mapping_val, + ], + /*packed=*/ true, + ); + + coverageinfo::save_func_record_to_mod(cx, func_name_hash, func_record_val, is_used); +} + +/// When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s and counters for +/// the functions that went through codegen; such as public functions and "used" functions +/// (functions referenced by other "used" or public items). Any other functions considered unused, +/// or "Unreachable", were still parsed and processed through the MIR stage, but were not +/// codegenned. (Note that `-Clink-dead-code` can force some unused code to be codegenned, but +/// that flag is known to cause other errors, when combined with `-C instrument-coverage`; and +/// `-Clink-dead-code` will not generate code for unused generic functions.) +/// +/// We can find the unused functions (including generic functions) by the set difference of all MIR +/// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query +/// `codegened_and_inlined_items`). +/// +/// These unused functions are then codegen'd in one of the CGUs which is marked as the +/// "code coverage dead code cgu" during the partitioning process. This prevents us from generating +/// code regions for the same function more than once which can lead to linker errors regarding +/// duplicate symbols. +fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { + assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); + + let tcx = cx.tcx; + + let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); + + let eligible_def_ids: DefIdSet = tcx + .mir_keys(()) + .iter() + .filter_map(|local_def_id| { + let def_id = local_def_id.to_def_id(); + let kind = tcx.def_kind(def_id); + // `mir_keys` will give us `DefId`s for all kinds of things, not + // just "functions", like consts, statics, etc. Filter those out. + // If `ignore_unused_generics` was specified, filter out any + // generic functions from consideration as well. + if !matches!( + kind, + DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Generator + ) { + return None; + } else if ignore_unused_generics + && tcx.generics_of(def_id).requires_monomorphization(tcx) + { + return None; + } + Some(local_def_id.to_def_id()) + }) + .collect(); + + let codegenned_def_ids = tcx.codegened_and_inlined_items(()); + + for &non_codegenned_def_id in eligible_def_ids.difference(codegenned_def_ids) { + let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id); + + // If a function is marked `#[no_coverage]`, then skip generating a + // dead code stub for it. + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) { + debug!("skipping unused fn marked #[no_coverage]: {:?}", non_codegenned_def_id); + continue; + } + + debug!("generating unused fn: {:?}", non_codegenned_def_id); + cx.define_unused_fn(non_codegenned_def_id); + } +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs new file mode 100644 index 00000000000..98ba38356a4 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -0,0 +1,385 @@ +use crate::llvm; + +use crate::abi::Abi; +use crate::builder::Builder; +use crate::common::CodegenCx; + +use libc::c_uint; +use llvm::coverageinfo::CounterMappingRegion; +use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage}; +use rustc_codegen_ssa::traits::{ + BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, + MiscMethods, StaticMethods, +}; +use rustc_data_structures::fx::FxHashMap; +use rustc_hir as hir; +use rustc_hir::def_id::DefId; +use rustc_llvm::RustString; +use rustc_middle::bug; +use rustc_middle::mir::coverage::{ + CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op, +}; +use rustc_middle::ty; +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::subst::InternalSubsts; +use rustc_middle::ty::Instance; + +use std::cell::RefCell; +use std::ffi::CString; + +use std::iter; +use tracing::debug; + +pub mod mapgen; + +const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START; + +const VAR_ALIGN_BYTES: usize = 8; + +/// A context object for maintaining all state needed by the coverageinfo module. +pub struct CrateCoverageContext<'ll, 'tcx> { + // Coverage data for each instrumented function identified by DefId. + pub(crate) function_coverage_map: RefCell, FunctionCoverage<'tcx>>>, + pub(crate) pgo_func_name_var_map: RefCell, &'ll llvm::Value>>, +} + +impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { + pub fn new() -> Self { + Self { + function_coverage_map: Default::default(), + pgo_func_name_var_map: Default::default(), + } + } + + pub fn take_function_coverage_map(&self) -> FxHashMap, FunctionCoverage<'tcx>> { + self.function_coverage_map.replace(FxHashMap::default()) + } +} + +impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn coverageinfo_finalize(&self) { + mapgen::finalize(self) + } + + fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { + if let Some(coverage_context) = self.coverage_context() { + debug!("getting pgo_func_name_var for instance={:?}", instance); + let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); + pgo_func_name_var_map + .entry(instance) + .or_insert_with(|| create_pgo_func_name_var(self, instance)) + } else { + bug!("Could not get the `coverage_context`"); + } + } + + /// Functions with MIR-based coverage are normally codegenned _only_ if + /// called. LLVM coverage tools typically expect every function to be + /// defined (even if unused), with at least one call to LLVM intrinsic + /// `instrprof.increment`. + /// + /// Codegen a small function that will never be called, with one counter + /// that will never be incremented. + /// + /// For used/called functions, the coverageinfo was already added to the + /// `function_coverage_map` (keyed by function `Instance`) during codegen. + /// But in this case, since the unused function was _not_ previously + /// codegenned, collect the coverage `CodeRegion`s from the MIR and add + /// them. The first `CodeRegion` is used to add a single counter, with the + /// same counter ID used in the injected `instrprof.increment` intrinsic + /// call. Since the function is never called, all other `CodeRegion`s can be + /// added as `unreachable_region`s. + fn define_unused_fn(&self, def_id: DefId) { + let instance = declare_unused_fn(self, def_id); + codegen_unused_fn_and_counter(self, instance); + add_unused_function_coverage(self, instance, def_id); + } +} + +impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + fn set_function_source_hash( + &mut self, + instance: Instance<'tcx>, + function_source_hash: u64, + ) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "ensuring function source hash is set for instance={:?}; function_source_hash={}", + instance, function_source_hash, + ); + let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + coverage_map + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .set_function_source_hash(function_source_hash); + true + } else { + false + } + } + + fn add_coverage_counter( + &mut self, + instance: Instance<'tcx>, + id: CounterValueReference, + region: CodeRegion, + ) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", + instance, id, region, + ); + let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + coverage_map + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .add_counter(id, region); + true + } else { + false + } + } + + fn add_coverage_counter_expression( + &mut self, + instance: Instance<'tcx>, + id: InjectedExpressionId, + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + region: Option, + ) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; \ + region: {:?}", + instance, id, lhs, op, rhs, region, + ); + let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + coverage_map + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .add_counter_expression(id, lhs, op, rhs, region); + true + } else { + false + } + } + + fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "adding unreachable code to coverage_map: instance={:?}, at {:?}", + instance, region, + ); + let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + coverage_map + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .add_unreachable_region(region); + true + } else { + false + } + } +} + +fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<'tcx> { + let tcx = cx.tcx; + + let instance = Instance::new( + def_id, + InternalSubsts::for_item(tcx, def_id, |param, _| { + if let ty::GenericParamDefKind::Lifetime = param.kind { + tcx.lifetimes.re_erased.into() + } else { + tcx.mk_param_from_def(param) + } + }), + ); + + let llfn = cx.declare_fn( + tcx.symbol_name(instance).name, + cx.fn_abi_of_fn_ptr( + ty::Binder::dummy(tcx.mk_fn_sig( + iter::once(tcx.mk_unit()), + tcx.mk_unit(), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + )), + ty::List::empty(), + ), + ); + + llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); + llvm::set_visibility(llfn, llvm::Visibility::Default); + + assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none()); + + instance +} + +fn codegen_unused_fn_and_counter<'tcx>(cx: &CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { + let llfn = cx.get_fn(instance); + let llbb = Builder::append_block(cx, llfn, "unused_function"); + let mut bx = Builder::build(cx, llbb); + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(0); + let num_counters = bx.const_u32(1); + let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID)); + debug!( + "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, + index={:?}) for unused function: {:?}", + fn_name, hash, num_counters, index, instance + ); + bx.instrprof_increment(fn_name, hash, num_counters, index); + bx.ret_void(); +} + +fn add_unused_function_coverage<'tcx>( + cx: &CodegenCx<'_, 'tcx>, + instance: Instance<'tcx>, + def_id: DefId, +) { + let tcx = cx.tcx; + + let mut function_coverage = FunctionCoverage::unused(tcx, instance); + for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { + if index == 0 { + // Insert at least one real counter so the LLVM CoverageMappingReader will find expected + // definitions. + function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); + } else { + function_coverage.add_unreachable_region(code_region.clone()); + } + } + + if let Some(coverage_context) = cx.coverage_context() { + coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); + } else { + bug!("Could not get the `coverage_context`"); + } +} + +/// Calls llvm::createPGOFuncNameVar() with the given function instance's +/// mangled function name. The LLVM API returns an llvm::GlobalVariable +/// containing the function name, with the specific variable name and linkage +/// required by LLVM InstrProf source-based coverage instrumentation. Use +/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per +/// `Instance`. +fn create_pgo_func_name_var<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, +) -> &'ll llvm::Value { + let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name) + .expect("error converting function name to C string"); + let llfn = cx.get_fn(instance); + unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } +} + +pub(crate) fn write_filenames_section_to_buffer<'a>( + filenames: impl IntoIterator, + buffer: &RustString, +) { + let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::>(); + unsafe { + llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( + c_str_vec.as_ptr(), + c_str_vec.len(), + buffer, + ); + } +} + +pub(crate) fn write_mapping_to_buffer( + virtual_file_mapping: Vec, + expressions: Vec, + mapping_regions: Vec, + buffer: &RustString, +) { + unsafe { + llvm::LLVMRustCoverageWriteMappingToBuffer( + virtual_file_mapping.as_ptr(), + virtual_file_mapping.len() as c_uint, + expressions.as_ptr(), + expressions.len() as c_uint, + mapping_regions.as_ptr(), + mapping_regions.len() as c_uint, + buffer, + ); + } +} + +pub(crate) fn hash_str(strval: &str) -> u64 { + let strval = CString::new(strval).expect("null error converting hashable str to C string"); + unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } +} + +pub(crate) fn hash_bytes(bytes: Vec) -> u64 { + unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } +} + +pub(crate) fn mapping_version() -> u32 { + unsafe { llvm::LLVMRustCoverageMappingVersion() } +} + +pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + cov_data_val: &'ll llvm::Value, +) { + let covmap_var_name = llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteMappingVarNameToString(s); + }) + .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); + debug!("covmap var name: {:?}", covmap_var_name); + + let covmap_section_name = llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); + }) + .expect("Rust Coverage section name failed UTF-8 conversion"); + debug!("covmap section name: {:?}", covmap_section_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); + llvm::set_initializer(llglobal, cov_data_val); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::set_section(llglobal, &covmap_section_name); + llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); + cx.add_used_global(llglobal); +} + +pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + func_name_hash: u64, + func_record_val: &'ll llvm::Value, + is_used: bool, +) { + // Assign a name to the function record. This is used to merge duplicates. + // + // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that + // are included-but-not-used. If (or when) Rust generates functions that are + // included-but-not-used, note that a dummy description for a function included-but-not-used + // in a Crate can be replaced by full description provided by a different Crate. The two kinds + // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by + // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. + let func_record_var_name = + format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); + debug!("function record var name: {:?}", func_record_var_name); + + let func_record_section_name = llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); + }) + .expect("Rust Coverage function record section name failed UTF-8 conversion"); + debug!("function record section name: {:?}", func_record_section_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); + llvm::set_initializer(llglobal, func_record_val); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); + llvm::set_visibility(llglobal, llvm::Visibility::Hidden); + llvm::set_section(llglobal, &func_record_section_name); + llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); + llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); + cx.add_used_global(llglobal); +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs new file mode 100644 index 00000000000..99e4ded62f1 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -0,0 +1,126 @@ +use super::metadata::file_metadata; +use super::utils::DIB; +use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext}; +use rustc_codegen_ssa::traits::*; + +use crate::common::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::{DILocation, DIScope}; +use rustc_middle::mir::{Body, SourceScope}; +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::{self, Instance}; +use rustc_session::config::DebugInfo; + +use rustc_index::bit_set::BitSet; +use rustc_index::vec::Idx; + +/// Produces DIScope DIEs for each MIR Scope which has variables defined in it. +// FIXME(eddyb) almost all of this should be in `rustc_codegen_ssa::mir::debuginfo`. +pub fn compute_mir_scopes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + mir: &Body<'tcx>, + debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, +) { + // Find all scopes with variables defined in them. + let variables = if cx.sess().opts.debuginfo == DebugInfo::Full { + let mut vars = BitSet::new_empty(mir.source_scopes.len()); + // FIXME(eddyb) take into account that arguments always have debuginfo, + // irrespective of their name (assuming full debuginfo is enabled). + // NOTE(eddyb) actually, on second thought, those are always in the + // function scope, which always exists. + for var_debug_info in &mir.var_debug_info { + vars.insert(var_debug_info.source_info.scope); + } + Some(vars) + } else { + // Nothing to emit, of course. + None + }; + let mut instantiated = BitSet::new_empty(mir.source_scopes.len()); + // Instantiate all scopes. + for idx in 0..mir.source_scopes.len() { + let scope = SourceScope::new(idx); + make_mir_scope(cx, instance, mir, &variables, debug_context, &mut instantiated, scope); + } + assert!(instantiated.count() == mir.source_scopes.len()); +} + +fn make_mir_scope<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + mir: &Body<'tcx>, + variables: &Option>, + debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, + instantiated: &mut BitSet, + scope: SourceScope, +) { + if instantiated.contains(scope) { + return; + } + + let scope_data = &mir.source_scopes[scope]; + let parent_scope = if let Some(parent) = scope_data.parent_scope { + make_mir_scope(cx, instance, mir, variables, debug_context, instantiated, parent); + debug_context.scopes[parent] + } else { + // The root is the function itself. + let loc = cx.lookup_debug_loc(mir.span.lo()); + debug_context.scopes[scope] = DebugScope { + file_start_pos: loc.file.start_pos, + file_end_pos: loc.file.end_pos, + ..debug_context.scopes[scope] + }; + instantiated.insert(scope); + return; + }; + + if let Some(vars) = variables && !vars.contains(scope) && scope_data.inlined.is_none() { + // Do not create a DIScope if there are no variables defined in this + // MIR `SourceScope`, and it's not `inlined`, to avoid debuginfo bloat. + debug_context.scopes[scope] = parent_scope; + instantiated.insert(scope); + return; + } + + let loc = cx.lookup_debug_loc(scope_data.span.lo()); + let file_metadata = file_metadata(cx, &loc.file); + + let dbg_scope = match scope_data.inlined { + Some((callee, _)) => { + // FIXME(eddyb) this would be `self.monomorphize(&callee)` + // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. + let callee = cx.tcx.subst_and_normalize_erasing_regions( + instance.substs, + ty::ParamEnv::reveal_all(), + callee, + ); + let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); + cx.dbg_scope_fn(callee, callee_fn_abi, None) + } + None => unsafe { + llvm::LLVMRustDIBuilderCreateLexicalBlock( + DIB(cx), + parent_scope.dbg_scope, + file_metadata, + loc.line, + loc.col, + ) + }, + }; + + let inlined_at = scope_data.inlined.map(|(_, callsite_span)| { + // FIXME(eddyb) this doesn't account for the macro-related + // `Span` fixups that `rustc_codegen_ssa::mir::debuginfo` does. + let callsite_scope = parent_scope.adjust_dbg_scope_for_span(cx, callsite_span); + cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span) + }); + + debug_context.scopes[scope] = DebugScope { + dbg_scope, + inlined_at: inlined_at.or(parent_scope.inlined_at), + file_start_pos: loc.file.start_pos, + file_end_pos: loc.file.end_pos, + }; + instantiated.insert(scope); +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md new file mode 100644 index 00000000000..aaec4e68c17 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md @@ -0,0 +1,131 @@ +# Debug Info Module + +This module serves the purpose of generating debug symbols. We use LLVM's +[source level debugging](https://llvm.org/docs/SourceLevelDebugging.html) +features for generating the debug information. The general principle is +this: + +Given the right metadata in the LLVM IR, the LLVM code generator is able to +create DWARF debug symbols for the given code. The +[metadata](https://llvm.org/docs/LangRef.html#metadata-type) is structured +much like DWARF *debugging information entries* (DIE), representing type +information such as datatype layout, function signatures, block layout, +variable location and scope information, etc. It is the purpose of this +module to generate correct metadata and insert it into the LLVM IR. + +As the exact format of metadata trees may change between different LLVM +versions, we now use LLVM +[DIBuilder](https://llvm.org/docs/doxygen/html/classllvm_1_1DIBuilder.html) +to create metadata where possible. This will hopefully ease the adaption of +this module to future LLVM versions. + +The public API of the module is a set of functions that will insert the +correct metadata into the LLVM IR when called with the right parameters. +The module is thus driven from an outside client with functions like +`debuginfo::create_local_var_metadata(bx: block, local: &ast::local)`. + +Internally the module will try to reuse already created metadata by +utilizing a cache. The way to get a shared metadata node when needed is +thus to just call the corresponding function in this module: +```ignore (illustrative) +let file_metadata = file_metadata(cx, file); +``` +The function will take care of probing the cache for an existing node for +that exact file path. + +All private state used by the module is stored within either the +CodegenUnitDebugContext struct (owned by the CodegenCx) or the +FunctionDebugContext (owned by the FunctionCx). + +This file consists of three conceptual sections: +1. The public interface of the module +2. Module-internal metadata creation functions +3. Minor utility functions + + +## Recursive Types + +Some kinds of types, such as structs and enums can be recursive. That means +that the type definition of some type X refers to some other type which in +turn (transitively) refers to X. This introduces cycles into the type +referral graph. A naive algorithm doing an on-demand, depth-first traversal +of this graph when describing types, can get trapped in an endless loop +when it reaches such a cycle. + +For example, the following simple type for a singly-linked list... + +``` +struct List { + value: i32, + tail: Option>, +} +``` + +will generate the following callstack with a naive DFS algorithm: + +```ignore (illustrative) +describe(t = List) + describe(t = i32) + describe(t = Option>) + describe(t = Box) + describe(t = List) // at the beginning again... + ... +``` + +To break cycles like these, we use "stubs". That is, when +the algorithm encounters a possibly recursive type (any struct or enum), it +immediately creates a type description node and inserts it into the cache +*before* describing the members of the type. This type description is just +a stub (as type members are not described and added to it yet) but it +allows the algorithm to already refer to the type. After the stub is +inserted into the cache, the algorithm continues as before. If it now +encounters a recursive reference, it will hit the cache and does not try to +describe the type anew. This behavior is encapsulated in the +`type_map::build_type_with_children()` function. + + +## Source Locations and Line Information + +In addition to data type descriptions the debugging information must also +allow to map machine code locations back to source code locations in order +to be useful. This functionality is also handled in this module. The +following functions allow to control source mappings: + ++ `set_source_location()` ++ `clear_source_location()` ++ `start_emitting_source_locations()` + +`set_source_location()` allows to set the current source location. All IR +instructions created after a call to this function will be linked to the +given source location, until another location is specified with +`set_source_location()` or the source location is cleared with +`clear_source_location()`. In the later case, subsequent IR instruction +will not be linked to any source location. As you can see, this is a +stateful API (mimicking the one in LLVM), so be careful with source +locations set by previous calls. It's probably best to not rely on any +specific state being present at a given point in code. + +One topic that deserves some extra attention is *function prologues*. At +the beginning of a function's machine code there are typically a few +instructions for loading argument values into allocas and checking if +there's enough stack space for the function to execute. This *prologue* is +not visible in the source code and LLVM puts a special PROLOGUE END marker +into the line table at the first non-prologue instruction of the function. +In order to find out where the prologue ends, LLVM looks for the first +instruction in the function body that is linked to a source location. So, +when generating prologue instructions we have to make sure that we don't +emit source location information until the 'real' function body begins. For +this reason, source location emission is disabled by default for any new +function being codegened and is only activated after a call to the third +function from the list above, `start_emitting_source_locations()`. This +function should be called right before regularly starting to codegen the +top-level block of the given function. + +There is one exception to the above rule: `llvm.dbg.declare` instruction +must be linked to the source location of the variable being declared. For +function parameters these `llvm.dbg.declare` instructions typically occur +in the middle of the prologue, however, they are ignored by LLVM's prologue +detection. The `create_argument_metadata()` and related functions take care +of linking the `llvm.dbg.declare` instructions to the correct source +locations even while source location emission is still disabled, so there +is no need to do anything special with source location handling here. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs new file mode 100644 index 00000000000..80fd9726fc7 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -0,0 +1,120 @@ +// .debug_gdb_scripts binary section. + +use crate::llvm; + +use crate::builder::Builder; +use crate::common::CodegenCx; +use crate::value::Value; +use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; +use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::bug; +use rustc_session::config::{CrateType, DebugInfo}; + +use rustc_span::symbol::sym; +use rustc_span::DebuggerVisualizerType; + +/// Inserts a side-effect free instruction sequence that makes sure that the +/// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. +pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, '_, '_>) { + if needs_gdb_debug_scripts_section(bx) { + let gdb_debug_scripts_section = + bx.const_bitcast(get_or_insert_gdb_debug_scripts_section_global(bx), bx.type_i8p()); + // Load just the first byte as that's all that's necessary to force + // LLVM to keep around the reference to the global. + let volative_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); + unsafe { + llvm::LLVMSetAlignment(volative_load_instruction, 1); + } + } +} + +/// Allocates the global variable responsible for the .debug_gdb_scripts binary +/// section. +pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Value { + let c_section_var_name = "__rustc_debug_gdb_scripts_section__\0"; + let section_var_name = &c_section_var_name[..c_section_var_name.len() - 1]; + + let section_var = + unsafe { llvm::LLVMGetNamedGlobal(cx.llmod, c_section_var_name.as_ptr().cast()) }; + + section_var.unwrap_or_else(|| { + let section_name = b".debug_gdb_scripts\0"; + let mut section_contents = Vec::new(); + + // Add the pretty printers for the standard library first. + section_contents.extend_from_slice(b"\x01gdb_load_rust_pretty_printers.py\0"); + + // Next, add the pretty printers that were specified via the `#[debugger_visualizer]` attribute. + let visualizers = collect_debugger_visualizers_transitive( + cx.tcx, + DebuggerVisualizerType::GdbPrettyPrinter, + ); + let crate_name = cx.tcx.crate_name(LOCAL_CRATE); + for (index, visualizer) in visualizers.iter().enumerate() { + // The initial byte `4` instructs GDB that the following pretty printer + // is defined inline as opposed to in a standalone file. + section_contents.extend_from_slice(b"\x04"); + let vis_name = format!("pretty-printer-{}-{}\n", crate_name, index); + section_contents.extend_from_slice(vis_name.as_bytes()); + section_contents.extend_from_slice(&visualizer.src); + + // The final byte `0` tells GDB that the pretty printer has been + // fully defined and can continue searching for additional + // pretty printers. + section_contents.extend_from_slice(b"\0"); + } + + unsafe { + let section_contents = section_contents.as_slice(); + let llvm_type = cx.type_array(cx.type_i8(), section_contents.len() as u64); + + let section_var = cx + .define_global(section_var_name, llvm_type) + .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); + llvm::LLVMSetSection(section_var, section_name.as_ptr().cast()); + llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); + llvm::LLVMSetGlobalConstant(section_var, llvm::True); + llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); + llvm::LLVMRustSetLinkage(section_var, llvm::Linkage::LinkOnceODRLinkage); + // This should make sure that the whole section is not larger than + // the string it contains. Otherwise we get a warning from GDB. + llvm::LLVMSetAlignment(section_var, 1); + section_var + } + }) +} + +pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { + let omit_gdb_pretty_printer_section = + cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); + + // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create + // ODR violations at link time, this section will not be emitted for rlibs since + // each rlib could produce a different set of visualizers that would be embedded + // in the `.debug_gdb_scripts` section. For that reason, we make sure that the + // section is only emitted for leaf crates. + let embed_visualizers = cx.sess().crate_types().iter().any(|&crate_type| match crate_type { + CrateType::Executable | CrateType::Dylib | CrateType::Cdylib | CrateType::Staticlib => { + // These are crate types for which we will embed pretty printers since they + // are treated as leaf crates. + true + } + CrateType::ProcMacro => { + // We could embed pretty printers for proc macro crates too but it does not + // seem like a good default, since this is a rare use case and we don't + // want to slow down the common case. + false + } + CrateType::Rlib => { + // As per the above description, embedding pretty printers for rlibs could + // lead to ODR violations so we skip this crate type as well. + false + } + }); + + !omit_gdb_pretty_printer_section + && cx.sess().opts.debuginfo != DebugInfo::None + && cx.sess().target.emit_debug_gdb_scripts + && embed_visualizers +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs new file mode 100644 index 00000000000..bd84100e0e8 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -0,0 +1,1618 @@ +use self::type_map::DINodeCreationResult; +use self::type_map::Stub; +use self::type_map::UniqueTypeId; + +use super::namespace::mangled_name_of_instance; +use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name}; +use super::utils::{ + create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, DIB, +}; +use super::CodegenUnitDebugContext; + +use crate::abi; +use crate::common::CodegenCx; +use crate::debuginfo::metadata::type_map::build_type_with_children; +use crate::debuginfo::utils::fat_pointer_kind; +use crate::debuginfo::utils::FatPtrKind; +use crate::llvm; +use crate::llvm::debuginfo::{ + DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, +}; +use crate::value::Value; + +use cstr::cstr; +use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo; +use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; +use rustc_codegen_ssa::traits::*; +use rustc_fs_util::path_to_c_string; +use rustc_hir::def::CtorKind; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_index::vec::{Idx, IndexVec}; +use rustc_middle::bug; +use rustc_middle::mir::{self, GeneratorLayout}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::subst::GenericArgKind; +use rustc_middle::ty::{ + self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, +}; +use rustc_session::config::{self, DebugInfo, Lto}; +use rustc_span::symbol::Symbol; +use rustc_span::FileName; +use rustc_span::{self, FileNameDisplayPreference, SourceFile}; +use rustc_symbol_mangling::typeid_for_trait_ref; +use rustc_target::abi::{Align, Size}; +use smallvec::smallvec; +use tracing::debug; + +use libc::{c_char, c_longlong, c_uint}; +use std::borrow::Cow; +use std::fmt::{self, Write}; +use std::hash::{Hash, Hasher}; +use std::iter; +use std::path::{Path, PathBuf}; +use std::ptr; +use tracing::instrument; + +impl PartialEq for llvm::Metadata { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self, other) + } +} + +impl Eq for llvm::Metadata {} + +impl Hash for llvm::Metadata { + fn hash(&self, hasher: &mut H) { + (self as *const Self).hash(hasher); + } +} + +impl fmt::Debug for llvm::Metadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (self as *const Self).fmt(f) + } +} + +// From DWARF 5. +// See http://www.dwarfstd.org/ShowIssue.php?issue=140129.1. +const DW_LANG_RUST: c_uint = 0x1c; +#[allow(non_upper_case_globals)] +const DW_ATE_boolean: c_uint = 0x02; +#[allow(non_upper_case_globals)] +const DW_ATE_float: c_uint = 0x04; +#[allow(non_upper_case_globals)] +const DW_ATE_signed: c_uint = 0x05; +#[allow(non_upper_case_globals)] +const DW_ATE_unsigned: c_uint = 0x07; +#[allow(non_upper_case_globals)] +const DW_ATE_UTF: c_uint = 0x10; + +pub(super) const UNKNOWN_LINE_NUMBER: c_uint = 0; +pub(super) const UNKNOWN_COLUMN_NUMBER: c_uint = 0; + +const NO_SCOPE_METADATA: Option<&DIScope> = None; +/// A function that returns an empty list of generic parameter debuginfo nodes. +const NO_GENERICS: for<'ll> fn(&CodegenCx<'ll, '_>) -> SmallVec<&'ll DIType> = |_| SmallVec::new(); + +// SmallVec is used quite a bit in this module, so create a shorthand. +// The actual number of elements is not so important. +pub type SmallVec = smallvec::SmallVec<[T; 16]>; + +mod enums; +mod type_map; + +pub(crate) use type_map::TypeMap; + +/// Returns from the enclosing function if the type debuginfo node with the given +/// unique ID can be found in the type map. +macro_rules! return_if_di_node_created_in_meantime { + ($cx: expr, $unique_type_id: expr) => { + if let Some(di_node) = debug_context($cx).type_map.di_node_for_unique_id($unique_type_id) { + return DINodeCreationResult::new(di_node, true); + } + }; +} + +/// Extract size and alignment from a TyAndLayout. +fn size_and_align_of<'tcx>(ty_and_layout: TyAndLayout<'tcx>) -> (Size, Align) { + (ty_and_layout.size, ty_and_layout.align.abi) +} + +/// Creates debuginfo for a fixed size array (e.g. `[u64; 123]`). +/// For slices (that is, "arrays" of unknown size) use [build_slice_type_di_node]. +fn build_fixed_size_array_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, + array_type: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { + let ty::Array(element_type, len) = array_type.kind() else { + bug!("build_fixed_size_array_di_node() called with non-ty::Array type `{:?}`", array_type) + }; + + let element_type_di_node = type_di_node(cx, *element_type); + + return_if_di_node_created_in_meantime!(cx, unique_type_id); + + let (size, align) = cx.size_and_align_of(array_type); + + let upper_bound = len.eval_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong; + + let subrange = + unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) }; + + let subscripts = create_DIArray(DIB(cx), &[subrange]); + let di_node = unsafe { + llvm::LLVMRustDIBuilderCreateArrayType( + DIB(cx), + size.bits(), + align.bits() as u32, + element_type_di_node, + subscripts, + ) + }; + + DINodeCreationResult::new(di_node, false) +} + +/// Creates debuginfo for built-in pointer-like things: +/// +/// - ty::Ref +/// - ty::RawPtr +/// - ty::Adt in the case it's Box +/// +/// At some point we might want to remove the special handling of Box +/// and treat it the same as other smart pointers (like Rc, Arc, ...). +fn build_pointer_or_reference_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ptr_type: Ty<'tcx>, + pointee_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + // The debuginfo generated by this function is only valid if `ptr_type` is really just + // a (fat) pointer. Make sure it is not called for e.g. `Box`. + debug_assert_eq!( + cx.size_and_align_of(ptr_type), + cx.size_and_align_of(cx.tcx.mk_mut_ptr(pointee_type)) + ); + + let pointee_type_di_node = type_di_node(cx, pointee_type); + + return_if_di_node_created_in_meantime!(cx, unique_type_id); + + let (thin_pointer_size, thin_pointer_align) = + cx.size_and_align_of(cx.tcx.mk_imm_ptr(cx.tcx.types.unit)); + let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); + + match fat_pointer_kind(cx, pointee_type) { + None => { + // This is a thin pointer. Create a regular pointer type and give it the correct name. + debug_assert_eq!( + (thin_pointer_size, thin_pointer_align), + cx.size_and_align_of(ptr_type), + "ptr_type={}, pointee_type={}", + ptr_type, + pointee_type, + ); + + let di_node = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + pointee_type_di_node, + thin_pointer_size.bits(), + thin_pointer_align.bits() as u32, + 0, // Ignore DWARF address space. + ptr_type_debuginfo_name.as_ptr().cast(), + ptr_type_debuginfo_name.len(), + ) + }; + + DINodeCreationResult { di_node, already_stored_in_typemap: false } + } + Some(fat_pointer_kind) => { + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &ptr_type_debuginfo_name, + cx.size_and_align_of(ptr_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |cx, owner| { + // FIXME: If this fat pointer is a `Box` then we don't want to use its + // type layout and instead use the layout of the raw pointer inside + // of it. + // The proper way to handle this is to not treat Box as a pointer + // at all and instead emit regular struct debuginfo for it. We just + // need to make sure that we don't break existing debuginfo consumers + // by doing that (at least not without a warning period). + let layout_type = + if ptr_type.is_box() { cx.tcx.mk_mut_ptr(pointee_type) } else { ptr_type }; + + let layout = cx.layout_of(layout_type); + let addr_field = layout.field(cx, abi::FAT_PTR_ADDR); + let extra_field = layout.field(cx, abi::FAT_PTR_EXTRA); + + let (addr_field_name, extra_field_name) = match fat_pointer_kind { + FatPtrKind::Dyn => ("pointer", "vtable"), + FatPtrKind::Slice => ("data_ptr", "length"), + }; + + debug_assert_eq!(abi::FAT_PTR_ADDR, 0); + debug_assert_eq!(abi::FAT_PTR_EXTRA, 1); + + // The data pointer type is a regular, thin pointer, regardless of whether this + // is a slice or a trait object. + let data_ptr_type_di_node = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + pointee_type_di_node, + addr_field.size.bits(), + addr_field.align.abi.bits() as u32, + 0, // Ignore DWARF address space. + std::ptr::null(), + 0, + ) + }; + + smallvec![ + build_field_di_node( + cx, + owner, + addr_field_name, + (addr_field.size, addr_field.align.abi), + layout.fields.offset(abi::FAT_PTR_ADDR), + DIFlags::FlagZero, + data_ptr_type_di_node, + ), + build_field_di_node( + cx, + owner, + extra_field_name, + (extra_field.size, extra_field.align.abi), + layout.fields.offset(abi::FAT_PTR_EXTRA), + DIFlags::FlagZero, + type_di_node(cx, extra_field.ty), + ), + ] + }, + NO_GENERICS, + ) + } + } +} + +fn build_subroutine_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + // It's possible to create a self-referential + // type in Rust by using 'impl trait': + // + // fn foo() -> impl Copy { foo } + // + // Unfortunately LLVM's API does not allow us to create recursive subroutine types. + // In order to work around that restriction we place a marker type in the type map, + // before creating the actual type. If the actual type is recursive, it will hit the + // marker type. So we end up with a type that looks like + // + // fn foo() -> + // + // Once that is created, we replace the marker in the typemap with the actual type. + debug_context(cx) + .type_map + .unique_id_to_di_node + .borrow_mut() + .insert(unique_type_id, recursion_marker_type_di_node(cx)); + + let fn_ty = unique_type_id.expect_ty(); + let signature = cx + .tcx + .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), fn_ty.fn_sig(cx.tcx)); + + let signature_di_nodes: SmallVec<_> = iter::once( + // return type + match signature.output().kind() { + ty::Tuple(tys) if tys.is_empty() => { + // this is a "void" function + None + } + _ => Some(type_di_node(cx, signature.output())), + }, + ) + .chain( + // regular arguments + signature.inputs().iter().map(|&argument_type| Some(type_di_node(cx, argument_type))), + ) + .collect(); + + debug_context(cx).type_map.unique_id_to_di_node.borrow_mut().remove(&unique_type_id); + + let fn_di_node = unsafe { + llvm::LLVMRustDIBuilderCreateSubroutineType( + DIB(cx), + create_DIArray(DIB(cx), &signature_di_nodes[..]), + ) + }; + + // This is actually a function pointer, so wrap it in pointer DI. + let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); + let di_node = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + fn_di_node, + cx.tcx.data_layout.pointer_size.bits(), + cx.tcx.data_layout.pointer_align.abi.bits() as u32, + 0, // Ignore DWARF address space. + name.as_ptr().cast(), + name.len(), + ) + }; + + DINodeCreationResult::new(di_node, false) +} + +/// Create debuginfo for `dyn SomeTrait` types. Currently these are empty structs +/// we with the correct type name (e.g. "dyn SomeTrait + Sync"). +fn build_dyn_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + dyn_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + if let ty::Dynamic(..) = dyn_type.kind() { + let type_name = compute_debuginfo_type_name(cx.tcx, dyn_type, true); + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + cx.size_and_align_of(dyn_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |_, _| smallvec![], + NO_GENERICS, + ) + } else { + bug!( + "Only ty::Dynamic is valid for build_dyn_type_di_node(). Found {:?} instead.", + dyn_type + ) + } +} + +/// Create debuginfo for `[T]` and `str`. These are unsized. +/// +/// NOTE: We currently emit just emit the debuginfo for the element type here +/// (i.e. `T` for slices and `u8` for `str`), so that we end up with +/// `*const T` for the `data_ptr` field of the corresponding fat-pointer +/// debuginfo of `&[T]`. +/// +/// It would be preferable and more accurate if we emitted a DIArray of T +/// without an upper bound instead. That is, LLVM already supports emitting +/// debuginfo of arrays of unknown size. But GDB currently seems to end up +/// in an infinite loop when confronted with such a type. +/// +/// As a side effect of the current encoding every instance of a type like +/// `struct Foo { unsized_field: [u8] }` will look like +/// `struct Foo { unsized_field: u8 }` in debuginfo. If the length of the +/// slice is zero, then accessing `unsized_field` in the debugger would +/// result in an out-of-bounds access. +fn build_slice_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + slice_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let element_type = match slice_type.kind() { + ty::Slice(element_type) => *element_type, + ty::Str => cx.tcx.types.u8, + _ => { + bug!( + "Only ty::Slice is valid for build_slice_type_di_node(). Found {:?} instead.", + slice_type + ) + } + }; + + let element_type_di_node = type_di_node(cx, element_type); + return_if_di_node_created_in_meantime!(cx, unique_type_id); + DINodeCreationResult { di_node: element_type_di_node, already_stored_in_typemap: false } +} + +/// Get the debuginfo node for the given type. +/// +/// This function will look up the debuginfo node in the TypeMap. If it can't find it, it +/// will create the node by dispatching to the corresponding `build_*_di_node()` function. +pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { + let unique_type_id = UniqueTypeId::for_ty(cx.tcx, t); + + if let Some(existing_di_node) = debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) + { + return existing_di_node; + } + + debug!("type_di_node: {:?}", t); + + let DINodeCreationResult { di_node, already_stored_in_typemap } = match *t.kind() { + ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { + build_basic_type_di_node(cx, t) + } + ty::Tuple(elements) if elements.is_empty() => build_basic_type_di_node(cx, t), + ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t), + ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id), + ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id), + ty::Foreign(..) => build_foreign_type_di_node(cx, t, unique_type_id), + ty::RawPtr(ty::TypeAndMut { ty: pointee_type, .. }) | ty::Ref(_, pointee_type, _) => { + build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id) + } + // Box may have a non-ZST allocator A. In that case, we + // cannot treat Box as just an owned alias of `*mut T`. + ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { + build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) + } + ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), + ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), + ty::Generator(..) => enums::build_generator_di_node(cx, unique_type_id), + ty::Adt(def, ..) => match def.adt_kind() { + AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), + AdtKind::Union => build_union_type_di_node(cx, unique_type_id), + AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id), + }, + ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), + // Type parameters from polymorphized functions. + ty::Param(_) => build_param_type_di_node(cx, t), + _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t), + }; + + { + if already_stored_in_typemap { + // Make sure that we really do have a `TypeMap` entry for the unique type ID. + let di_node_for_uid = + match debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) { + Some(di_node) => di_node, + None => { + bug!( + "expected type debuginfo node for unique \ + type ID '{:?}' to already be in \ + the `debuginfo::TypeMap` but it \ + was not.", + unique_type_id, + ); + } + }; + + debug_assert_eq!(di_node_for_uid as *const _, di_node as *const _); + } else { + debug_context(cx).type_map.insert(unique_type_id, di_node); + } + } + + di_node +} + +// FIXME(mw): Cache this via a regular UniqueTypeId instead of an extra field in the debug context. +fn recursion_marker_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll DIType { + *debug_context(cx).recursion_marker_type.get_or_init(move || { + unsafe { + // The choice of type here is pretty arbitrary - + // anything reading the debuginfo for a recursive + // type is going to see *something* weird - the only + // question is what exactly it will see. + // + // FIXME: the name `` does not fit the naming scheme + // of other types. + // + // FIXME: it might make sense to use an actual pointer type here + // so that debuggers can show the address. + let name = ""; + llvm::LLVMRustDIBuilderCreateBasicType( + DIB(cx), + name.as_ptr().cast(), + name.len(), + cx.tcx.data_layout.pointer_size.bits(), + DW_ATE_unsigned, + ) + } + }) +} + +fn hex_encode(data: &[u8]) -> String { + let mut hex_string = String::with_capacity(data.len() * 2); + for byte in data.iter() { + write!(&mut hex_string, "{:02x}", byte).unwrap(); + } + hex_string +} + +pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { + let cache_key = Some((source_file.name_hash, source_file.src_hash)); + return debug_context(cx) + .created_files + .borrow_mut() + .entry(cache_key) + .or_insert_with(|| alloc_new_file_metadata(cx, source_file)); + + #[instrument(skip(cx, source_file), level = "debug")] + fn alloc_new_file_metadata<'ll>( + cx: &CodegenCx<'ll, '_>, + source_file: &SourceFile, + ) -> &'ll DIFile { + debug!(?source_file.name); + + let (directory, file_name) = match &source_file.name { + FileName::Real(filename) => { + let working_directory = &cx.sess().opts.working_dir; + debug!(?working_directory); + + let filename = cx + .sess() + .source_map() + .path_mapping() + .to_embeddable_absolute_path(filename.clone(), working_directory); + + // Construct the absolute path of the file + let abs_path = filename.remapped_path_if_available(); + debug!(?abs_path); + + if let Ok(rel_path) = + abs_path.strip_prefix(working_directory.remapped_path_if_available()) + { + // If the compiler's working directory (which also is the DW_AT_comp_dir of + // the compilation unit) is a prefix of the path we are about to emit, then + // only emit the part relative to the working directory. + // Because of path remapping we sometimes see strange things here: `abs_path` + // might actually look like a relative path + // (e.g. `/src/lib.rs`), so if we emit it without + // taking the working directory into account, downstream tooling will + // interpret it as `//src/lib.rs`, + // which makes no sense. Usually in such cases the working directory will also + // be remapped to `` or some other prefix of the path + // we are remapping, so we end up with + // `//src/lib.rs`. + // By moving the working directory portion into the `directory` part of the + // DIFile, we allow LLVM to emit just the relative path for DWARF, while + // still emitting the correct absolute path for CodeView. + ( + working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), abs_path.to_string_lossy().into_owned()) + } + } + other => ("".into(), other.prefer_remapped().to_string_lossy().into_owned()), + }; + + let hash_kind = match source_file.src_hash.kind { + rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5, + rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1, + rustc_span::SourceFileHashAlgorithm::Sha256 => llvm::ChecksumKind::SHA256, + }; + let hash_value = hex_encode(source_file.src_hash.hash_bytes()); + + unsafe { + llvm::LLVMRustDIBuilderCreateFile( + DIB(cx), + file_name.as_ptr().cast(), + file_name.len(), + directory.as_ptr().cast(), + directory.len(), + hash_kind, + hash_value.as_ptr().cast(), + hash_value.len(), + ) + } + } +} + +pub fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { + debug_context(cx).created_files.borrow_mut().entry(None).or_insert_with(|| unsafe { + let file_name = ""; + let directory = ""; + let hash_value = ""; + + llvm::LLVMRustDIBuilderCreateFile( + DIB(cx), + file_name.as_ptr().cast(), + file_name.len(), + directory.as_ptr().cast(), + directory.len(), + llvm::ChecksumKind::None, + hash_value.as_ptr().cast(), + hash_value.len(), + ) + }) +} + +trait MsvcBasicName { + fn msvc_basic_name(self) -> &'static str; +} + +impl MsvcBasicName for ty::IntTy { + fn msvc_basic_name(self) -> &'static str { + match self { + ty::IntTy::Isize => "ptrdiff_t", + ty::IntTy::I8 => "__int8", + ty::IntTy::I16 => "__int16", + ty::IntTy::I32 => "__int32", + ty::IntTy::I64 => "__int64", + ty::IntTy::I128 => "__int128", + } + } +} + +impl MsvcBasicName for ty::UintTy { + fn msvc_basic_name(self) -> &'static str { + match self { + ty::UintTy::Usize => "size_t", + ty::UintTy::U8 => "unsigned __int8", + ty::UintTy::U16 => "unsigned __int16", + ty::UintTy::U32 => "unsigned __int32", + ty::UintTy::U64 => "unsigned __int64", + ty::UintTy::U128 => "unsigned __int128", + } + } +} + +impl MsvcBasicName for ty::FloatTy { + fn msvc_basic_name(self) -> &'static str { + match self { + ty::FloatTy::F32 => "float", + ty::FloatTy::F64 => "double", + } + } +} + +fn build_basic_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { + debug!("build_basic_type_di_node: {:?}", t); + + // When targeting MSVC, emit MSVC style type names for compatibility with + // .natvis visualizers (and perhaps other existing native debuggers?) + let cpp_like_debuginfo = cpp_like_debuginfo(cx.tcx); + + let (name, encoding) = match t.kind() { + ty::Never => ("!", DW_ATE_unsigned), + ty::Tuple(elements) if elements.is_empty() => { + if cpp_like_debuginfo { + return build_tuple_type_di_node(cx, UniqueTypeId::for_ty(cx.tcx, t)); + } else { + ("()", DW_ATE_unsigned) + } + } + ty::Bool => ("bool", DW_ATE_boolean), + ty::Char => ("char", DW_ATE_UTF), + ty::Int(int_ty) if cpp_like_debuginfo => (int_ty.msvc_basic_name(), DW_ATE_signed), + ty::Uint(uint_ty) if cpp_like_debuginfo => (uint_ty.msvc_basic_name(), DW_ATE_unsigned), + ty::Float(float_ty) if cpp_like_debuginfo => (float_ty.msvc_basic_name(), DW_ATE_float), + ty::Int(int_ty) => (int_ty.name_str(), DW_ATE_signed), + ty::Uint(uint_ty) => (uint_ty.name_str(), DW_ATE_unsigned), + ty::Float(float_ty) => (float_ty.name_str(), DW_ATE_float), + _ => bug!("debuginfo::build_basic_type_di_node - `t` is invalid type"), + }; + + let ty_di_node = unsafe { + llvm::LLVMRustDIBuilderCreateBasicType( + DIB(cx), + name.as_ptr().cast(), + name.len(), + cx.size_of(t).bits(), + encoding, + ) + }; + + if !cpp_like_debuginfo { + return DINodeCreationResult::new(ty_di_node, false); + } + + let typedef_name = match t.kind() { + ty::Int(int_ty) => int_ty.name_str(), + ty::Uint(uint_ty) => uint_ty.name_str(), + ty::Float(float_ty) => float_ty.name_str(), + _ => return DINodeCreationResult::new(ty_di_node, false), + }; + + let typedef_di_node = unsafe { + llvm::LLVMRustDIBuilderCreateTypedef( + DIB(cx), + ty_di_node, + typedef_name.as_ptr().cast(), + typedef_name.len(), + unknown_file_metadata(cx), + 0, + None, + ) + }; + + DINodeCreationResult::new(typedef_di_node, false) +} + +fn build_foreign_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + debug!("build_foreign_type_di_node: {:?}", t); + + let &ty::Foreign(def_id) = unique_type_id.expect_ty().kind() else { + bug!("build_foreign_type_di_node() called with unexpected type: {:?}", unique_type_id.expect_ty()); + }; + + build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &compute_debuginfo_type_name(cx.tcx, t, false), + cx.size_and_align_of(t), + Some(get_namespace_for_item(cx, def_id)), + DIFlags::FlagZero, + ), + |_, _| smallvec![], + NO_GENERICS, + ) +} + +fn build_param_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { + debug!("build_param_type_di_node: {:?}", t); + let name = format!("{:?}", t); + DINodeCreationResult { + di_node: unsafe { + llvm::LLVMRustDIBuilderCreateBasicType( + DIB(cx), + name.as_ptr().cast(), + name.len(), + Size::ZERO.bits(), + DW_ATE_unsigned, + ) + }, + already_stored_in_typemap: false, + } +} + +pub fn build_compile_unit_di_node<'ll, 'tcx>( + tcx: TyCtxt<'tcx>, + codegen_unit_name: &str, + debug_context: &CodegenUnitDebugContext<'ll, 'tcx>, +) -> &'ll DIDescriptor { + let mut name_in_debuginfo = match tcx.sess.local_crate_source_file { + Some(ref path) => path.clone(), + None => PathBuf::from(tcx.crate_name(LOCAL_CRATE).as_str()), + }; + + // To avoid breaking split DWARF, we need to ensure that each codegen unit + // has a unique `DW_AT_name`. This is because there's a remote chance that + // different codegen units for the same module will have entirely + // identical DWARF entries for the purpose of the DWO ID, which would + // violate Appendix F ("Split Dwarf Object Files") of the DWARF 5 + // specification. LLVM uses the algorithm specified in section 7.32 "Type + // Signature Computation" to compute the DWO ID, which does not include + // any fields that would distinguish compilation units. So we must embed + // the codegen unit name into the `DW_AT_name`. (Issue #88521.) + // + // Additionally, the OSX linker has an idiosyncrasy where it will ignore + // some debuginfo if multiple object files with the same `DW_AT_name` are + // linked together. + // + // As a workaround for these two issues, we generate unique names for each + // object file. Those do not correspond to an actual source file but that + // is harmless. + name_in_debuginfo.push("@"); + name_in_debuginfo.push(codegen_unit_name); + + debug!("build_compile_unit_di_node: {:?}", name_in_debuginfo); + let rustc_producer = + format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"),); + // FIXME(#41252) Remove "clang LLVM" if we can get GDB and LLVM to play nice. + let producer = format!("clang LLVM ({})", rustc_producer); + + let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); + let work_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped); + let flags = "\0"; + let output_filenames = tcx.output_filenames(()); + let split_name = if tcx.sess.target_can_use_split_dwarf() { + output_filenames + .split_dwarf_path( + tcx.sess.split_debuginfo(), + tcx.sess.opts.unstable_opts.split_dwarf_kind, + Some(codegen_unit_name), + ) + // We get a path relative to the working directory from split_dwarf_path + .map(|f| tcx.sess.source_map().path_mapping().map_prefix(f).0) + } else { + None + } + .unwrap_or_default(); + let split_name = split_name.to_str().unwrap(); + + // FIXME(#60020): + // + // This should actually be + // + // let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); + // + // That is, we should set LLVM's emission kind to `LineTablesOnly` if + // we are compiling with "limited" debuginfo. However, some of the + // existing tools relied on slightly more debuginfo being generated than + // would be the case with `LineTablesOnly`, and we did not want to break + // these tools in a "drive-by fix", without a good idea or plan about + // what limited debuginfo should exactly look like. So for now we keep + // the emission kind as `FullDebug`. + // + // See https://github.com/rust-lang/rust/issues/60020 for details. + let kind = DebugEmissionKind::FullDebug; + assert!(tcx.sess.opts.debuginfo != DebugInfo::None); + + unsafe { + let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( + debug_context.builder, + name_in_debuginfo.as_ptr().cast(), + name_in_debuginfo.len(), + work_dir.as_ptr().cast(), + work_dir.len(), + llvm::ChecksumKind::None, + ptr::null(), + 0, + ); + + let unit_metadata = llvm::LLVMRustDIBuilderCreateCompileUnit( + debug_context.builder, + DW_LANG_RUST, + compile_unit_file, + producer.as_ptr().cast(), + producer.len(), + tcx.sess.opts.optimize != config::OptLevel::No, + flags.as_ptr().cast(), + 0, + // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead + // put the path supplied to `MCSplitDwarfFile` into the debug info of the final + // output(s). + split_name.as_ptr().cast(), + split_name.len(), + kind, + 0, + tcx.sess.opts.unstable_opts.split_dwarf_inlining, + ); + + if tcx.sess.opts.unstable_opts.profile { + let cu_desc_metadata = + llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata); + let default_gcda_path = &output_filenames.with_extension("gcda"); + let gcda_path = + tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); + + let gcov_cu_info = [ + path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), + path_to_mdstring(debug_context.llcontext, gcda_path), + cu_desc_metadata, + ]; + let gcov_metadata = llvm::LLVMMDNodeInContext( + debug_context.llcontext, + gcov_cu_info.as_ptr(), + gcov_cu_info.len() as c_uint, + ); + + let llvm_gcov_ident = cstr!("llvm.gcov"); + llvm::LLVMAddNamedMetadataOperand( + debug_context.llmod, + llvm_gcov_ident.as_ptr(), + gcov_metadata, + ); + } + + // Insert `llvm.ident` metadata on the wasm targets since that will + // get hooked up to the "producer" sections `processed-by` information. + if tcx.sess.target.is_like_wasm { + let name_metadata = llvm::LLVMMDStringInContext( + debug_context.llcontext, + rustc_producer.as_ptr().cast(), + rustc_producer.as_bytes().len() as c_uint, + ); + llvm::LLVMAddNamedMetadataOperand( + debug_context.llmod, + cstr!("llvm.ident").as_ptr(), + llvm::LLVMMDNodeInContext(debug_context.llcontext, &name_metadata, 1), + ); + } + + return unit_metadata; + }; + + fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll Value { + let path_str = path_to_c_string(path); + unsafe { + llvm::LLVMMDStringInContext( + llcx, + path_str.as_ptr(), + path_str.as_bytes().len() as c_uint, + ) + } + } +} + +/// Creates a `DW_TAG_member` entry inside the DIE represented by the given `type_di_node`. +fn build_field_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + owner: &'ll DIScope, + name: &str, + size_and_align: (Size, Align), + offset: Size, + flags: DIFlags, + type_di_node: &'ll DIType, +) -> &'ll DIType { + unsafe { + llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + owner, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size_and_align.0.bits(), + size_and_align.1.bits() as u32, + offset.bits(), + flags, + type_di_node, + ) + } +} + +/// Creates the debuginfo node for a Rust struct type. Maybe be a regular struct or a tuple-struct. +fn build_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let struct_type = unique_type_id.expect_ty(); + let ty::Adt(adt_def, _) = struct_type.kind() else { + bug!("build_struct_type_di_node() called with non-struct-type: {:?}", struct_type); + }; + debug_assert!(adt_def.is_struct()); + let containing_scope = get_namespace_for_item(cx, adt_def.did()); + let struct_type_and_layout = cx.layout_of(struct_type); + let variant_def = adt_def.non_enum_variant(); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &compute_debuginfo_type_name(cx.tcx, struct_type, false), + size_and_align_of(struct_type_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| { + variant_def + .fields + .iter() + .enumerate() + .map(|(i, f)| { + let field_name = if variant_def.ctor_kind == CtorKind::Fn { + // This is a tuple struct + tuple_field_name(i) + } else { + // This is struct with named fields + Cow::Borrowed(f.name.as_str()) + }; + let field_layout = struct_type_and_layout.field(cx, i); + build_field_di_node( + cx, + owner, + &field_name[..], + (field_layout.size, field_layout.align.abi), + struct_type_and_layout.fields.offset(i), + DIFlags::FlagZero, + type_di_node(cx, field_layout.ty), + ) + }) + .collect() + }, + |cx| build_generic_type_param_di_nodes(cx, struct_type), + ) +} + +//=----------------------------------------------------------------------------- +// Tuples +//=----------------------------------------------------------------------------- + +/// Returns names of captured upvars for closures and generators. +/// +/// Here are some examples: +/// - `name__field1__field2` when the upvar is captured by value. +/// - `_ref__name__field` when the upvar is captured by reference. +/// +/// For generators this only contains upvars that are shared by all states. +fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> SmallVec { + let body = tcx.optimized_mir(def_id); + + body.var_debug_info + .iter() + .filter_map(|var| { + let is_ref = match var.value { + mir::VarDebugInfoContents::Place(place) if place.local == mir::Local::new(1) => { + // The projection is either `[.., Field, Deref]` or `[.., Field]`. It + // implies whether the variable is captured by value or by reference. + matches!(place.projection.last().unwrap(), mir::ProjectionElem::Deref) + } + _ => return None, + }; + let prefix = if is_ref { "_ref__" } else { "" }; + Some(prefix.to_owned() + var.name.as_str()) + }) + .collect() +} + +/// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or generator. +/// For a generator, this will handle upvars shared by all states. +fn build_upvar_field_di_nodes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + closure_or_generator_ty: Ty<'tcx>, + closure_or_generator_di_node: &'ll DIType, +) -> SmallVec<&'ll DIType> { + let (&def_id, up_var_tys) = match closure_or_generator_ty.kind() { + ty::Generator(def_id, substs, _) => { + let upvar_tys: SmallVec<_> = substs.as_generator().prefix_tys().collect(); + (def_id, upvar_tys) + } + ty::Closure(def_id, substs) => { + let upvar_tys: SmallVec<_> = substs.as_closure().upvar_tys().collect(); + (def_id, upvar_tys) + } + _ => { + bug!( + "build_upvar_field_di_nodes() called with non-closure-or-generator-type: {:?}", + closure_or_generator_ty + ) + } + }; + + debug_assert!( + up_var_tys + .iter() + .all(|&t| t == cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) + ); + + let capture_names = closure_saved_names_of_captured_variables(cx.tcx, def_id); + let layout = cx.layout_of(closure_or_generator_ty); + + up_var_tys + .into_iter() + .zip(capture_names.iter()) + .enumerate() + .map(|(index, (up_var_ty, capture_name))| { + build_field_di_node( + cx, + closure_or_generator_di_node, + capture_name, + cx.size_and_align_of(up_var_ty), + layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, up_var_ty), + ) + }) + .collect() +} + +/// Builds the DW_TAG_structure_type debuginfo node for a Rust tuple type. +fn build_tuple_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let tuple_type = unique_type_id.expect_ty(); + let &ty::Tuple(component_types) = tuple_type.kind() else { + bug!("build_tuple_type_di_node() called with non-tuple-type: {:?}", tuple_type) + }; + + let tuple_type_and_layout = cx.layout_of(tuple_type); + let type_name = compute_debuginfo_type_name(cx.tcx, tuple_type, false); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + size_and_align_of(tuple_type_and_layout), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + // Fields: + |cx, tuple_di_node| { + component_types + .into_iter() + .enumerate() + .map(|(index, component_type)| { + build_field_di_node( + cx, + tuple_di_node, + &tuple_field_name(index), + cx.size_and_align_of(component_type), + tuple_type_and_layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, component_type), + ) + }) + .collect() + }, + NO_GENERICS, + ) +} + +/// Builds the debuginfo node for a closure environment. +fn build_closure_env_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let closure_env_type = unique_type_id.expect_ty(); + let &ty::Closure(def_id, _substs) = closure_env_type.kind() else { + bug!("build_closure_env_di_node() called with non-closure-type: {:?}", closure_env_type) + }; + let containing_scope = get_namespace_for_item(cx, def_id); + let type_name = compute_debuginfo_type_name(cx.tcx, closure_env_type, false); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + cx.size_and_align_of(closure_env_type), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| build_upvar_field_di_nodes(cx, closure_env_type, owner), + NO_GENERICS, + ) +} + +/// Build the debuginfo node for a Rust `union` type. +fn build_union_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let union_type = unique_type_id.expect_ty(); + let (union_def_id, variant_def) = match union_type.kind() { + ty::Adt(def, _) => (def.did(), def.non_enum_variant()), + _ => bug!("build_union_type_di_node on a non-ADT"), + }; + let containing_scope = get_namespace_for_item(cx, union_def_id); + let union_ty_and_layout = cx.layout_of(union_type); + let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Union, + unique_type_id, + &type_name, + size_and_align_of(union_ty_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| { + variant_def + .fields + .iter() + .enumerate() + .map(|(i, f)| { + let field_layout = union_ty_and_layout.field(cx, i); + build_field_di_node( + cx, + owner, + f.name.as_str(), + size_and_align_of(field_layout), + Size::ZERO, + DIFlags::FlagZero, + type_di_node(cx, field_layout.ty), + ) + }) + .collect() + }, + // Generics: + |cx| build_generic_type_param_di_nodes(cx, union_type), + ) +} + +// FIXME(eddyb) maybe precompute this? Right now it's computed once +// per generator monomorphization, but it doesn't depend on substs. +fn generator_layout_and_saved_local_names<'tcx>( + tcx: TyCtxt<'tcx>, + def_id: DefId, +) -> (&'tcx GeneratorLayout<'tcx>, IndexVec>) { + let body = tcx.optimized_mir(def_id); + let generator_layout = body.generator_layout().unwrap(); + let mut generator_saved_local_names = IndexVec::from_elem(None, &generator_layout.field_tys); + + let state_arg = mir::Local::new(1); + for var in &body.var_debug_info { + let mir::VarDebugInfoContents::Place(place) = &var.value else { continue }; + if place.local != state_arg { + continue; + } + match place.projection[..] { + [ + // Deref of the `Pin<&mut Self>` state argument. + mir::ProjectionElem::Field(..), + mir::ProjectionElem::Deref, + // Field of a variant of the state. + mir::ProjectionElem::Downcast(_, variant), + mir::ProjectionElem::Field(field, _), + ] => { + let name = &mut generator_saved_local_names + [generator_layout.variant_fields[variant][field]]; + if name.is_none() { + name.replace(var.name); + } + } + _ => {} + } + } + (generator_layout, generator_saved_local_names) +} + +/// Computes the type parameters for a type, if any, for the given metadata. +fn build_generic_type_param_di_nodes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, +) -> SmallVec<&'ll DIType> { + if let ty::Adt(def, substs) = *ty.kind() { + if substs.types().next().is_some() { + let generics = cx.tcx.generics_of(def.did()); + let names = get_parameter_names(cx, generics); + let template_params: SmallVec<_> = iter::zip(substs, names) + .filter_map(|(kind, name)| { + if let GenericArgKind::Type(ty) = kind.unpack() { + let actual_type = + cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); + let actual_type_di_node = type_di_node(cx, actual_type); + let name = name.as_str(); + Some(unsafe { + llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( + DIB(cx), + None, + name.as_ptr().cast(), + name.len(), + actual_type_di_node, + ) + }) + } else { + None + } + }) + .collect(); + + return template_params; + } + } + + return smallvec![]; + + fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec { + let mut names = generics + .parent + .map_or_else(Vec::new, |def_id| get_parameter_names(cx, cx.tcx.generics_of(def_id))); + names.extend(generics.params.iter().map(|param| param.name)); + names + } +} + +/// Creates debug information for the given global variable. +/// +/// Adds the created debuginfo nodes directly to the crate's IR. +pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, global: &'ll Value) { + if cx.dbg_cx.is_none() { + return; + } + + // Only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo != DebugInfo::Full { + return; + } + + let tcx = cx.tcx; + + // We may want to remove the namespace scope if we're in an extern block (see + // https://github.com/rust-lang/rust/pull/46457#issuecomment-351750952). + let var_scope = get_namespace_for_item(cx, def_id); + let span = tcx.def_span(def_id); + + let (file_metadata, line_number) = if !span.is_dummy() { + let loc = cx.lookup_debug_loc(span.lo()); + (file_metadata(cx, &loc.file), loc.line) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + + let is_local_to_unit = is_node_local_to_unit(cx, def_id); + let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all()); + let type_di_node = type_di_node(cx, variable_type); + let var_name = tcx.item_name(def_id); + let var_name = var_name.as_str(); + let linkage_name = mangled_name_of_instance(cx, Instance::mono(tcx, def_id)).name; + // When empty, linkage_name field is omitted, + // which is what we want for no_mangle statics + let linkage_name = if var_name == linkage_name { "" } else { linkage_name }; + + let global_align = cx.align_of(variable_type); + + unsafe { + llvm::LLVMRustDIBuilderCreateStaticVariable( + DIB(cx), + Some(var_scope), + var_name.as_ptr().cast(), + var_name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + file_metadata, + line_number, + type_di_node, + is_local_to_unit, + global, + None, + global_align.bits() as u32, + ); + } +} + +/// Generates LLVM debuginfo for a vtable. +/// +/// The vtable type looks like a struct with a field for each function pointer and super-trait +/// pointer it contains (plus the `size` and `align` fields). +/// +/// Except for `size`, `align`, and `drop_in_place`, the field names don't try to mirror +/// the name of the method they implement. This can be implemented in the future once there +/// is a proper disambiguation scheme for dealing with methods from different traits that have +/// the same name. +fn build_vtable_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + poly_trait_ref: Option>, +) -> &'ll DIType { + let tcx = cx.tcx; + + let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref { + let trait_ref = poly_trait_ref.with_self_ty(tcx, ty); + let trait_ref = tcx.erase_regions(trait_ref); + + tcx.vtable_entries(trait_ref) + } else { + TyCtxt::COMMON_VTABLE_ENTRIES + }; + + // All function pointers are described as opaque pointers. This could be improved in the future + // by describing them as actual function pointers. + let void_pointer_ty = tcx.mk_imm_ptr(tcx.types.unit); + let void_pointer_type_di_node = type_di_node(cx, void_pointer_ty); + let usize_di_node = type_di_node(cx, tcx.types.usize); + let (pointer_size, pointer_align) = cx.size_and_align_of(void_pointer_ty); + // If `usize` is not pointer-sized and -aligned then the size and alignment computations + // for the vtable as a whole would be wrong. Let's make sure this holds even on weird + // platforms. + assert_eq!(cx.size_and_align_of(tcx.types.usize), (pointer_size, pointer_align)); + + let vtable_type_name = + compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::Type); + let unique_type_id = UniqueTypeId::for_vtable_ty(tcx, ty, poly_trait_ref); + let size = pointer_size * vtable_entries.len() as u64; + + // This gets mapped to a DW_AT_containing_type attribute which allows GDB to correlate + // the vtable to the type it is for. + let vtable_holder = type_di_node(cx, ty); + + build_type_with_children( + cx, + type_map::stub( + cx, + Stub::VTableTy { vtable_holder }, + unique_type_id, + &vtable_type_name, + (size, pointer_align), + NO_SCOPE_METADATA, + DIFlags::FlagArtificial, + ), + |cx, vtable_type_di_node| { + vtable_entries + .iter() + .enumerate() + .filter_map(|(index, vtable_entry)| { + let (field_name, field_type_di_node) = match vtable_entry { + ty::VtblEntry::MetadataDropInPlace => { + ("drop_in_place".to_string(), void_pointer_type_di_node) + } + ty::VtblEntry::Method(_) => { + // Note: This code does not try to give a proper name to each method + // because their might be multiple methods with the same name + // (coming from different traits). + (format!("__method{}", index), void_pointer_type_di_node) + } + ty::VtblEntry::TraitVPtr(_) => { + (format!("__super_trait_ptr{}", index), void_pointer_type_di_node) + } + ty::VtblEntry::MetadataAlign => ("align".to_string(), usize_di_node), + ty::VtblEntry::MetadataSize => ("size".to_string(), usize_di_node), + ty::VtblEntry::Vacant => return None, + }; + + let field_offset = pointer_size * index as u64; + + Some(build_field_di_node( + cx, + vtable_type_di_node, + &field_name, + (pointer_size, pointer_align), + field_offset, + DIFlags::FlagZero, + field_type_di_node, + )) + }) + .collect() + }, + NO_GENERICS, + ) + .di_node +} + +fn vcall_visibility_metadata<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + trait_ref: Option>, + vtable: &'ll Value, +) { + enum VCallVisibility { + Public = 0, + LinkageUnit = 1, + TranslationUnit = 2, + } + + let Some(trait_ref) = trait_ref else { return }; + + let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); + let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); + let trait_def_id = trait_ref_self.def_id(); + let trait_vis = cx.tcx.visibility(trait_def_id); + + let cgus = cx.sess().codegen_units(); + let single_cgu = cgus == 1; + + let lto = cx.sess().lto(); + + // Since LLVM requires full LTO for the virtual function elimination optimization to apply, + // only the `Lto::Fat` cases are relevant currently. + let vcall_visibility = match (lto, trait_vis, single_cgu) { + // If there is not LTO and the visibility in public, we have to assume that the vtable can + // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. + (Lto::No | Lto::ThinLocal, Visibility::Public, _) + | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => { + VCallVisibility::Public + } + // With LTO and a quasi-public visibility, the usages of the functions of the vtable are + // all known by the `LinkageUnit`. + // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also + // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. + (Lto::Fat | Lto::Thin, Visibility::Public, _) + | ( + Lto::ThinLocal | Lto::Thin | Lto::Fat, + Visibility::Restricted(_) | Visibility::Invisible, + false, + ) => VCallVisibility::LinkageUnit, + // If there is only one CGU, private vtables can only be seen by that CGU/translation unit + // and therefore we know of all usages of functions in the vtable. + (_, Visibility::Restricted(_) | Visibility::Invisible, true) => { + VCallVisibility::TranslationUnit + } + }; + + let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); + + unsafe { + let typeid = llvm::LLVMMDStringInContext( + cx.llcx, + trait_ref_typeid.as_ptr() as *const c_char, + trait_ref_typeid.as_bytes().len() as c_uint, + ); + let v = [cx.const_usize(0), typeid]; + llvm::LLVMRustGlobalAddMetadata( + vtable, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + cx.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ); + let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); + let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); + llvm::LLVMGlobalSetMetadata( + vtable, + llvm::MetadataType::MD_vcall_visibility as c_uint, + vcall_visibility_metadata, + ); + } +} + +/// Creates debug information for the given vtable, which is for the +/// given type. +/// +/// Adds the created metadata nodes directly to the crate's IR. +pub fn create_vtable_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + poly_trait_ref: Option>, + vtable: &'ll Value, +) { + // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in + // LLVM at the moment. + if cx.sess().opts.unstable_opts.virtual_function_elimination && cx.sess().lto() == Lto::Fat { + vcall_visibility_metadata(cx, ty, poly_trait_ref, vtable); + } + + if cx.dbg_cx.is_none() { + return; + } + + // Only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo != DebugInfo::Full { + return; + } + + let vtable_name = + compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable); + let vtable_type_di_node = build_vtable_type_di_node(cx, ty, poly_trait_ref); + let linkage_name = ""; + + unsafe { + llvm::LLVMRustDIBuilderCreateStaticVariable( + DIB(cx), + NO_SCOPE_METADATA, + vtable_name.as_ptr().cast(), + vtable_name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + vtable_type_di_node, + true, + vtable, + None, + 0, + ); + } +} + +/// Creates an "extension" of an existing `DIScope` into another file. +pub fn extend_scope_to_file<'ll>( + cx: &CodegenCx<'ll, '_>, + scope_metadata: &'ll DIScope, + file: &SourceFile, +) -> &'ll DILexicalBlock { + let file_metadata = file_metadata(cx, file); + unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlockFile(DIB(cx), scope_metadata, file_metadata) } +} + +pub fn tuple_field_name(field_index: usize) -> Cow<'static, str> { + const TUPLE_FIELD_NAMES: [&'static str; 16] = [ + "__0", "__1", "__2", "__3", "__4", "__5", "__6", "__7", "__8", "__9", "__10", "__11", + "__12", "__13", "__14", "__15", + ]; + TUPLE_FIELD_NAMES + .get(field_index) + .map(|s| Cow::from(*s)) + .unwrap_or_else(|| Cow::from(format!("__{}", field_index))) +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs new file mode 100644 index 00000000000..d6e2c8ccdf4 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -0,0 +1,514 @@ +use std::borrow::Cow; + +use libc::c_uint; +use rustc_codegen_ssa::debuginfo::{ + type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo, +}; +use rustc_middle::{ + bug, + ty::{ + self, + layout::{LayoutOf, TyAndLayout}, + util::Discr, + AdtDef, GeneratorSubsts, + }, +}; +use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; +use smallvec::smallvec; + +use crate::{ + common::CodegenCx, + debuginfo::{ + metadata::{ + build_field_di_node, closure_saved_names_of_captured_variables, + enums::tag_base_type, + file_metadata, generator_layout_and_saved_local_names, size_and_align_of, + type_map::{self, UniqueTypeId}, + unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA, + UNKNOWN_LINE_NUMBER, + }, + utils::DIB, + }, + llvm::{ + self, + debuginfo::{DIFile, DIFlags, DIType}, + }, +}; + +/// In CPP-like mode, we generate a union of structs for each variant and an +/// explicit discriminant field roughly equivalent to the following C/C++ code: +/// +/// ```c +/// union enum$<{fully-qualified-name}> { +/// struct {variant 0 name} { +/// +/// } variant0; +/// +/// {name} discriminant; +/// } +/// ``` +/// +/// As you can see, the type name is wrapped `enum$`. This way we can have a +/// single NatVis rule for handling all enums. +/// +/// At the LLVM IR level this looks like +/// +/// ```txt +/// DW_TAG_union_type (top-level type for enum) +/// DW_TAG_member (member for variant 1) +/// DW_TAG_member (member for variant 2) +/// DW_TAG_member (member for variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// DW_TAG_enumeration_type (type of tag) +/// ``` +/// +/// The above encoding applies for enums with a direct tag. For niche-tag we have to do things +/// differently in order to allow a NatVis visualizer to extract all the information needed: +/// We generate a union of two fields, one for the dataful variant +/// and one that just points to the discriminant (which is some field within the dataful variant). +/// We also create a DW_TAG_enumeration_type DIE that contains tag values for the non-dataful +/// variants and make the discriminant field that type. We then use NatVis to render the enum type +/// correctly in Windbg/VS. This will generate debuginfo roughly equivalent to the following C: +/// +/// ```c +/// union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> { +/// struct { +/// +/// } dataful_variant; +/// enum Discriminant$ { +/// +/// } discriminant; +/// } +/// ``` +/// +/// The NatVis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>` +/// and evaluates `this.discriminant`. If the value is between the min niche and max +/// niche, then the enum is in the dataful variant and `this.dataful_variant` is +/// rendered. Otherwise, the enum is in one of the non-dataful variants. In that +/// case, we just need to render the name of the `this.discriminant` enum. +pub(super) fn build_enum_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let enum_type = unique_type_id.expect_ty(); + let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { + bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) + }; + + let enum_type_and_layout = cx.layout_of(enum_type); + let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); + + debug_assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + type_map::Stub::Union, + unique_type_id, + &enum_type_name, + cx.size_and_align_of(enum_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |cx, enum_type_di_node| { + match enum_type_and_layout.variants { + Variants::Single { index: variant_index } => { + if enum_adt_def.variants().is_empty() { + // Uninhabited enums have Variants::Single. We don't generate + // any members for them. + return smallvec![]; + } + + build_single_variant_union_fields( + cx, + enum_adt_def, + enum_type_and_layout, + enum_type_di_node, + variant_index, + ) + } + Variants::Multiple { + tag_encoding: TagEncoding::Direct, + ref variants, + tag_field, + .. + } => build_union_fields_for_direct_tag_enum( + cx, + enum_adt_def, + enum_type_and_layout, + enum_type_di_node, + &mut variants.indices(), + tag_field, + ), + Variants::Multiple { + tag_encoding: TagEncoding::Niche { dataful_variant, .. }, + ref variants, + tag_field, + .. + } => build_union_fields_for_niche_tag_enum( + cx, + enum_adt_def, + enum_type_and_layout, + enum_type_di_node, + dataful_variant, + &mut variants.indices(), + tag_field, + ), + } + }, + NO_GENERICS, + ) +} + +/// A generator debuginfo node looks the same as a that of an enum type. +/// +/// See [build_enum_type_di_node] for more information. +pub(super) fn build_generator_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let generator_type = unique_type_id.expect_ty(); + let generator_type_and_layout = cx.layout_of(generator_type); + let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); + + debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + type_map::Stub::Union, + unique_type_id, + &generator_type_name, + size_and_align_of(generator_type_and_layout), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |cx, generator_type_di_node| match generator_type_and_layout.variants { + Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => { + build_union_fields_for_direct_tag_generator( + cx, + generator_type_and_layout, + generator_type_di_node, + ) + } + Variants::Single { .. } + | Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => { + bug!( + "Encountered generator with non-direct-tag layout: {:?}", + generator_type_and_layout + ) + } + }, + NO_GENERICS, + ) +} + +fn build_single_variant_union_fields<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_index: VariantIdx, +) -> SmallVec<&'ll DIType> { + let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); + let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout.ty, + enum_type_di_node, + variant_index, + enum_adt_def.variant(variant_index), + variant_layout, + ); + + // NOTE: The field name of the union is the same as the variant name, not "variant0". + let variant_name = enum_adt_def.variant(variant_index).name.as_str(); + + smallvec![build_field_di_node( + cx, + enum_type_di_node, + variant_name, + // NOTE: We use the size and align of the entire type, not from variant_layout + // since the later is sometimes smaller (if it has fewer fields). + size_and_align_of(enum_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + variant_struct_type_di_node, + )] +} + +fn build_union_fields_for_direct_tag_enum<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_indices: &mut dyn Iterator, + tag_field: usize, +) -> SmallVec<&'ll DIType> { + let variant_field_infos: SmallVec> = variant_indices + .map(|variant_index| { + let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); + + VariantFieldInfo { + variant_index, + variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout.ty, + enum_type_di_node, + variant_index, + enum_adt_def.variant(variant_index), + variant_layout, + ), + source_info: None, + } + }) + .collect(); + + let discr_type_name = cx.tcx.item_name(enum_adt_def.did()); + let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + let discr_type_di_node = super::build_enumeration_type_di_node( + cx, + discr_type_name.as_str(), + tag_base_type, + &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { + (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) + }), + enum_type_di_node, + ); + + build_union_fields_for_direct_tag_enum_or_generator( + cx, + enum_type_and_layout, + enum_type_di_node, + &variant_field_infos, + discr_type_di_node, + tag_field, + ) +} + +fn build_union_fields_for_niche_tag_enum<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + dataful_variant_index: VariantIdx, + variant_indices: &mut dyn Iterator, + tag_field: usize, +) -> SmallVec<&'ll DIType> { + let dataful_variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout.ty, + enum_type_di_node, + dataful_variant_index, + &enum_adt_def.variant(dataful_variant_index), + enum_type_and_layout.for_variant(cx, dataful_variant_index), + ); + + let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + // Create an DW_TAG_enumerator for each variant except the dataful one. + let discr_type_di_node = super::build_enumeration_type_di_node( + cx, + "Discriminant$", + tag_base_type, + &mut variant_indices.filter_map(|variant_index| { + if let Some(discr_val) = + super::compute_discriminant_value(cx, enum_type_and_layout, variant_index) + { + let discr = Discr { val: discr_val as u128, ty: tag_base_type }; + let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); + Some((discr, variant_name)) + } else { + debug_assert_eq!(variant_index, dataful_variant_index); + None + } + }), + enum_type_di_node, + ); + + smallvec![ + build_field_di_node( + cx, + enum_type_di_node, + "dataful_variant", + size_and_align_of(enum_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + dataful_variant_struct_type_di_node, + ), + build_field_di_node( + cx, + enum_type_di_node, + "discriminant", + cx.size_and_align_of(tag_base_type), + enum_type_and_layout.fields.offset(tag_field), + DIFlags::FlagZero, + discr_type_di_node, + ), + ] +} + +fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + generator_type_and_layout: TyAndLayout<'tcx>, + generator_type_di_node: &'ll DIType, +) -> SmallVec<&'ll DIType> { + let Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } = generator_type_and_layout.variants else { + bug!("This function only supports layouts with directly encoded tags.") + }; + + let (generator_def_id, generator_substs) = match generator_type_and_layout.ty.kind() { + &ty::Generator(def_id, substs, _) => (def_id, substs.as_generator()), + _ => unreachable!(), + }; + + let (generator_layout, state_specific_upvar_names) = + generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + + let common_upvar_names = closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx); + + // Build the type node for each field. + let variant_field_infos: SmallVec> = variant_range + .map(|variant_index| { + let variant_struct_type_di_node = super::build_generator_variant_struct_type_di_node( + cx, + variant_index, + generator_type_and_layout, + generator_type_di_node, + generator_layout, + &state_specific_upvar_names, + &common_upvar_names, + ); + + let span = generator_layout.variant_source_info[variant_index].span; + let source_info = if !span.is_dummy() { + let loc = cx.lookup_debug_loc(span.lo()); + Some((file_metadata(cx, &loc.file), loc.line as c_uint)) + } else { + None + }; + + VariantFieldInfo { variant_index, variant_struct_type_di_node, source_info } + }) + .collect(); + + let tag_base_type = tag_base_type(cx, generator_type_and_layout); + let discr_type_name = "Discriminant$"; + let discr_type_di_node = super::build_enumeration_type_di_node( + cx, + discr_type_name, + tag_base_type, + &mut generator_substs + .discriminants(generator_def_id, cx.tcx) + .map(|(variant_index, discr)| (discr, GeneratorSubsts::variant_name(variant_index))), + generator_type_di_node, + ); + + build_union_fields_for_direct_tag_enum_or_generator( + cx, + generator_type_and_layout, + generator_type_di_node, + &variant_field_infos[..], + discr_type_di_node, + tag_field, + ) +} + +/// This is a helper function shared between enums and generators that makes sure fields have the +/// expect names. +fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_field_infos: &[VariantFieldInfo<'ll>], + discr_type_di_node: &'ll DIType, + tag_field: usize, +) -> SmallVec<&'ll DIType> { + let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1); + + // We create a field in the union for each variant ... + unions_fields.extend(variant_field_infos.into_iter().map(|variant_member_info| { + let (file_di_node, line_number) = variant_member_info + .source_info + .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); + + let field_name = variant_union_field_name(variant_member_info.variant_index); + let (size, align) = size_and_align_of(enum_type_and_layout); + + // We use LLVMRustDIBuilderCreateMemberType() member type directly because + // the build_field_di_node() function does not support specifying a source location, + // which is something that we don't do anywhere else. + unsafe { + llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + enum_type_di_node, + field_name.as_ptr().cast(), + field_name.len(), + file_di_node, + line_number, + // NOTE: We use the size and align of the entire type, not from variant_layout + // since the later is sometimes smaller (if it has fewer fields). + size.bits(), + align.bits() as u32, + // Union fields are always at offset zero + Size::ZERO.bits(), + DIFlags::FlagZero, + variant_member_info.variant_struct_type_di_node, + ) + } + })); + + debug_assert_eq!( + cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), + cx.size_and_align_of(super::tag_base_type(cx, enum_type_and_layout)) + ); + + // ... and a field for the discriminant. + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + "discriminant", + cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), + enum_type_and_layout.fields.offset(tag_field), + DIFlags::FlagZero, + discr_type_di_node, + )); + + unions_fields +} + +/// Information about a single field of the top-level DW_TAG_union_type. +struct VariantFieldInfo<'ll> { + variant_index: VariantIdx, + variant_struct_type_di_node: &'ll DIType, + source_info: Option<(&'ll DIFile, c_uint)>, +} + +fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> { + const PRE_ALLOCATED: [&str; 16] = [ + "variant0", + "variant1", + "variant2", + "variant3", + "variant4", + "variant5", + "variant6", + "variant7", + "variant8", + "variant9", + "variant10", + "variant11", + "variant12", + "variant13", + "variant14", + "variant15", + ]; + + PRE_ALLOCATED + .get(variant_index.as_usize()) + .map(|&s| Cow::from(s)) + .unwrap_or_else(|| format!("variant{}", variant_index.as_usize()).into()) +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs new file mode 100644 index 00000000000..73e01d0453b --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -0,0 +1,437 @@ +use rustc_codegen_ssa::debuginfo::{ + type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}, + wants_c_like_enum_debuginfo, +}; +use rustc_hir::def::CtorKind; +use rustc_index::vec::IndexVec; +use rustc_middle::{ + bug, + mir::{Field, GeneratorLayout, GeneratorSavedLocal}, + ty::{ + self, + layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, + util::Discr, + AdtDef, GeneratorSubsts, Ty, VariantDef, + }, +}; +use rustc_span::Symbol; +use rustc_target::abi::{HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants}; +use std::borrow::Cow; + +use crate::{ + common::CodegenCx, + debuginfo::{ + metadata::{ + build_field_di_node, build_generic_type_param_di_nodes, type_di_node, + type_map::{self, Stub}, + unknown_file_metadata, UNKNOWN_LINE_NUMBER, + }, + utils::{create_DIArray, get_namespace_for_item, DIB}, + }, + llvm::{ + self, + debuginfo::{DIFlags, DIType}, + }, +}; + +use super::{ + size_and_align_of, + type_map::{DINodeCreationResult, UniqueTypeId}, + SmallVec, +}; + +mod cpp_like; +mod native; + +pub(super) fn build_enum_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let enum_type = unique_type_id.expect_ty(); + let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { + bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) + }; + + let enum_type_and_layout = cx.layout_of(enum_type); + + if wants_c_like_enum_debuginfo(enum_type_and_layout) { + return build_c_style_enum_di_node(cx, enum_adt_def, enum_type_and_layout); + } + + if cpp_like_debuginfo(cx.tcx) { + cpp_like::build_enum_type_di_node(cx, unique_type_id) + } else { + native::build_enum_type_di_node(cx, unique_type_id) + } +} + +pub(super) fn build_generator_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + if cpp_like_debuginfo(cx.tcx) { + cpp_like::build_generator_di_node(cx, unique_type_id) + } else { + native::build_generator_di_node(cx, unique_type_id) + } +} + +/// Build the debuginfo node for a C-style enum, i.e. an enum the variants of which have no fields. +/// +/// The resulting debuginfo will be a DW_TAG_enumeration_type. +fn build_c_style_enum_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, +) -> DINodeCreationResult<'ll> { + let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); + DINodeCreationResult { + di_node: build_enumeration_type_di_node( + cx, + &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false), + tag_base_type(cx, enum_type_and_layout), + &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { + (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) + }), + containing_scope, + ), + already_stored_in_typemap: false, + } +} + +/// Extract the type with which we want to describe the tag of the given enum or generator. +fn tag_base_type<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, +) -> Ty<'tcx> { + debug_assert!(match enum_type_and_layout.ty.kind() { + ty::Generator(..) => true, + ty::Adt(adt_def, _) => adt_def.is_enum(), + _ => false, + }); + + match enum_type_and_layout.layout.variants() { + // A single-variant enum has no discriminant. + Variants::Single { .. } => { + bug!("tag_base_type() called for enum without tag: {:?}", enum_type_and_layout) + } + + Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } => { + // Niche tags are always normalized to unsized integers of the correct size. + match tag.primitive() { + Primitive::Int(t, _) => t, + Primitive::F32 => Integer::I32, + Primitive::F64 => Integer::I64, + Primitive::Pointer => { + // If the niche is the NULL value of a reference, then `discr_enum_ty` will be + // a RawPtr. CodeView doesn't know what to do with enums whose base type is a + // pointer so we fix this up to just be `usize`. + // DWARF might be able to deal with this but with an integer type we are on + // the safe side there too. + cx.data_layout().ptr_sized_integer() + } + } + .to_ty(cx.tcx, false) + } + + Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => { + // Direct tags preserve the sign. + tag.primitive().to_ty(cx.tcx) + } + } +} + +/// Build a DW_TAG_enumeration_type debuginfo node, with the given base type and variants. +/// This is a helper function and does not register anything in the type map by itself. +/// +/// `variants` is an iterator of (discr-value, variant-name). +/// +// NOTE: Handling of discriminant values is somewhat inconsistent. They can appear as u128, +// u64, and i64. Here everything gets mapped to i64 because that's what LLVM's API expects. +fn build_enumeration_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + type_name: &str, + base_type: Ty<'tcx>, + variants: &mut dyn Iterator, Cow<'tcx, str>)>, + containing_scope: &'ll DIType, +) -> &'ll DIType { + let is_unsigned = match base_type.kind() { + ty::Int(_) => false, + ty::Uint(_) => true, + _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."), + }; + + let enumerator_di_nodes: SmallVec> = variants + .map(|(discr, variant_name)| { + unsafe { + Some(llvm::LLVMRustDIBuilderCreateEnumerator( + DIB(cx), + variant_name.as_ptr().cast(), + variant_name.len(), + // FIXME: what if enumeration has i128 discriminant? + discr.val as i64, + is_unsigned, + )) + } + }) + .collect(); + + let (size, align) = cx.size_and_align_of(base_type); + + unsafe { + llvm::LLVMRustDIBuilderCreateEnumerationType( + DIB(cx), + containing_scope, + type_name.as_ptr().cast(), + type_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + create_DIArray(DIB(cx), &enumerator_di_nodes[..]), + type_di_node(cx, base_type), + true, + ) + } +} + +/// Build the debuginfo node for the struct type describing a single variant of an enum. +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// ``` +/// +/// In CPP-like mode, we have the exact same descriptions for each variant too: +/// +/// ```txt +/// DW_TAG_union_type (top-level type for enum) +/// DW_TAG_member (member for variant 1) +/// DW_TAG_member (member for variant 2) +/// DW_TAG_member (member for variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// DW_TAG_enumeration_type (type of tag) +/// ``` +/// +/// The node looks like: +/// +/// ```txt +/// DW_TAG_structure_type +/// DW_AT_name +/// DW_AT_byte_size 0x00000010 +/// DW_AT_alignment 0x00000008 +/// DW_TAG_member +/// DW_AT_name +/// DW_AT_type <0x0000018e> +/// DW_AT_alignment 0x00000004 +/// DW_AT_data_member_location 4 +/// DW_TAG_member +/// DW_AT_name +/// DW_AT_type <0x00000195> +/// DW_AT_alignment 0x00000008 +/// DW_AT_data_member_location 8 +/// ... +/// ``` +/// +/// The type of a variant is always a struct type with the name of the variant +/// and a DW_TAG_member for each field (but not the discriminant). +fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type: Ty<'tcx>, + enum_type_di_node: &'ll DIType, + variant_index: VariantIdx, + variant_def: &VariantDef, + variant_layout: TyAndLayout<'tcx>, +) -> &'ll DIType { + debug_assert_eq!(variant_layout.ty, enum_type); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + UniqueTypeId::for_enum_variant_struct_type(cx.tcx, enum_type, variant_index), + variant_def.name.as_str(), + // NOTE: We use size and align of enum_type, not from variant_layout: + cx.size_and_align_of(enum_type), + Some(enum_type_di_node), + DIFlags::FlagZero, + ), + |cx, struct_type_di_node| { + (0..variant_layout.fields.count()) + .map(|field_index| { + let field_name = if variant_def.ctor_kind != CtorKind::Fn { + // Fields have names + Cow::from(variant_def.fields[field_index].name.as_str()) + } else { + // Tuple-like + super::tuple_field_name(field_index) + }; + + let field_layout = variant_layout.field(cx, field_index); + + build_field_di_node( + cx, + struct_type_di_node, + &field_name, + (field_layout.size, field_layout.align.abi), + variant_layout.fields.offset(field_index), + DIFlags::FlagZero, + type_di_node(cx, field_layout.ty), + ) + }) + .collect() + }, + |cx| build_generic_type_param_di_nodes(cx, enum_type), + ) + .di_node +} + +/// Build the struct type for describing a single generator state. +/// See [build_generator_variant_struct_type_di_node]. +/// +/// ```txt +/// +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// +/// ``` +pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + variant_index: VariantIdx, + generator_type_and_layout: TyAndLayout<'tcx>, + generator_type_di_node: &'ll DIType, + generator_layout: &GeneratorLayout<'tcx>, + state_specific_upvar_names: &IndexVec>, + common_upvar_names: &[String], +) -> &'ll DIType { + let variant_name = GeneratorSubsts::variant_name(variant_index); + let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( + cx.tcx, + generator_type_and_layout.ty, + variant_index, + ); + + let variant_layout = generator_type_and_layout.for_variant(cx, variant_index); + + let generator_substs = match generator_type_and_layout.ty.kind() { + ty::Generator(_, substs, _) => substs.as_generator(), + _ => unreachable!(), + }; + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &variant_name, + size_and_align_of(generator_type_and_layout), + Some(generator_type_di_node), + DIFlags::FlagZero, + ), + |cx, variant_struct_type_di_node| { + // Fields that just belong to this variant/state + let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) + .map(|field_index| { + let generator_saved_local = generator_layout.variant_fields[variant_index] + [Field::from_usize(field_index)]; + let field_name_maybe = state_specific_upvar_names[generator_saved_local]; + let field_name = field_name_maybe + .as_ref() + .map(|s| Cow::from(s.as_str())) + .unwrap_or_else(|| super::tuple_field_name(field_index)); + + let field_type = variant_layout.field(cx, field_index).ty; + + build_field_di_node( + cx, + variant_struct_type_di_node, + &field_name, + cx.size_and_align_of(field_type), + variant_layout.fields.offset(field_index), + DIFlags::FlagZero, + type_di_node(cx, field_type), + ) + }) + .collect(); + + // Fields that are common to all states + let common_fields: SmallVec<_> = generator_substs + .prefix_tys() + .enumerate() + .map(|(index, upvar_ty)| { + build_field_di_node( + cx, + variant_struct_type_di_node, + &common_upvar_names[index], + cx.size_and_align_of(upvar_ty), + generator_type_and_layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, upvar_ty), + ) + }) + .collect(); + + state_specific_fields.into_iter().chain(common_fields.into_iter()).collect() + }, + |cx| build_generic_type_param_di_nodes(cx, generator_type_and_layout.ty), + ) + .di_node +} + +/// Returns the discriminant value corresponding to the variant index. +/// +/// Will return `None` if there is less than two variants (because then the enum won't have) +/// a tag, and if this is the dataful variant of a niche-layout enum (because then there is no +/// single discriminant value). +fn compute_discriminant_value<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + variant_index: VariantIdx, +) -> Option { + match enum_type_and_layout.layout.variants() { + &Variants::Single { .. } => None, + &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => Some( + enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val + as u64, + ), + &Variants::Multiple { + tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, + tag, + .. + } => { + if variant_index == dataful_variant { + None + } else { + let value = (variant_index.as_u32() as u128) + .wrapping_sub(niche_variants.start().as_u32() as u128) + .wrapping_add(niche_start); + let value = tag.size(cx).truncate(value); + // NOTE(eddyb) do *NOT* remove this assert, until + // we pass the full 128-bit value to LLVM, otherwise + // truncation will be silent and remain undetected. + assert_eq!(value as u64 as u128, value); + Some(value as u64) + } + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs new file mode 100644 index 00000000000..f1935e0ec31 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -0,0 +1,441 @@ +use std::borrow::Cow; + +use crate::{ + common::CodegenCx, + debuginfo::{ + metadata::{ + closure_saved_names_of_captured_variables, + enums::tag_base_type, + file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, + type_map::{self, Stub, StubInfo, UniqueTypeId}, + unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, + UNKNOWN_LINE_NUMBER, + }, + utils::{create_DIArray, get_namespace_for_item, DIB}, + }, + llvm::{ + self, + debuginfo::{DIFile, DIFlags, DIType}, + }, +}; +use libc::c_uint; +use rustc_codegen_ssa::{ + debuginfo::{type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo}, + traits::ConstMethods, +}; +use rustc_middle::{ + bug, + ty::{ + self, + layout::{LayoutOf, TyAndLayout}, + }, +}; +use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; +use smallvec::smallvec; + +/// Build the debuginfo node for an enum type. The listing below shows how such a +/// type looks like at the LLVM IR/DWARF level. It is a `DW_TAG_structure_type` +/// with a single `DW_TAG_variant_part` that in turn contains a `DW_TAG_variant` +/// for each variant of the enum. The variant-part also contains a single member +/// describing the discriminant, and a nested struct type for each of the variants. +/// +/// ```txt +/// ---> DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +pub(super) fn build_enum_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let enum_type = unique_type_id.expect_ty(); + let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { + bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) + }; + + let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); + let enum_type_and_layout = cx.layout_of(enum_type); + let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); + + debug_assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &enum_type_name, + size_and_align_of(enum_type_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + |cx, enum_type_di_node| { + // Build the struct type for each variant. These will be referenced by the + // DW_TAG_variant DIEs inside of the DW_TAG_variant_part DIE. + // We also called the names for the corresponding DW_TAG_variant DIEs here. + let variant_member_infos: SmallVec<_> = enum_adt_def + .variant_range() + .map(|variant_index| VariantMemberInfo { + variant_index, + variant_name: Cow::from(enum_adt_def.variant(variant_index).name.as_str()), + variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( + cx, + enum_type, + enum_type_di_node, + variant_index, + enum_adt_def.variant(variant_index), + enum_type_and_layout.for_variant(cx, variant_index), + ), + source_info: None, + }) + .collect(); + + smallvec![build_enum_variant_part_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + &variant_member_infos[..], + )] + }, + // We don't seem to be emitting generic args on the enum type, it seems. Rather + // they get attached to the struct type of each variant. + NO_GENERICS, + ) +} + +/// Build the debuginfo node for a generator environment. It looks the same as the debuginfo for +/// an enum. See [build_enum_type_di_node] for more information. +/// +/// ```txt +/// +/// ---> DW_TAG_structure_type (top-level type for the generator) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// +/// ``` +pub(super) fn build_generator_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let generator_type = unique_type_id.expect_ty(); + let &ty::Generator(generator_def_id, _, _ ) = generator_type.kind() else { + bug!("build_generator_di_node() called with non-generator type: `{:?}`", generator_type) + }; + + let containing_scope = get_namespace_for_item(cx, generator_def_id); + let generator_type_and_layout = cx.layout_of(generator_type); + + debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); + + let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &generator_type_name, + size_and_align_of(generator_type_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + |cx, generator_type_di_node| { + let (generator_layout, state_specific_upvar_names) = + generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + + let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = generator_type_and_layout.variants else { + bug!( + "Encountered generator with non-direct-tag layout: {:?}", + generator_type_and_layout + ) + }; + + let common_upvar_names = + closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + + // Build variant struct types + let variant_struct_type_di_nodes: SmallVec<_> = variants + .indices() + .map(|variant_index| { + // FIXME: This is problematic because just a number is not a valid identifier. + // GeneratorSubsts::variant_name(variant_index), would be consistent + // with enums? + let variant_name = format!("{}", variant_index.as_usize()).into(); + + let span = generator_layout.variant_source_info[variant_index].span; + let source_info = if !span.is_dummy() { + let loc = cx.lookup_debug_loc(span.lo()); + Some((file_metadata(cx, &loc.file), loc.line)) + } else { + None + }; + + VariantMemberInfo { + variant_index, + variant_name, + variant_struct_type_di_node: + super::build_generator_variant_struct_type_di_node( + cx, + variant_index, + generator_type_and_layout, + generator_type_di_node, + generator_layout, + &state_specific_upvar_names, + &common_upvar_names, + ), + source_info, + } + }) + .collect(); + + smallvec![build_enum_variant_part_di_node( + cx, + generator_type_and_layout, + generator_type_di_node, + &variant_struct_type_di_nodes[..], + )] + }, + // We don't seem to be emitting generic args on the generator type, it seems. Rather + // they get attached to the struct type of each variant. + NO_GENERICS, + ) +} + +/// Builds the DW_TAG_variant_part of an enum or generator debuginfo node: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// ---> DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +fn build_enum_variant_part_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_member_infos: &[VariantMemberInfo<'_, 'll>], +) -> &'ll DIType { + let tag_member_di_node = + build_discr_member_di_node(cx, enum_type_and_layout, enum_type_di_node); + + let variant_part_unique_type_id = + UniqueTypeId::for_enum_variant_part(cx.tcx, enum_type_and_layout.ty); + + let stub = StubInfo::new( + cx, + variant_part_unique_type_id, + |cx, variant_part_unique_type_id_str| unsafe { + let variant_part_name = ""; + llvm::LLVMRustDIBuilderCreateVariantPart( + DIB(cx), + enum_type_di_node, + variant_part_name.as_ptr().cast(), + variant_part_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + enum_type_and_layout.size.bits(), + enum_type_and_layout.align.abi.bits() as u32, + DIFlags::FlagZero, + tag_member_di_node, + create_DIArray(DIB(cx), &[]), + variant_part_unique_type_id_str.as_ptr().cast(), + variant_part_unique_type_id_str.len(), + ) + }, + ); + + type_map::build_type_with_children( + cx, + stub, + |cx, variant_part_di_node| { + variant_member_infos + .iter() + .map(|variant_member_info| { + build_enum_variant_member_di_node( + cx, + enum_type_and_layout, + variant_part_di_node, + variant_member_info, + ) + }) + .collect() + }, + NO_GENERICS, + ) + .di_node +} + +/// Builds the DW_TAG_member describing where we can find the tag of an enum. +/// Returns `None` if the enum does not have a tag. +/// +/// ```txt +/// +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// ---> DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// +/// ``` +fn build_discr_member_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_or_generator_type_and_layout: TyAndLayout<'tcx>, + enum_or_generator_type_di_node: &'ll DIType, +) -> Option<&'ll DIType> { + let tag_name = match enum_or_generator_type_and_layout.ty.kind() { + ty::Generator(..) => "__state", + _ => "", + }; + + // NOTE: This is actually wrong. This will become a member of + // of the DW_TAG_variant_part. But, due to LLVM's API, that + // can only be constructed with this DW_TAG_member already in created. + // In LLVM IR the wrong scope will be listed but when DWARF is + // generated from it, the DW_TAG_member will be a child the + // DW_TAG_variant_part. + let containing_scope = enum_or_generator_type_di_node; + + match enum_or_generator_type_and_layout.layout.variants() { + // A single-variant enum has no discriminant. + &Variants::Single { .. } => None, + + &Variants::Multiple { tag_field, .. } => { + let tag_base_type = tag_base_type(cx, enum_or_generator_type_and_layout); + let (size, align) = cx.size_and_align_of(tag_base_type); + + unsafe { + Some(llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + containing_scope, + tag_name.as_ptr().cast(), + tag_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + enum_or_generator_type_and_layout.fields.offset(tag_field).bits(), + DIFlags::FlagArtificial, + type_di_node(cx, tag_base_type), + )) + } + } + } +} + +/// Build the debuginfo node for `DW_TAG_variant`: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// ---> DW_TAG_variant (variant 1) +/// ---> DW_TAG_variant (variant 2) +/// ---> DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +/// +/// This node looks like: +/// +/// ```txt +/// DW_TAG_variant +/// DW_AT_discr_value 0 +/// DW_TAG_member +/// DW_AT_name None +/// DW_AT_type <0x000002a1> +/// DW_AT_alignment 0x00000002 +/// DW_AT_data_member_location 0 +/// ``` +/// +/// The DW_AT_discr_value is optional, and is omitted if +/// - This is the only variant of a univariant enum (i.e. their is no discriminant) +/// - This is the "dataful" variant of a niche-layout enum +/// (where only the other variants are identified by a single value) +/// +/// There is only ever a single member, the type of which is a struct that describes the +/// fields of the variant (excluding the discriminant). The name of the member is the name +/// of the variant as given in the source code. The DW_AT_data_member_location is always +/// zero. +/// +/// Note that the LLVM DIBuilder API is a bit unintuitive here. The DW_TAG_variant subtree +/// (including the DW_TAG_member) is built by a single call to +/// `LLVMRustDIBuilderCreateVariantMemberType()`. +fn build_enum_variant_member_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + variant_part_di_node: &'ll DIType, + variant_member_info: &VariantMemberInfo<'_, 'll>, +) -> &'ll DIType { + let variant_index = variant_member_info.variant_index; + let discr_value = super::compute_discriminant_value(cx, enum_type_and_layout, variant_index); + + let (file_di_node, line_number) = variant_member_info + .source_info + .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); + + unsafe { + llvm::LLVMRustDIBuilderCreateVariantMemberType( + DIB(cx), + variant_part_di_node, + variant_member_info.variant_name.as_ptr().cast(), + variant_member_info.variant_name.len(), + file_di_node, + line_number, + enum_type_and_layout.size.bits(), + enum_type_and_layout.align.abi.bits() as u32, + Size::ZERO.bits(), + discr_value.map(|v| cx.const_u64(v)), + DIFlags::FlagZero, + variant_member_info.variant_struct_type_di_node, + ) + } +} + +/// Information needed for building a `DW_TAG_variant`: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// ---> DW_TAG_variant (variant 1) +/// ---> DW_TAG_variant (variant 2) +/// ---> DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +struct VariantMemberInfo<'a, 'll> { + variant_index: VariantIdx, + variant_name: Cow<'a, str>, + variant_struct_type_di_node: &'ll DIType, + source_info: Option<(&'ll DIFile, c_uint)>, +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs new file mode 100644 index 00000000000..ce2f419c4ac --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -0,0 +1,267 @@ +use std::cell::RefCell; + +use rustc_data_structures::{ + fingerprint::Fingerprint, + fx::FxHashMap, + stable_hasher::{HashStable, StableHasher}, +}; +use rustc_middle::{ + bug, + ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}, +}; +use rustc_target::abi::{Align, Size, VariantIdx}; + +use crate::{ + common::CodegenCx, + debuginfo::utils::{create_DIArray, debug_context, DIB}, + llvm::{ + self, + debuginfo::{DIFlags, DIScope, DIType}, + }, +}; + +use super::{unknown_file_metadata, SmallVec, UNKNOWN_LINE_NUMBER}; + +mod private { + // This type cannot be constructed outside of this module because + // it has a private field. We make use of this in order to prevent + // `UniqueTypeId` from being constructed directly, without asserting + // the preconditions. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] + pub struct HiddenZst; +} + +/// A unique identifier for anything that we create a debuginfo node for. +/// The types it contains are expected to already be normalized (which +/// is debug_asserted in the constructors). +/// +/// Note that there are some things that only show up in debuginfo, like +/// the separate type descriptions for each enum variant. These get an ID +/// too because they have their own debuginfo node in LLVM IR. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] +pub(super) enum UniqueTypeId<'tcx> { + /// The ID of a regular type as it shows up at the language level. + Ty(Ty<'tcx>, private::HiddenZst), + /// The ID for the single DW_TAG_variant_part nested inside the top-level + /// DW_TAG_structure_type that describes enums and generators. + VariantPart(Ty<'tcx>, private::HiddenZst), + /// The ID for the artificial struct type describing a single enum variant. + VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst), + /// The ID of the artificial type we create for VTables. + VTableTy(Ty<'tcx>, Option>, private::HiddenZst), +} + +impl<'tcx> UniqueTypeId<'tcx> { + pub fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self { + debug_assert_eq!(t, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)); + UniqueTypeId::Ty(t, private::HiddenZst) + } + + pub fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self { + debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + UniqueTypeId::VariantPart(enum_ty, private::HiddenZst) + } + + pub fn for_enum_variant_struct_type( + tcx: TyCtxt<'tcx>, + enum_ty: Ty<'tcx>, + variant_idx: VariantIdx, + ) -> Self { + debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst) + } + + pub fn for_vtable_ty( + tcx: TyCtxt<'tcx>, + self_type: Ty<'tcx>, + implemented_trait: Option>, + ) -> Self { + debug_assert_eq!( + self_type, + tcx.normalize_erasing_regions(ParamEnv::reveal_all(), self_type) + ); + debug_assert_eq!( + implemented_trait, + tcx.normalize_erasing_regions(ParamEnv::reveal_all(), implemented_trait) + ); + UniqueTypeId::VTableTy(self_type, implemented_trait, private::HiddenZst) + } + + /// Generates a string version of this [UniqueTypeId], which can be used as the `UniqueId` + /// argument of the various `LLVMRustDIBuilderCreate*Type()` methods. + /// + /// Right now this takes the form of a hex-encoded opaque hash value. + pub fn generate_unique_id_string(self, tcx: TyCtxt<'tcx>) -> String { + let mut hasher = StableHasher::new(); + tcx.with_stable_hashing_context(|mut hcx| { + hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher)) + }); + hasher.finish::().to_hex() + } + + pub fn expect_ty(self) -> Ty<'tcx> { + match self { + UniqueTypeId::Ty(ty, _) => ty, + _ => bug!("Expected `UniqueTypeId::Ty` but found `{:?}`", self), + } + } +} + +/// The `TypeMap` is where the debug context holds the type metadata nodes +/// created so far. The debuginfo nodes are identified by `UniqueTypeId`. +#[derive(Default)] +pub(crate) struct TypeMap<'ll, 'tcx> { + pub(super) unique_id_to_di_node: RefCell, &'ll DIType>>, +} + +impl<'ll, 'tcx> TypeMap<'ll, 'tcx> { + /// Adds a `UniqueTypeId` to metadata mapping to the `TypeMap`. The method will + /// fail if the mapping already exists. + pub(super) fn insert(&self, unique_type_id: UniqueTypeId<'tcx>, metadata: &'ll DIType) { + if self.unique_id_to_di_node.borrow_mut().insert(unique_type_id, metadata).is_some() { + bug!("type metadata for unique ID '{:?}' is already in the `TypeMap`!", unique_type_id); + } + } + + pub(super) fn di_node_for_unique_id( + &self, + unique_type_id: UniqueTypeId<'tcx>, + ) -> Option<&'ll DIType> { + self.unique_id_to_di_node.borrow().get(&unique_type_id).cloned() + } +} + +pub struct DINodeCreationResult<'ll> { + pub di_node: &'ll DIType, + pub already_stored_in_typemap: bool, +} + +impl<'ll> DINodeCreationResult<'ll> { + pub fn new(di_node: &'ll DIType, already_stored_in_typemap: bool) -> Self { + DINodeCreationResult { di_node, already_stored_in_typemap } + } +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum Stub<'ll> { + Struct, + Union, + VTableTy { vtable_holder: &'ll DIType }, +} + +pub struct StubInfo<'ll, 'tcx> { + metadata: &'ll DIType, + unique_type_id: UniqueTypeId<'tcx>, +} + +impl<'ll, 'tcx> StubInfo<'ll, 'tcx> { + pub(super) fn new( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, + build: impl FnOnce(&CodegenCx<'ll, 'tcx>, /* unique_type_id_str: */ &str) -> &'ll DIType, + ) -> StubInfo<'ll, 'tcx> { + let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); + let di_node = build(cx, &unique_type_id_str); + StubInfo { metadata: di_node, unique_type_id } + } +} + +/// Create a stub debuginfo node onto which fields and nested types can be attached. +pub(super) fn stub<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + kind: Stub<'ll>, + unique_type_id: UniqueTypeId<'tcx>, + name: &str, + (size, align): (Size, Align), + containing_scope: Option<&'ll DIScope>, + flags: DIFlags, +) -> StubInfo<'ll, 'tcx> { + let empty_array = create_DIArray(DIB(cx), &[]); + let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); + + let metadata = match kind { + Stub::Struct | Stub::VTableTy { .. } => { + let vtable_holder = match kind { + Stub::VTableTy { vtable_holder } => Some(vtable_holder), + _ => None, + }; + unsafe { + llvm::LLVMRustDIBuilderCreateStructType( + DIB(cx), + containing_scope, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + flags, + None, + empty_array, + 0, + vtable_holder, + unique_type_id_str.as_ptr().cast(), + unique_type_id_str.len(), + ) + } + } + Stub::Union => unsafe { + llvm::LLVMRustDIBuilderCreateUnionType( + DIB(cx), + containing_scope, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + flags, + Some(empty_array), + 0, + unique_type_id_str.as_ptr().cast(), + unique_type_id_str.len(), + ) + }, + }; + StubInfo { metadata, unique_type_id } +} + +/// This function enables creating debuginfo nodes that can recursively refer to themselves. +/// It will first insert the given stub into the type map and only then execute the `members` +/// and `generics` closures passed in. These closures have access to the stub so they can +/// directly attach fields to them. If the type of a field transitively refers back +/// to the type currently being built, the stub will already be found in the type map, +/// which effectively breaks the recursion cycle. +pub(super) fn build_type_with_children<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + stub_info: StubInfo<'ll, 'tcx>, + members: impl FnOnce(&CodegenCx<'ll, 'tcx>, &'ll DIType) -> SmallVec<&'ll DIType>, + generics: impl FnOnce(&CodegenCx<'ll, 'tcx>) -> SmallVec<&'ll DIType>, +) -> DINodeCreationResult<'ll> { + debug_assert_eq!( + debug_context(cx).type_map.di_node_for_unique_id(stub_info.unique_type_id), + None + ); + + debug_context(cx).type_map.insert(stub_info.unique_type_id, stub_info.metadata); + + let members: SmallVec<_> = + members(cx, stub_info.metadata).into_iter().map(|node| Some(node)).collect(); + let generics: SmallVec> = + generics(cx).into_iter().map(|node| Some(node)).collect(); + + if !(members.is_empty() && generics.is_empty()) { + unsafe { + let members_array = create_DIArray(DIB(cx), &members[..]); + let generics_array = create_DIArray(DIB(cx), &generics[..]); + llvm::LLVMRustDICompositeTypeReplaceArrays( + DIB(cx), + stub_info.metadata, + Some(members_array), + Some(generics_array), + ); + } + } + + DINodeCreationResult { di_node: stub_info.metadata, already_stored_in_typemap: true } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs new file mode 100644 index 00000000000..cf591295b84 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -0,0 +1,614 @@ +#![doc = include_str!("doc.md")] + +use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; + +use self::metadata::{file_metadata, type_di_node}; +use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER}; +use self::namespace::mangled_name_of_instance; +use self::utils::{create_DIArray, is_node_local_to_unit, DIB}; + +use crate::abi::FnAbi; +use crate::builder::Builder; +use crate::common::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::{ + DIArray, DIBuilder, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DIType, + DIVariable, +}; +use crate::value::Value; + +use rustc_codegen_ssa::debuginfo::type_names; +use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use rustc_hir::def_id::{DefId, DefIdMap}; +use rustc_index::vec::IndexVec; +use rustc_middle::mir; +use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; +use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitable}; +use rustc_session::config::{self, DebugInfo}; +use rustc_session::Session; +use rustc_span::symbol::Symbol; +use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span}; +use rustc_target::abi::Size; + +use libc::c_uint; +use smallvec::SmallVec; +use std::cell::OnceCell; +use std::cell::RefCell; +use std::iter; +use tracing::debug; + +mod create_scope_map; +pub mod gdb; +pub mod metadata; +mod namespace; +mod utils; + +pub use self::create_scope_map::compute_mir_scopes; +pub use self::metadata::build_global_var_di_node; +pub use self::metadata::extend_scope_to_file; + +#[allow(non_upper_case_globals)] +const DW_TAG_auto_variable: c_uint = 0x100; +#[allow(non_upper_case_globals)] +const DW_TAG_arg_variable: c_uint = 0x101; + +/// A context object for maintaining all state needed by the debuginfo module. +pub struct CodegenUnitDebugContext<'ll, 'tcx> { + llcontext: &'ll llvm::Context, + llmod: &'ll llvm::Module, + builder: &'ll mut DIBuilder<'ll>, + created_files: RefCell, &'ll DIFile>>, + + type_map: metadata::TypeMap<'ll, 'tcx>, + namespace_map: RefCell>, + recursion_marker_type: OnceCell<&'ll DIType>, +} + +impl Drop for CodegenUnitDebugContext<'_, '_> { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustDIBuilderDispose(&mut *(self.builder as *mut _)); + } + } +} + +impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { + pub fn new(llmod: &'ll llvm::Module) -> Self { + debug!("CodegenUnitDebugContext::new"); + let builder = unsafe { llvm::LLVMRustDIBuilderCreate(llmod) }; + // DIBuilder inherits context from the module, so we'd better use the same one + let llcontext = unsafe { llvm::LLVMGetModuleContext(llmod) }; + CodegenUnitDebugContext { + llcontext, + llmod, + builder, + created_files: Default::default(), + type_map: Default::default(), + namespace_map: RefCell::new(Default::default()), + recursion_marker_type: OnceCell::new(), + } + } + + pub fn finalize(&self, sess: &Session) { + unsafe { + llvm::LLVMRustDIBuilderFinalize(self.builder); + + if !sess.target.is_like_msvc { + // Debuginfo generation in LLVM by default uses a higher + // version of dwarf than macOS currently understands. We can + // instruct LLVM to emit an older version of dwarf, however, + // for macOS to understand. For more info see #11352 + // This can be overridden using --llvm-opts -dwarf-version,N. + // Android has the same issue (#22398) + let dwarf_version = sess + .opts + .unstable_opts + .dwarf_version + .unwrap_or(sess.target.default_dwarf_version); + llvm::LLVMRustAddModuleFlag( + self.llmod, + llvm::LLVMModFlagBehavior::Warning, + "Dwarf Version\0".as_ptr().cast(), + dwarf_version, + ); + } else { + // Indicate that we want CodeView debug information on MSVC + llvm::LLVMRustAddModuleFlag( + self.llmod, + llvm::LLVMModFlagBehavior::Warning, + "CodeView\0".as_ptr().cast(), + 1, + ) + } + + // Prevent bitcode readers from deleting the debug info. + let ptr = "Debug Info Version\0".as_ptr(); + llvm::LLVMRustAddModuleFlag( + self.llmod, + llvm::LLVMModFlagBehavior::Warning, + ptr.cast(), + llvm::LLVMRustDebugMetadataVersion(), + ); + } + } +} + +/// Creates any deferred debug metadata nodes +pub fn finalize(cx: &CodegenCx<'_, '_>) { + if let Some(dbg_cx) = &cx.dbg_cx { + debug!("finalize"); + + if gdb::needs_gdb_debug_scripts_section(cx) { + // Add a .debug_gdb_scripts section to this compile-unit. This will + // cause GDB to try and load the gdb_load_rust_pretty_printers.py file, + // which activates the Rust pretty printers for binary this section is + // contained in. + gdb::get_or_insert_gdb_debug_scripts_section_global(cx); + } + + dbg_cx.finalize(cx.sess()); + } +} + +impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { + // FIXME(eddyb) find a common convention for all of the debuginfo-related + // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). + fn dbg_var_addr( + &mut self, + dbg_var: &'ll DIVariable, + dbg_loc: &'ll DILocation, + variable_alloca: Self::Value, + direct_offset: Size, + indirect_offsets: &[Size], + ) { + // Convert the direct and indirect offsets to address ops. + // FIXME(eddyb) use `const`s instead of getting the values via FFI, + // the values should match the ones in the DWARF standard anyway. + let op_deref = || unsafe { llvm::LLVMRustDIBuilderCreateOpDeref() }; + let op_plus_uconst = || unsafe { llvm::LLVMRustDIBuilderCreateOpPlusUconst() }; + let mut addr_ops = SmallVec::<[u64; 8]>::new(); + + if direct_offset.bytes() > 0 { + addr_ops.push(op_plus_uconst()); + addr_ops.push(direct_offset.bytes() as u64); + } + for &offset in indirect_offsets { + addr_ops.push(op_deref()); + if offset.bytes() > 0 { + addr_ops.push(op_plus_uconst()); + addr_ops.push(offset.bytes() as u64); + } + } + + unsafe { + // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. + llvm::LLVMRustDIBuilderInsertDeclareAtEnd( + DIB(self.cx()), + variable_alloca, + dbg_var, + addr_ops.as_ptr(), + addr_ops.len() as c_uint, + dbg_loc, + self.llbb(), + ); + } + } + + fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { + unsafe { + let dbg_loc_as_llval = llvm::LLVMRustMetadataAsValue(self.cx().llcx, dbg_loc); + llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc_as_llval); + } + } + + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { + gdb::insert_reference_to_gdb_debug_scripts_section_global(self) + } + + fn set_var_name(&mut self, value: &'ll Value, name: &str) { + // Avoid wasting time if LLVM value names aren't even enabled. + if self.sess().fewer_names() { + return; + } + + // Only function parameters and instructions are local to a function, + // don't change the name of anything else (e.g. globals). + let param_or_inst = unsafe { + llvm::LLVMIsAArgument(value).is_some() || llvm::LLVMIsAInstruction(value).is_some() + }; + if !param_or_inst { + return; + } + + // Avoid replacing the name if it already exists. + // While we could combine the names somehow, it'd + // get noisy quick, and the usefulness is dubious. + if llvm::get_value_name(value).is_empty() { + llvm::set_value_name(value, name.as_bytes()); + } + } +} + +/// A source code location used to generate debug information. +// FIXME(eddyb) rename this to better indicate it's a duplicate of +// `rustc_span::Loc` rather than `DILocation`, perhaps by making +// `lookup_char_pos` return the right information instead. +pub struct DebugLoc { + /// Information about the original source file. + pub file: Lrc, + /// The (1-based) line number. + pub line: u32, + /// The (1-based) column number. + pub col: u32, +} + +impl CodegenCx<'_, '_> { + /// Looks up debug source information about a `BytePos`. + // FIXME(eddyb) rename this to better indicate it's a duplicate of + // `lookup_char_pos` rather than `dbg_loc`, perhaps by making + // `lookup_char_pos` return the right information instead. + pub fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { + let (file, line, col) = match self.sess().source_map().lookup_line(pos) { + Ok(SourceFileAndLine { sf: file, line }) => { + let line_pos = file.line_begin_pos(pos); + + // Use 1-based indexing. + let line = (line + 1) as u32; + let col = (pos - line_pos).to_u32() + 1; + + (file, line, col) + } + Err(file) => (file, UNKNOWN_LINE_NUMBER, UNKNOWN_COLUMN_NUMBER), + }; + + // For MSVC, omit the column number. + // Otherwise, emit it. This mimics clang behaviour. + // See discussion in https://github.com/rust-lang/rust/issues/42921 + if self.sess().target.is_like_msvc { + DebugLoc { file, line, col: UNKNOWN_COLUMN_NUMBER } + } else { + DebugLoc { file, line, col } + } + } +} + +impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn create_function_debug_context( + &self, + instance: Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + llfn: &'ll Value, + mir: &mir::Body<'tcx>, + ) -> Option> { + if self.sess().opts.debuginfo == DebugInfo::None { + return None; + } + + // Initialize fn debug context (including scopes). + let empty_scope = DebugScope { + dbg_scope: self.dbg_scope_fn(instance, fn_abi, Some(llfn)), + inlined_at: None, + file_start_pos: BytePos(0), + file_end_pos: BytePos(0), + }; + let mut fn_debug_context = + FunctionDebugContext { scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes) }; + + // Fill in all the scopes, with the information from the MIR body. + compute_mir_scopes(self, instance, mir, &mut fn_debug_context); + + Some(fn_debug_context) + } + + fn dbg_scope_fn( + &self, + instance: Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + maybe_definition_llfn: Option<&'ll Value>, + ) -> &'ll DIScope { + let tcx = self.tcx; + + let def_id = instance.def_id(); + let containing_scope = get_containing_scope(self, instance); + let span = tcx.def_span(def_id); + let loc = self.lookup_debug_loc(span.lo()); + let file_metadata = file_metadata(self, &loc.file); + + let function_type_metadata = unsafe { + let fn_signature = get_function_signature(self, fn_abi); + llvm::LLVMRustDIBuilderCreateSubroutineType(DIB(self), fn_signature) + }; + + let mut name = String::new(); + type_names::push_item_name(tcx, def_id, false, &mut name); + + // Find the enclosing function, in case this is a closure. + let enclosing_fn_def_id = tcx.typeck_root_def_id(def_id); + + // We look up the generics of the enclosing function and truncate the substs + // to their length in order to cut off extra stuff that might be in there for + // closures or generators. + let generics = tcx.generics_of(enclosing_fn_def_id); + let substs = instance.substs.truncate_to(tcx, generics); + + type_names::push_generic_params( + tcx, + tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs), + &mut name, + ); + + let template_parameters = get_template_parameters(self, generics, substs); + + let linkage_name = &mangled_name_of_instance(self, instance).name; + // Omit the linkage_name if it is the same as subprogram name. + let linkage_name = if &name == linkage_name { "" } else { linkage_name }; + + // FIXME(eddyb) does this need to be separate from `loc.line` for some reason? + let scope_line = loc.line; + + let mut flags = DIFlags::FlagPrototyped; + + if fn_abi.ret.layout.abi.is_uninhabited() { + flags |= DIFlags::FlagNoReturn; + } + + let mut spflags = DISPFlags::SPFlagDefinition; + if is_node_local_to_unit(self, def_id) { + spflags |= DISPFlags::SPFlagLocalToUnit; + } + if self.sess().opts.optimize != config::OptLevel::No { + spflags |= DISPFlags::SPFlagOptimized; + } + if let Some((id, _)) = tcx.entry_fn(()) { + if id == def_id { + spflags |= DISPFlags::SPFlagMainSubprogram; + } + } + + unsafe { + return llvm::LLVMRustDIBuilderCreateFunction( + DIB(self), + containing_scope, + name.as_ptr().cast(), + name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + file_metadata, + loc.line, + function_type_metadata, + scope_line, + flags, + spflags, + maybe_definition_llfn, + template_parameters, + None, + ); + } + + fn get_function_signature<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + ) -> &'ll DIArray { + if cx.sess().opts.debuginfo == DebugInfo::Limited { + return create_DIArray(DIB(cx), &[]); + } + + let mut signature = Vec::with_capacity(fn_abi.args.len() + 1); + + // Return type -- llvm::DIBuilder wants this at index 0 + signature.push(if fn_abi.ret.is_ignore() { + None + } else { + Some(type_di_node(cx, fn_abi.ret.layout.ty)) + }); + + // Arguments types + if cx.sess().target.is_like_msvc { + // FIXME(#42800): + // There is a bug in MSDIA that leads to a crash when it encounters + // a fixed-size array of `u8` or something zero-sized in a + // function-type (see #40477). + // As a workaround, we replace those fixed-size arrays with a + // pointer-type. So a function `fn foo(a: u8, b: [u8; 4])` would + // appear as `fn foo(a: u8, b: *const u8)` in debuginfo, + // and a function `fn bar(x: [(); 7])` as `fn bar(x: *const ())`. + // This transformed type is wrong, but these function types are + // already inaccurate due to ABI adjustments (see #42800). + signature.extend(fn_abi.args.iter().map(|arg| { + let t = arg.layout.ty; + let t = match t.kind() { + ty::Array(ct, _) + if (*ct == cx.tcx.types.u8) || cx.layout_of(*ct).is_zst() => + { + cx.tcx.mk_imm_ptr(*ct) + } + _ => t, + }; + Some(type_di_node(cx, t)) + })); + } else { + signature + .extend(fn_abi.args.iter().map(|arg| Some(type_di_node(cx, arg.layout.ty)))); + } + + create_DIArray(DIB(cx), &signature[..]) + } + + fn get_template_parameters<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + generics: &ty::Generics, + substs: SubstsRef<'tcx>, + ) -> &'ll DIArray { + if substs.types().next().is_none() { + return create_DIArray(DIB(cx), &[]); + } + + // Again, only create type information if full debuginfo is enabled + let template_params: Vec<_> = if cx.sess().opts.debuginfo == DebugInfo::Full { + let names = get_parameter_names(cx, generics); + iter::zip(substs, names) + .filter_map(|(kind, name)| { + if let GenericArgKind::Type(ty) = kind.unpack() { + let actual_type = + cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); + let actual_type_metadata = type_di_node(cx, actual_type); + let name = name.as_str(); + Some(unsafe { + Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( + DIB(cx), + None, + name.as_ptr().cast(), + name.len(), + actual_type_metadata, + )) + }) + } else { + None + } + }) + .collect() + } else { + vec![] + }; + + create_DIArray(DIB(cx), &template_params) + } + + fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec { + let mut names = generics.parent.map_or_else(Vec::new, |def_id| { + get_parameter_names(cx, cx.tcx.generics_of(def_id)) + }); + names.extend(generics.params.iter().map(|param| param.name)); + names + } + + fn get_containing_scope<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + ) -> &'ll DIScope { + // First, let's see if this is a method within an inherent impl. Because + // if yes, we want to make the result subroutine DIE a child of the + // subroutine's self-type. + let self_type = cx.tcx.impl_of_method(instance.def_id()).and_then(|impl_def_id| { + // If the method does *not* belong to a trait, proceed + if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { + let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions( + instance.substs, + ty::ParamEnv::reveal_all(), + cx.tcx.type_of(impl_def_id), + ); + + // Only "class" methods are generally understood by LLVM, + // so avoid methods on other types (e.g., `<*mut T>::null`). + match impl_self_ty.kind() { + ty::Adt(def, ..) if !def.is_box() => { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full + && !impl_self_ty.needs_subst() + { + Some(type_di_node(cx, impl_self_ty)) + } else { + Some(namespace::item_namespace(cx, def.did())) + } + } + _ => None, + } + } else { + // For trait method impls we still use the "parallel namespace" + // strategy + None + } + }); + + self_type.unwrap_or_else(|| { + namespace::item_namespace( + cx, + DefId { + krate: instance.def_id().krate, + index: cx + .tcx + .def_key(instance.def_id()) + .parent + .expect("get_containing_scope: missing parent?"), + }, + ) + }) + } + } + + fn dbg_loc( + &self, + scope: &'ll DIScope, + inlined_at: Option<&'ll DILocation>, + span: Span, + ) -> &'ll DILocation { + let DebugLoc { line, col, .. } = self.lookup_debug_loc(span.lo()); + + unsafe { llvm::LLVMRustDIBuilderCreateDebugLocation(line, col, scope, inlined_at) } + } + + fn create_vtable_debuginfo( + &self, + ty: Ty<'tcx>, + trait_ref: Option>, + vtable: Self::Value, + ) { + metadata::create_vtable_di_node(self, ty, trait_ref, vtable) + } + + fn extend_scope_to_file( + &self, + scope_metadata: &'ll DIScope, + file: &rustc_span::SourceFile, + ) -> &'ll DILexicalBlock { + metadata::extend_scope_to_file(self, scope_metadata, file) + } + + fn debuginfo_finalize(&self) { + finalize(self) + } + + // FIXME(eddyb) find a common convention for all of the debuginfo-related + // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). + fn create_dbg_var( + &self, + variable_name: Symbol, + variable_type: Ty<'tcx>, + scope_metadata: &'ll DIScope, + variable_kind: VariableKind, + span: Span, + ) -> &'ll DIVariable { + let loc = self.lookup_debug_loc(span.lo()); + let file_metadata = file_metadata(self, &loc.file); + + let type_metadata = type_di_node(self, variable_type); + + let (argument_index, dwarf_tag) = match variable_kind { + ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), + LocalVariable => (0, DW_TAG_auto_variable), + }; + let align = self.align_of(variable_type); + + let name = variable_name.as_str(); + unsafe { + llvm::LLVMRustDIBuilderCreateVariable( + DIB(self), + dwarf_tag, + scope_metadata, + name.as_ptr().cast(), + name.len(), + file_metadata, + loc.line, + type_metadata, + true, + DIFlags::FlagZero, + argument_index, + align.bytes() as u32, + ) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs new file mode 100644 index 00000000000..d5ea48c311b --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -0,0 +1,48 @@ +// Namespace Handling. + +use super::utils::{debug_context, DIB}; +use rustc_codegen_ssa::debuginfo::type_names; +use rustc_middle::ty::{self, Instance}; + +use crate::common::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::DIScope; +use rustc_hir::def_id::DefId; + +pub fn mangled_name_of_instance<'a, 'tcx>( + cx: &CodegenCx<'a, 'tcx>, + instance: Instance<'tcx>, +) -> ty::SymbolName<'tcx> { + let tcx = cx.tcx; + tcx.symbol_name(instance) +} + +pub fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { + if let Some(&scope) = debug_context(cx).namespace_map.borrow().get(&def_id) { + return scope; + } + + let def_key = cx.tcx.def_key(def_id); + let parent_scope = def_key + .parent + .map(|parent| item_namespace(cx, DefId { krate: def_id.krate, index: parent })); + + let namespace_name_string = { + let mut output = String::new(); + type_names::push_item_name(cx.tcx, def_id, false, &mut output); + output + }; + + let scope = unsafe { + llvm::LLVMRustDIBuilderCreateNameSpace( + DIB(cx), + parent_scope, + namespace_name_string.as_ptr().cast(), + namespace_name_string.len(), + false, // ExportSymbols (only relevant for C++ anonymous namespaces) + ) + }; + + debug_context(cx).namespace_map.borrow_mut().insert(def_id, scope); + scope +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs new file mode 100644 index 00000000000..8f243673907 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -0,0 +1,99 @@ +// Utility Functions. + +use super::namespace::item_namespace; +use super::CodegenUnitDebugContext; + +use rustc_hir::def_id::DefId; +use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; +use rustc_middle::ty::{self, DefIdTree, Ty}; +use tracing::trace; + +use crate::common::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::{DIArray, DIBuilder, DIDescriptor, DIScope}; + +pub fn is_node_local_to_unit(cx: &CodegenCx<'_, '_>, def_id: DefId) -> bool { + // The is_local_to_unit flag indicates whether a function is local to the + // current compilation unit (i.e., if it is *static* in the C-sense). The + // *reachable* set should provide a good approximation of this, as it + // contains everything that might leak out of the current crate (by being + // externally visible or by being inlined into something externally + // visible). It might better to use the `exported_items` set from + // `driver::CrateAnalysis` in the future, but (atm) this set is not + // available in the codegen pass. + !cx.tcx.is_reachable_non_generic(def_id) +} + +#[allow(non_snake_case)] +pub fn create_DIArray<'ll>( + builder: &DIBuilder<'ll>, + arr: &[Option<&'ll DIDescriptor>], +) -> &'ll DIArray { + unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) } +} + +#[inline] +pub fn debug_context<'a, 'll, 'tcx>( + cx: &'a CodegenCx<'ll, 'tcx>, +) -> &'a CodegenUnitDebugContext<'ll, 'tcx> { + cx.dbg_cx.as_ref().unwrap() +} + +#[inline] +#[allow(non_snake_case)] +pub fn DIB<'a, 'll>(cx: &'a CodegenCx<'ll, '_>) -> &'a DIBuilder<'ll> { + cx.dbg_cx.as_ref().unwrap().builder +} + +pub fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { + item_namespace(cx, cx.tcx.parent(def_id)) +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum FatPtrKind { + Slice, + Dyn, +} + +/// Determines if `pointee_ty` is slice-like or trait-object-like, i.e. +/// if the second field of the fat pointer is a length or a vtable-pointer. +/// If `pointee_ty` does not require a fat pointer (because it is Sized) then +/// the function returns `None`. +pub(crate) fn fat_pointer_kind<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + pointee_ty: Ty<'tcx>, +) -> Option { + let pointee_tail_ty = cx.tcx.struct_tail_erasing_lifetimes(pointee_ty, cx.param_env()); + let layout = cx.layout_of(pointee_tail_ty); + trace!( + "fat_pointer_kind: {:?} has layout {:?} (is_unsized? {})", + pointee_tail_ty, + layout, + layout.is_unsized() + ); + + if !layout.is_unsized() { + return None; + } + + match *pointee_tail_ty.kind() { + ty::Str | ty::Slice(_) => Some(FatPtrKind::Slice), + ty::Dynamic(..) => Some(FatPtrKind::Dyn), + ty::Foreign(_) => { + // Assert that pointers to foreign types really are thin: + debug_assert_eq!( + cx.size_of(cx.tcx.mk_imm_ptr(pointee_tail_ty)), + cx.size_of(cx.tcx.mk_imm_ptr(cx.tcx.types.u8)) + ); + None + } + _ => { + // For all other pointee types we should already have returned None + // at the beginning of the function. + panic!( + "fat_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {:?}", + pointee_tail_ty + ) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs new file mode 100644 index 00000000000..fa0ecd18fc8 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -0,0 +1,146 @@ +//! Declare various LLVM values. +//! +//! Prefer using functions and methods from this module rather than calling LLVM +//! functions directly. These functions do some additional work to ensure we do +//! the right thing given the preconceptions of codegen. +//! +//! Some useful guidelines: +//! +//! * Use declare_* family of methods if you are declaring, but are not +//! interested in defining the Value they return. +//! * Use define_* family of methods when you might be defining the Value. +//! * When in doubt, define. + +use crate::abi::{FnAbi, FnAbiLlvmExt}; +use crate::attributes; +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm::AttributePlace::Function; +use crate::type_::Type; +use crate::value::Value; +use rustc_codegen_ssa::traits::TypeMembershipMethods; +use rustc_middle::ty::Ty; +use rustc_symbol_mangling::typeid::typeid_for_fnabi; +use smallvec::SmallVec; +use tracing::debug; + +/// Declare a function. +/// +/// If there’s a value with the same name already declared, the function will +/// update the declaration and return existing Value instead. +fn declare_raw_fn<'ll>( + cx: &CodegenCx<'ll, '_>, + name: &str, + callconv: llvm::CallConv, + unnamed: llvm::UnnamedAddr, + ty: &'ll Type, +) -> &'ll Value { + debug!("declare_raw_fn(name={:?}, ty={:?})", name, ty); + let llfn = unsafe { + llvm::LLVMRustGetOrInsertFunction(cx.llmod, name.as_ptr().cast(), name.len(), ty) + }; + + llvm::SetFunctionCallConv(llfn, callconv); + llvm::SetUnnamedAddress(llfn, unnamed); + + let mut attrs = SmallVec::<[_; 4]>::new(); + + if cx.tcx.sess.opts.cg.no_redzone.unwrap_or(cx.tcx.sess.target.disable_redzone) { + attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx)); + } + + attrs.extend(attributes::non_lazy_bind_attr(cx)); + + attributes::apply_to_llfn(llfn, Function, &attrs); + + llfn +} + +impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { + /// Declare a global value. + /// + /// If there’s a value with the same name already declared, the function will + /// return its Value instead. + pub fn declare_global(&self, name: &str, ty: &'ll Type) -> &'ll Value { + debug!("declare_global(name={:?})", name); + unsafe { llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_ptr().cast(), name.len(), ty) } + } + + /// Declare a C ABI function. + /// + /// Only use this for foreign function ABIs and glue. For Rust functions use + /// `declare_fn` instead. + /// + /// If there’s a value with the same name already declared, the function will + /// update the declaration and return existing Value instead. + pub fn declare_cfn( + &self, + name: &str, + unnamed: llvm::UnnamedAddr, + fn_type: &'ll Type, + ) -> &'ll Value { + declare_raw_fn(self, name, llvm::CCallConv, unnamed, fn_type) + } + + /// Declare a Rust function. + /// + /// If there’s a value with the same name already declared, the function will + /// update the declaration and return existing Value instead. + pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Value { + debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi); + + // Function addresses in Rust are never significant, allowing functions to + // be merged. + let llfn = declare_raw_fn( + self, + name, + fn_abi.llvm_cconv(), + llvm::UnnamedAddr::Global, + fn_abi.llvm_type(self), + ); + fn_abi.apply_attrs_llfn(self, llfn); + + if self.tcx.sess.is_sanitizer_cfi_enabled() { + let typeid = typeid_for_fnabi(self.tcx, fn_abi); + self.set_type_metadata(llfn, typeid); + } + + llfn + } + + /// Declare a global with an intention to define it. + /// + /// Use this function when you intend to define a global. This function will + /// return `None` if the name already has a definition associated with it. In that + /// case an error should be reported to the user, because it usually happens due + /// to user’s fault (e.g., misuse of `#[no_mangle]` or `#[export_name]` attributes). + pub fn define_global(&self, name: &str, ty: &'ll Type) -> Option<&'ll Value> { + if self.get_defined_value(name).is_some() { + None + } else { + Some(self.declare_global(name, ty)) + } + } + + /// Declare a private global + /// + /// Use this function when you intend to define a global without a name. + pub fn define_private_global(&self, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMRustInsertPrivateGlobal(self.llmod, ty) } + } + + /// Gets declared value by name. + pub fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { + debug!("get_declared_value(name={:?})", name); + unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_ptr().cast(), name.len()) } + } + + /// Gets defined or externally defined (AvailableExternally linkage) value by + /// name. + pub fn get_defined_value(&self, name: &str) -> Option<&'ll Value> { + self.get_declared_value(name).and_then(|val| { + let declaration = unsafe { llvm::LLVMIsDeclaration(val) != 0 }; + if !declaration { Some(val) } else { None } + }) + } +} diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs new file mode 100644 index 00000000000..9f364749287 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -0,0 +1,1924 @@ +use crate::abi::{Abi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode}; +use crate::builder::Builder; +use crate::context::CodegenCx; +use crate::llvm; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::va_arg::emit_va_arg; +use crate::value::Value; + +use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh}; +use rustc_codegen_ssa::common::span_invalid_monomorphization_error; +use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; +use rustc_codegen_ssa::mir::operand::OperandRef; +use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_codegen_ssa::traits::*; +use rustc_hir as hir; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; +use rustc_middle::ty::{self, Ty}; +use rustc_middle::{bug, span_bug}; +use rustc_span::{sym, symbol::kw, Span, Symbol}; +use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; +use rustc_target::spec::{HasTargetSpec, PanicStrategy}; + +use std::cmp::Ordering; +use std::iter; + +fn get_simple_intrinsic<'ll>( + cx: &CodegenCx<'ll, '_>, + name: Symbol, +) -> Option<(&'ll Type, &'ll Value)> { + let llvm_name = match name { + sym::sqrtf32 => "llvm.sqrt.f32", + sym::sqrtf64 => "llvm.sqrt.f64", + sym::powif32 => "llvm.powi.f32", + sym::powif64 => "llvm.powi.f64", + sym::sinf32 => "llvm.sin.f32", + sym::sinf64 => "llvm.sin.f64", + sym::cosf32 => "llvm.cos.f32", + sym::cosf64 => "llvm.cos.f64", + sym::powf32 => "llvm.pow.f32", + sym::powf64 => "llvm.pow.f64", + sym::expf32 => "llvm.exp.f32", + sym::expf64 => "llvm.exp.f64", + sym::exp2f32 => "llvm.exp2.f32", + sym::exp2f64 => "llvm.exp2.f64", + sym::logf32 => "llvm.log.f32", + sym::logf64 => "llvm.log.f64", + sym::log10f32 => "llvm.log10.f32", + sym::log10f64 => "llvm.log10.f64", + sym::log2f32 => "llvm.log2.f32", + sym::log2f64 => "llvm.log2.f64", + sym::fmaf32 => "llvm.fma.f32", + sym::fmaf64 => "llvm.fma.f64", + sym::fabsf32 => "llvm.fabs.f32", + sym::fabsf64 => "llvm.fabs.f64", + sym::minnumf32 => "llvm.minnum.f32", + sym::minnumf64 => "llvm.minnum.f64", + sym::maxnumf32 => "llvm.maxnum.f32", + sym::maxnumf64 => "llvm.maxnum.f64", + sym::copysignf32 => "llvm.copysign.f32", + sym::copysignf64 => "llvm.copysign.f64", + sym::floorf32 => "llvm.floor.f32", + sym::floorf64 => "llvm.floor.f64", + sym::ceilf32 => "llvm.ceil.f32", + sym::ceilf64 => "llvm.ceil.f64", + sym::truncf32 => "llvm.trunc.f32", + sym::truncf64 => "llvm.trunc.f64", + sym::rintf32 => "llvm.rint.f32", + sym::rintf64 => "llvm.rint.f64", + sym::nearbyintf32 => "llvm.nearbyint.f32", + sym::nearbyintf64 => "llvm.nearbyint.f64", + sym::roundf32 => "llvm.round.f32", + sym::roundf64 => "llvm.round.f64", + _ => return None, + }; + Some(cx.get_intrinsic(llvm_name)) +} + +impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { + fn codegen_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + args: &[OperandRef<'tcx, &'ll Value>], + llresult: &'ll Value, + span: Span, + ) { + let tcx = self.tcx; + let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); + + let ty::FnDef(def_id, substs) = *callee_ty.kind() else { + bug!("expected fn item type, found {}", callee_ty); + }; + + let sig = callee_ty.fn_sig(tcx); + let sig = tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig); + let arg_tys = sig.inputs(); + let ret_ty = sig.output(); + let name = tcx.item_name(def_id); + + let llret_ty = self.layout_of(ret_ty).llvm_type(self); + let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); + + let simple = get_simple_intrinsic(self, name); + let llval = match name { + _ if simple.is_some() => { + let (simple_ty, simple_fn) = simple.unwrap(); + self.call( + simple_ty, + simple_fn, + &args.iter().map(|arg| arg.immediate()).collect::>(), + None, + ) + } + sym::likely => { + self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)]) + } + sym::unlikely => self + .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]), + kw::Try => { + try_intrinsic( + self, + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + llresult, + ); + return; + } + sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]), + sym::va_copy => { + self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()]) + } + sym::va_arg => { + match fn_abi.ret.layout.abi { + abi::Abi::Scalar(scalar) => { + match scalar.primitive() { + Primitive::Int(..) => { + if self.cx().size_of(ret_ty).bytes() < 4 { + // `va_arg` should not be called on an integer type + // less than 4 bytes in length. If it is, promote + // the integer to an `i32` and truncate the result + // back to the smaller type. + let promoted_result = emit_va_arg(self, args[0], tcx.types.i32); + self.trunc(promoted_result, llret_ty) + } else { + emit_va_arg(self, args[0], ret_ty) + } + } + Primitive::F64 | Primitive::Pointer => { + emit_va_arg(self, args[0], ret_ty) + } + // `va_arg` should never be used with the return type f32. + Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"), + } + } + _ => bug!("the va_arg intrinsic does not work with non-scalar types"), + } + } + + sym::volatile_load | sym::unaligned_volatile_load => { + let tp_ty = substs.type_at(0); + let ptr = args[0].immediate(); + let load = if let PassMode::Cast(ty) = fn_abi.ret.mode { + let llty = ty.llvm_type(self); + let ptr = self.pointercast(ptr, self.type_ptr_to(llty)); + self.volatile_load(llty, ptr) + } else { + self.volatile_load(self.layout_of(tp_ty).llvm_type(self), ptr) + }; + let align = if name == sym::unaligned_volatile_load { + 1 + } else { + self.align_of(tp_ty).bytes() as u32 + }; + unsafe { + llvm::LLVMSetAlignment(load, align); + } + self.to_immediate(load, self.layout_of(tp_ty)) + } + sym::volatile_store => { + let dst = args[0].deref(self.cx()); + args[1].val.volatile_store(self, dst); + return; + } + sym::unaligned_volatile_store => { + let dst = args[0].deref(self.cx()); + args[1].val.unaligned_volatile_store(self, dst); + return; + } + sym::prefetch_read_data + | sym::prefetch_write_data + | sym::prefetch_read_instruction + | sym::prefetch_write_instruction => { + let (rw, cache_type) = match name { + sym::prefetch_read_data => (0, 1), + sym::prefetch_write_data => (1, 1), + sym::prefetch_read_instruction => (0, 0), + sym::prefetch_write_instruction => (1, 0), + _ => bug!(), + }; + self.call_intrinsic( + "llvm.prefetch", + &[ + args[0].immediate(), + self.const_i32(rw), + args[1].immediate(), + self.const_i32(cache_type), + ], + ) + } + sym::ctlz + | sym::ctlz_nonzero + | sym::cttz + | sym::cttz_nonzero + | sym::ctpop + | sym::bswap + | sym::bitreverse + | sym::rotate_left + | sym::rotate_right + | sym::saturating_add + | sym::saturating_sub => { + let ty = arg_tys[0]; + match int_type_width_signed(ty, self) { + Some((width, signed)) => match name { + sym::ctlz | sym::cttz => { + let y = self.const_bool(false); + self.call_intrinsic( + &format!("llvm.{}.i{}", name, width), + &[args[0].immediate(), y], + ) + } + sym::ctlz_nonzero => { + let y = self.const_bool(true); + let llvm_name = &format!("llvm.ctlz.i{}", width); + self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) + } + sym::cttz_nonzero => { + let y = self.const_bool(true); + let llvm_name = &format!("llvm.cttz.i{}", width); + self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) + } + sym::ctpop => self.call_intrinsic( + &format!("llvm.ctpop.i{}", width), + &[args[0].immediate()], + ), + sym::bswap => { + if width == 8 { + args[0].immediate() // byte swap a u8/i8 is just a no-op + } else { + self.call_intrinsic( + &format!("llvm.bswap.i{}", width), + &[args[0].immediate()], + ) + } + } + sym::bitreverse => self.call_intrinsic( + &format!("llvm.bitreverse.i{}", width), + &[args[0].immediate()], + ), + sym::rotate_left | sym::rotate_right => { + let is_left = name == sym::rotate_left; + let val = args[0].immediate(); + let raw_shift = args[1].immediate(); + // rotate = funnel shift with first two args the same + let llvm_name = + &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); + self.call_intrinsic(llvm_name, &[val, val, raw_shift]) + } + sym::saturating_add | sym::saturating_sub => { + let is_add = name == sym::saturating_add; + let lhs = args[0].immediate(); + let rhs = args[1].immediate(); + let llvm_name = &format!( + "llvm.{}{}.sat.i{}", + if signed { 's' } else { 'u' }, + if is_add { "add" } else { "sub" }, + width + ); + self.call_intrinsic(llvm_name, &[lhs, rhs]) + } + _ => bug!(), + }, + None => { + span_invalid_monomorphization_error( + tcx.sess, + span, + &format!( + "invalid monomorphization of `{}` intrinsic: \ + expected basic integer type, found `{}`", + name, ty + ), + ); + return; + } + } + } + + sym::raw_eq => { + use abi::Abi::*; + let tp_ty = substs.type_at(0); + let layout = self.layout_of(tp_ty).layout; + let use_integer_compare = match layout.abi() { + Scalar(_) | ScalarPair(_, _) => true, + Uninhabited | Vector { .. } => false, + Aggregate { .. } => { + // For rusty ABIs, small aggregates are actually passed + // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), + // so we re-use that same threshold here. + layout.size() <= self.data_layout().pointer_size * 2 + } + }; + + let a = args[0].immediate(); + let b = args[1].immediate(); + if layout.size().bytes() == 0 { + self.const_bool(true) + } else if use_integer_compare { + let integer_ty = self.type_ix(layout.size().bits()); + let ptr_ty = self.type_ptr_to(integer_ty); + let a_ptr = self.bitcast(a, ptr_ty); + let a_val = self.load(integer_ty, a_ptr, layout.align().abi); + let b_ptr = self.bitcast(b, ptr_ty); + let b_val = self.load(integer_ty, b_ptr, layout.align().abi); + self.icmp(IntPredicate::IntEQ, a_val, b_val) + } else { + let i8p_ty = self.type_i8p(); + let a_ptr = self.bitcast(a, i8p_ty); + let b_ptr = self.bitcast(b, i8p_ty); + let n = self.const_usize(layout.size().bytes()); + let cmp = self.call_intrinsic("memcmp", &[a_ptr, b_ptr, n]); + match self.cx.sess().target.arch.as_ref() { + "avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)), + _ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)), + } + } + } + + sym::black_box => { + args[0].val.store(self, result); + + // We need to "use" the argument in some way LLVM can't introspect, and on + // targets that support it we can typically leverage inline assembly to do + // this. LLVM's interpretation of inline assembly is that it's, well, a black + // box. This isn't the greatest implementation since it probably deoptimizes + // more than we want, but it's so far good enough. + crate::asm::inline_asm_call( + self, + "", + "r,~{memory}", + &[result.llval], + self.type_void(), + true, + false, + llvm::AsmDialect::Att, + &[span], + false, + None, + ) + .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); + + // We have copied the value to `result` already. + return; + } + + _ if name.as_str().starts_with("simd_") => { + match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { + Ok(llval) => llval, + Err(()) => return, + } + } + + _ => bug!("unknown intrinsic '{}'", name), + }; + + if !fn_abi.ret.is_ignore() { + if let PassMode::Cast(ty) = fn_abi.ret.mode { + let ptr_llty = self.type_ptr_to(ty.llvm_type(self)); + let ptr = self.pointercast(result.llval, ptr_llty); + self.store(llval, ptr, result.align); + } else { + OperandRef::from_immediate_or_packed_pair(self, llval, result.layout) + .val + .store(self, result); + } + } + } + + fn abort(&mut self) { + self.call_intrinsic("llvm.trap", &[]); + } + + fn assume(&mut self, val: Self::Value) { + self.call_intrinsic("llvm.assume", &[val]); + } + + fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value { + self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) + } + + fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value { + // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time + // optimization pass replaces calls to this intrinsic with code to test type membership. + let i8p_ty = self.type_i8p(); + let bitcast = self.bitcast(pointer, i8p_ty); + self.call_intrinsic("llvm.type.test", &[bitcast, typeid]) + } + + fn type_checked_load( + &mut self, + llvtable: &'ll Value, + vtable_byte_offset: u64, + typeid: &'ll Value, + ) -> Self::Value { + let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); + self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]) + } + + fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { + self.call_intrinsic("llvm.va_start", &[va_list]) + } + + fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value { + self.call_intrinsic("llvm.va_end", &[va_list]) + } +} + +fn try_intrinsic<'ll>( + bx: &mut Builder<'_, 'll, '_>, + try_func: &'ll Value, + data: &'ll Value, + catch_func: &'ll Value, + dest: &'ll Value, +) { + if bx.sess().panic_strategy() == PanicStrategy::Abort { + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.call(try_func_ty, try_func, &[data], None); + // Return 0 unconditionally from the intrinsic call; + // we can never unwind. + let ret_align = bx.tcx().data_layout.i32_align.abi; + bx.store(bx.const_i32(0), dest, ret_align); + } else if wants_msvc_seh(bx.sess()) { + codegen_msvc_try(bx, try_func, data, catch_func, dest); + } else if bx.sess().target.os == "emscripten" { + codegen_emcc_try(bx, try_func, data, catch_func, dest); + } else { + codegen_gnu_try(bx, try_func, data, catch_func, dest); + } +} + +// MSVC's definition of the `rust_try` function. +// +// This implementation uses the new exception handling instructions in LLVM +// which have support in LLVM for SEH on MSVC targets. Although these +// instructions are meant to work for all targets, as of the time of this +// writing, however, LLVM does not recommend the usage of these new instructions +// as the old ones are still more optimized. +fn codegen_msvc_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, + try_func: &'ll Value, + data: &'ll Value, + catch_func: &'ll Value, + dest: &'ll Value, +) { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { + bx.set_personality_fn(bx.eh_personality()); + + let normal = bx.append_sibling_block("normal"); + let catchswitch = bx.append_sibling_block("catchswitch"); + let catchpad_rust = bx.append_sibling_block("catchpad_rust"); + let catchpad_foreign = bx.append_sibling_block("catchpad_foreign"); + let caught = bx.append_sibling_block("caught"); + + let try_func = llvm::get_param(bx.llfn(), 0); + let data = llvm::get_param(bx.llfn(), 1); + let catch_func = llvm::get_param(bx.llfn(), 2); + + // We're generating an IR snippet that looks like: + // + // declare i32 @rust_try(%try_func, %data, %catch_func) { + // %slot = alloca i8* + // invoke %try_func(%data) to label %normal unwind label %catchswitch + // + // normal: + // ret i32 0 + // + // catchswitch: + // %cs = catchswitch within none [%catchpad_rust, %catchpad_foreign] unwind to caller + // + // catchpad_rust: + // %tok = catchpad within %cs [%type_descriptor, 8, %slot] + // %ptr = load %slot + // call %catch_func(%data, %ptr) + // catchret from %tok to label %caught + // + // catchpad_foreign: + // %tok = catchpad within %cs [null, 64, null] + // call %catch_func(%data, null) + // catchret from %tok to label %caught + // + // caught: + // ret i32 1 + // } + // + // This structure follows the basic usage of throw/try/catch in LLVM. + // For example, compile this C++ snippet to see what LLVM generates: + // + // struct rust_panic { + // rust_panic(const rust_panic&); + // ~rust_panic(); + // + // void* x[2]; + // }; + // + // int __rust_try( + // void (*try_func)(void*), + // void *data, + // void (*catch_func)(void*, void*) noexcept + // ) { + // try { + // try_func(data); + // return 0; + // } catch(rust_panic& a) { + // catch_func(data, &a); + // return 1; + // } catch(...) { + // catch_func(data, NULL); + // return 1; + // } + // } + // + // More information can be found in libstd's seh.rs implementation. + let ptr_align = bx.tcx().data_layout.pointer_align.abi; + let slot = bx.alloca(bx.type_i8p(), ptr_align); + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.invoke(try_func_ty, try_func, &[data], normal, catchswitch, None); + + bx.switch_to_block(normal); + bx.ret(bx.const_i32(0)); + + bx.switch_to_block(catchswitch); + let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]); + + // We can't use the TypeDescriptor defined in libpanic_unwind because it + // might be in another DLL and the SEH encoding only supports specifying + // a TypeDescriptor from the current module. + // + // However this isn't an issue since the MSVC runtime uses string + // comparison on the type name to match TypeDescriptors rather than + // pointer equality. + // + // So instead we generate a new TypeDescriptor in each module that uses + // `try` and let the linker merge duplicate definitions in the same + // module. + // + // When modifying, make sure that the type_name string exactly matches + // the one used in src/libpanic_unwind/seh.rs. + let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_i8p()); + let type_name = bx.const_bytes(b"rust_panic\0"); + let type_info = + bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_i8p()), type_name], false); + let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info)); + unsafe { + llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); + llvm::SetUniqueComdat(bx.llmod, tydesc); + llvm::LLVMSetInitializer(tydesc, type_info); + } + + // The flag value of 8 indicates that we are catching the exception by + // reference instead of by value. We can't use catch by value because + // that requires copying the exception object, which we don't support + // since our exception object effectively contains a Box. + // + // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang + bx.switch_to_block(catchpad_rust); + let flags = bx.const_i32(8); + let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); + let ptr = bx.load(bx.type_i8p(), slot, ptr_align); + let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); + bx.call(catch_ty, catch_func, &[data, ptr], Some(&funclet)); + bx.catch_ret(&funclet, caught); + + // The flag value of 64 indicates a "catch-all". + bx.switch_to_block(catchpad_foreign); + let flags = bx.const_i32(64); + let null = bx.const_null(bx.type_i8p()); + let funclet = bx.catch_pad(cs, &[null, flags, null]); + bx.call(catch_ty, catch_func, &[data, null], Some(&funclet)); + bx.catch_ret(&funclet, caught); + + bx.switch_to_block(caught); + bx.ret(bx.const_i32(1)); + }); + + // Note that no invoke is used here because by definition this function + // can't panic (that's what it's catching). + let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let i32_align = bx.tcx().data_layout.i32_align.abi; + bx.store(ret, dest, i32_align); +} + +// Definition of the standard `try` function for Rust using the GNU-like model +// of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke` +// instructions). +// +// This codegen is a little surprising because we always call a shim +// function instead of inlining the call to `invoke` manually here. This is done +// because in LLVM we're only allowed to have one personality per function +// definition. The call to the `try` intrinsic is being inlined into the +// function calling it, and that function may already have other personality +// functions in play. By calling a shim we're guaranteed that our shim will have +// the right personality function. +fn codegen_gnu_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, + try_func: &'ll Value, + data: &'ll Value, + catch_func: &'ll Value, + dest: &'ll Value, +) { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { + // Codegens the shims described above: + // + // bx: + // invoke %try_func(%data) normal %normal unwind %catch + // + // normal: + // ret 0 + // + // catch: + // (%ptr, _) = landingpad + // call %catch_func(%data, %ptr) + // ret 1 + let then = bx.append_sibling_block("then"); + let catch = bx.append_sibling_block("catch"); + + let try_func = llvm::get_param(bx.llfn(), 0); + let data = llvm::get_param(bx.llfn(), 1); + let catch_func = llvm::get_param(bx.llfn(), 2); + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.invoke(try_func_ty, try_func, &[data], then, catch, None); + + bx.switch_to_block(then); + bx.ret(bx.const_i32(0)); + + // Type indicator for the exception being thrown. + // + // The first value in this tuple is a pointer to the exception object + // being thrown. The second value is a "selector" indicating which of + // the landing pad clauses the exception's type had been matched to. + // rust_try ignores the selector. + bx.switch_to_block(catch); + let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); + let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1); + let tydesc = bx.const_null(bx.type_i8p()); + bx.add_clause(vals, tydesc); + let ptr = bx.extract_value(vals, 0); + let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); + bx.call(catch_ty, catch_func, &[data, ptr], None); + bx.ret(bx.const_i32(1)); + }); + + // Note that no invoke is used here because by definition this function + // can't panic (that's what it's catching). + let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let i32_align = bx.tcx().data_layout.i32_align.abi; + bx.store(ret, dest, i32_align); +} + +// Variant of codegen_gnu_try used for emscripten where Rust panics are +// implemented using C++ exceptions. Here we use exceptions of a specific type +// (`struct rust_panic`) to represent Rust panics. +fn codegen_emcc_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, + try_func: &'ll Value, + data: &'ll Value, + catch_func: &'ll Value, + dest: &'ll Value, +) { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { + // Codegens the shims described above: + // + // bx: + // invoke %try_func(%data) normal %normal unwind %catch + // + // normal: + // ret 0 + // + // catch: + // (%ptr, %selector) = landingpad + // %rust_typeid = @llvm.eh.typeid.for(@_ZTI10rust_panic) + // %is_rust_panic = %selector == %rust_typeid + // %catch_data = alloca { i8*, i8 } + // %catch_data[0] = %ptr + // %catch_data[1] = %is_rust_panic + // call %catch_func(%data, %catch_data) + // ret 1 + let then = bx.append_sibling_block("then"); + let catch = bx.append_sibling_block("catch"); + + let try_func = llvm::get_param(bx.llfn(), 0); + let data = llvm::get_param(bx.llfn(), 1); + let catch_func = llvm::get_param(bx.llfn(), 2); + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.invoke(try_func_ty, try_func, &[data], then, catch, None); + + bx.switch_to_block(then); + bx.ret(bx.const_i32(0)); + + // Type indicator for the exception being thrown. + // + // The first value in this tuple is a pointer to the exception object + // being thrown. The second value is a "selector" indicating which of + // the landing pad clauses the exception's type had been matched to. + bx.switch_to_block(catch); + let tydesc = bx.eh_catch_typeinfo(); + let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); + let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2); + bx.add_clause(vals, tydesc); + bx.add_clause(vals, bx.const_null(bx.type_i8p())); + let ptr = bx.extract_value(vals, 0); + let selector = bx.extract_value(vals, 1); + + // Check if the typeid we got is the one for a Rust panic. + let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[tydesc]); + let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid); + let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool()); + + // We need to pass two values to catch_func (ptr and is_rust_panic), so + // create an alloca and pass a pointer to that. + let ptr_align = bx.tcx().data_layout.pointer_align.abi; + let i8_align = bx.tcx().data_layout.i8_align.abi; + let catch_data_type = bx.type_struct(&[bx.type_i8p(), bx.type_bool()], false); + let catch_data = bx.alloca(catch_data_type, ptr_align); + let catch_data_0 = + bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(0)]); + bx.store(ptr, catch_data_0, ptr_align); + let catch_data_1 = + bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(1)]); + bx.store(is_rust_panic, catch_data_1, i8_align); + let catch_data = bx.bitcast(catch_data, bx.type_i8p()); + + let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); + bx.call(catch_ty, catch_func, &[data, catch_data], None); + bx.ret(bx.const_i32(1)); + }); + + // Note that no invoke is used here because by definition this function + // can't panic (that's what it's catching). + let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let i32_align = bx.tcx().data_layout.i32_align.abi; + bx.store(ret, dest, i32_align); +} + +// Helper function to give a Block to a closure to codegen a shim function. +// This is currently primarily used for the `try` intrinsic functions above. +fn gen_fn<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + name: &str, + rust_fn_sig: ty::PolyFnSig<'tcx>, + codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), +) -> (&'ll Type, &'ll Value) { + let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); + let llty = fn_abi.llvm_type(cx); + let llfn = cx.declare_fn(name, fn_abi); + cx.set_frame_pointer_type(llfn); + cx.apply_target_cpu_attr(llfn); + // FIXME(eddyb) find a nicer way to do this. + unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) }; + let llbb = Builder::append_block(cx, llfn, "entry-block"); + let bx = Builder::build(cx, llbb); + codegen(bx); + (llty, llfn) +} + +// Helper function used to get a handle to the `__rust_try` function used to +// catch exceptions. +// +// This function is only generated once and is then cached. +fn get_rust_try_fn<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), +) -> (&'ll Type, &'ll Value) { + if let Some(llfn) = cx.rust_try_fn.get() { + return llfn; + } + + // Define the type up front for the signature of the rust_try function. + let tcx = cx.tcx; + let i8p = tcx.mk_mut_ptr(tcx.types.i8); + // `unsafe fn(*mut i8) -> ()` + let try_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( + iter::once(i8p), + tcx.mk_unit(), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + ))); + // `unsafe fn(*mut i8, *mut i8) -> ()` + let catch_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( + [i8p, i8p].iter().cloned(), + tcx.mk_unit(), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + ))); + // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` + let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig( + [try_fn_ty, i8p, catch_fn_ty].into_iter(), + tcx.types.i32, + false, + hir::Unsafety::Unsafe, + Abi::Rust, + )); + let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen); + cx.rust_try_fn.set(Some(rust_try)); + rust_try +} + +fn generic_simd_intrinsic<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + name: Symbol, + callee_ty: Ty<'tcx>, + args: &[OperandRef<'tcx, &'ll Value>], + ret_ty: Ty<'tcx>, + llret_ty: &'ll Type, + span: Span, +) -> Result<&'ll Value, ()> { + // macros for error handling: + #[allow(unused_macro_rules)] + macro_rules! emit_error { + ($msg: tt) => { + emit_error!($msg, ) + }; + ($msg: tt, $($fmt: tt)*) => { + span_invalid_monomorphization_error( + bx.sess(), span, + &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), + name, $($fmt)*)); + } + } + + macro_rules! return_error { + ($($fmt: tt)*) => { + { + emit_error!($($fmt)*); + return Err(()); + } + } + } + + macro_rules! require { + ($cond: expr, $($fmt: tt)*) => { + if !$cond { + return_error!($($fmt)*); + } + }; + } + + macro_rules! require_simd { + ($ty: expr, $position: expr) => { + require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty) + }; + } + + let tcx = bx.tcx(); + let sig = + tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx)); + let arg_tys = sig.inputs(); + + if name == sym::simd_select_bitmask { + require_simd!(arg_tys[1], "argument"); + let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + + let expected_int_bits = (len.max(8) - 1).next_power_of_two(); + let expected_bytes = len / 8 + ((len % 8 > 0) as u64); + + let mask_ty = arg_tys[0]; + let mask = match mask_ty.kind() { + ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + let place = PlaceRef::alloca(bx, args[0].layout); + args[0].val.store(bx, place); + let int_ty = bx.type_ix(expected_bytes * 8); + let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty)); + bx.load(int_ty, ptr, Align::ONE) + } + _ => return_error!( + "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`", + mask_ty, + expected_int_bits, + expected_bytes + ), + }; + + let i1 = bx.type_i1(); + let im = bx.type_ix(len); + let i1xn = bx.type_vector(i1, len); + let m_im = bx.trunc(mask, im); + let m_i1s = bx.bitcast(m_im, i1xn); + return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); + } + + // every intrinsic below takes a SIMD vector as its first argument + require_simd!(arg_tys[0], "input"); + let in_ty = arg_tys[0]; + + let comparison = match name { + sym::simd_eq => Some(hir::BinOpKind::Eq), + sym::simd_ne => Some(hir::BinOpKind::Ne), + sym::simd_lt => Some(hir::BinOpKind::Lt), + sym::simd_le => Some(hir::BinOpKind::Le), + sym::simd_gt => Some(hir::BinOpKind::Gt), + sym::simd_ge => Some(hir::BinOpKind::Ge), + _ => None, + }; + + let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx()); + if let Some(cmp_op) = comparison { + require_simd!(ret_ty, "return"); + + let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + "expected return type with length {} (same as input type `{}`), \ + found `{}` with length {}", + in_len, + in_ty, + ret_ty, + out_len + ); + require!( + bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, + "expected return type with integer elements, found `{}` with non-integer `{}`", + ret_ty, + out_ty + ); + + return Ok(compare_simd_types( + bx, + args[0].immediate(), + args[1].immediate(), + in_elem, + llret_ty, + cmp_op, + )); + } + + if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") { + // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer. + // If there is no suffix, use the index array length. + let n: u64 = if stripped.is_empty() { + // Make sure this is actually an array, since typeck only checks the length-suffixed + // version of this intrinsic. + match args[2].layout.ty.kind() { + ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { + len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| { + span_bug!(span, "could not evaluate shuffle index array length") + }) + } + _ => return_error!( + "simd_shuffle index must be an array of `u32`, got `{}`", + args[2].layout.ty + ), + } + } else { + stripped.parse().unwrap_or_else(|_| { + span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") + }) + }; + + require_simd!(ret_ty, "return"); + let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + out_len == n, + "expected return type of length {}, found `{}` with length {}", + n, + ret_ty, + out_len + ); + require!( + in_elem == out_ty, + "expected return element type `{}` (element of input `{}`), \ + found `{}` with element type `{}`", + in_elem, + in_ty, + ret_ty, + out_ty + ); + + let total_len = u128::from(in_len) * 2; + + let vector = args[2].immediate(); + + let indices: Option> = (0..n) + .map(|i| { + let arg_idx = i; + let val = bx.const_get_elt(vector, i as u64); + match bx.const_to_opt_u128(val, true) { + None => { + emit_error!("shuffle index #{} is not a constant", arg_idx); + None + } + Some(idx) if idx >= total_len => { + emit_error!( + "shuffle index #{} is out of bounds (limit {})", + arg_idx, + total_len + ); + None + } + Some(idx) => Some(bx.const_i32(idx as i32)), + } + }) + .collect(); + let Some(indices) = indices else { + return Ok(bx.const_null(llret_ty)); + }; + + return Ok(bx.shuffle_vector( + args[0].immediate(), + args[1].immediate(), + bx.const_vector(&indices), + )); + } + + if name == sym::simd_insert { + require!( + in_elem == arg_tys[2], + "expected inserted type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + arg_tys[2] + ); + return Ok(bx.insert_element( + args[0].immediate(), + args[2].immediate(), + args[1].immediate(), + )); + } + if name == sym::simd_extract { + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); + return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); + } + + if name == sym::simd_select { + let m_elem_ty = in_elem; + let m_len = in_len; + require_simd!(arg_tys[1], "argument"); + let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + require!( + m_len == v_len, + "mismatched lengths: mask length `{}` != other vector length `{}`", + m_len, + v_len + ); + match m_elem_ty.kind() { + ty::Int(_) => {} + _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty), + } + // truncate the mask to a vector of i1s + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, m_len as u64); + let m_i1s = bx.trunc(args[0].immediate(), i1xn); + return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); + } + + if name == sym::simd_bitmask { + // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a + // vector mask and returns the most significant bit (MSB) of each lane in the form + // of either: + // * an unsigned integer + // * an array of `u8` + // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits. + // + // The bit order of the result depends on the byte endianness, LSB-first for little + // endian and MSB-first for big endian. + let expected_int_bits = in_len.max(8); + let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64); + + // Integer vector : + let (i_xn, in_elem_bitwidth) = match in_elem.kind() { + ty::Int(i) => ( + args[0].immediate(), + i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), + ), + ty::Uint(i) => ( + args[0].immediate(), + i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), + ), + _ => return_error!( + "vector argument `{}`'s element type `{}`, expected integer element type", + in_ty, + in_elem + ), + }; + + // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position. + let shift_indices = + vec![ + bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _); + in_len as _ + ]; + let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice())); + // Truncate vector to an + let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len)); + // Bitcast to iN: + let i_ = bx.bitcast(i1xn, bx.type_ix(in_len)); + + match ret_ty.kind() { + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => { + // Zero-extend iN to the bitmask type: + return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); + } + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + // Zero-extend iN to the array length: + let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); + + // Convert the integer to a byte array + let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE); + bx.store(ze, ptr, Align::ONE); + let array_ty = bx.type_array(bx.type_i8(), expected_bytes); + let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty)); + return Ok(bx.load(array_ty, ptr, Align::ONE)); + } + _ => return_error!( + "cannot return `{}`, expected `u{}` or `[u8; {}]`", + ret_ty, + expected_int_bits, + expected_bytes + ), + } + } + + fn simd_simple_float_intrinsic<'ll, 'tcx>( + name: Symbol, + in_elem: Ty<'_>, + in_ty: Ty<'_>, + in_len: u64, + bx: &mut Builder<'_, 'll, 'tcx>, + span: Span, + args: &[OperandRef<'tcx, &'ll Value>], + ) -> Result<&'ll Value, ()> { + #[allow(unused_macro_rules)] + macro_rules! emit_error { + ($msg: tt) => { + emit_error!($msg, ) + }; + ($msg: tt, $($fmt: tt)*) => { + span_invalid_monomorphization_error( + bx.sess(), span, + &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), + name, $($fmt)*)); + } + } + macro_rules! return_error { + ($($fmt: tt)*) => { + { + emit_error!($($fmt)*); + return Err(()); + } + } + } + + let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() { + let elem_ty = bx.cx.type_float_from_ty(*f); + match f.bit_width() { + 32 => ("f32", elem_ty), + 64 => ("f64", elem_ty), + _ => { + return_error!( + "unsupported element type `{}` of floating-point vector `{}`", + f.name_str(), + in_ty + ); + } + } + } else { + return_error!("`{}` is not a floating-point type", in_ty); + }; + + let vec_ty = bx.type_vector(elem_ty, in_len); + + let (intr_name, fn_ty) = match name { + sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), + sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)), + sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)), + sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)), + _ => return_error!("unrecognized intrinsic `{}`", name), + }; + let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); + let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty); + let c = + bx.call(fn_ty, f, &args.iter().map(|arg| arg.immediate()).collect::>(), None); + Ok(c) + } + + if std::matches!( + name, + sym::simd_ceil + | sym::simd_fabs + | sym::simd_fcos + | sym::simd_fexp2 + | sym::simd_fexp + | sym::simd_flog10 + | sym::simd_flog2 + | sym::simd_flog + | sym::simd_floor + | sym::simd_fma + | sym::simd_fpow + | sym::simd_fpowi + | sym::simd_fsin + | sym::simd_fsqrt + | sym::simd_round + | sym::simd_trunc + ) { + return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args); + } + + // FIXME: use: + // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182 + // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81 + fn llvm_vector_str( + elem_ty: Ty<'_>, + vec_len: u64, + no_pointers: usize, + bx: &Builder<'_, '_, '_>, + ) -> String { + let p0s: String = "p0".repeat(no_pointers); + match *elem_ty.kind() { + ty::Int(v) => format!( + "v{}{}i{}", + vec_len, + p0s, + // Normalize to prevent crash if v: IntTy::Isize + v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() + ), + ty::Uint(v) => format!( + "v{}{}i{}", + vec_len, + p0s, + // Normalize to prevent crash if v: UIntTy::Usize + v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() + ), + ty::Float(v) => format!("v{}{}f{}", vec_len, p0s, v.bit_width()), + _ => unreachable!(), + } + } + + fn llvm_vector_ty<'ll>( + cx: &CodegenCx<'ll, '_>, + elem_ty: Ty<'_>, + vec_len: u64, + mut no_pointers: usize, + ) -> &'ll Type { + // FIXME: use cx.layout_of(ty).llvm_type() ? + let mut elem_ty = match *elem_ty.kind() { + ty::Int(v) => cx.type_int_from_ty(v), + ty::Uint(v) => cx.type_uint_from_ty(v), + ty::Float(v) => cx.type_float_from_ty(v), + _ => unreachable!(), + }; + while no_pointers > 0 { + elem_ty = cx.type_ptr_to(elem_ty); + no_pointers -= 1; + } + cx.type_vector(elem_ty, vec_len) + } + + if name == sym::simd_gather { + // simd_gather(values: , pointers: , + // mask: ) -> + // * N: number of elements in the input vectors + // * T: type of the element to load + // * M: any integer width is supported, will be truncated to i1 + + // All types must be simd vector types + require_simd!(in_ty, "first"); + require_simd!(arg_tys[1], "second"); + require_simd!(arg_tys[2], "third"); + require_simd!(ret_ty, "return"); + + // Of the same length: + let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "second", + in_len, + in_ty, + arg_tys[1], + out_len + ); + require!( + in_len == out_len2, + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "third", + in_len, + in_ty, + arg_tys[2], + out_len2 + ); + + // The return type must match the first argument type + require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty); + + // This counts how many pointers + fn ptr_count(t: Ty<'_>) -> usize { + match t.kind() { + ty::RawPtr(p) => 1 + ptr_count(p.ty), + _ => 0, + } + } + + // Non-ptr type + fn non_ptr(t: Ty<'_>) -> Ty<'_> { + match t.kind() { + ty::RawPtr(p) => non_ptr(p.ty), + _ => t, + } + } + + // The second argument must be a simd vector with an element type that's a pointer + // to the element type of the first argument + let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); + let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (pointer_count, underlying_ty) = match element_ty1.kind() { + ty::RawPtr(p) if p.ty == in_elem => (ptr_count(element_ty1), non_ptr(element_ty1)), + _ => { + require!( + false, + "expected element type `{}` of second argument `{}` \ + to be a pointer to the element type `{}` of the first \ + argument `{}`, found `{}` != `*_ {}`", + element_ty1, + arg_tys[1], + in_elem, + in_ty, + element_ty1, + in_elem + ); + unreachable!(); + } + }; + assert!(pointer_count > 0); + assert_eq!(pointer_count - 1, ptr_count(element_ty0)); + assert_eq!(underlying_ty, non_ptr(element_ty0)); + + // The element type of the third argument must be a signed integer type of any width: + let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); + match element_ty2.kind() { + ty::Int(_) => (), + _ => { + require!( + false, + "expected element type `{}` of third argument `{}` \ + to be a signed integer type", + element_ty2, + arg_tys[2] + ); + } + } + + // Alignment of T, must be a constant integer value: + let alignment_ty = bx.type_i32(); + let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); + + // Truncate the mask vector to a vector of i1s: + let (mask, mask_ty) = { + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, in_len); + (bx.trunc(args[2].immediate(), i1xn), i1xn) + }; + + // Type of the vector of pointers: + let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); + let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx); + + // Type of the vector of elements: + let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); + let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx); + + let llvm_intrinsic = + format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); + let fn_ty = bx.type_func( + &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], + llvm_elem_vec_ty, + ); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = + bx.call(fn_ty, f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None); + return Ok(v); + } + + if name == sym::simd_scatter { + // simd_scatter(values: , pointers: , + // mask: ) -> () + // * N: number of elements in the input vectors + // * T: type of the element to load + // * M: any integer width is supported, will be truncated to i1 + + // All types must be simd vector types + require_simd!(in_ty, "first"); + require_simd!(arg_tys[1], "second"); + require_simd!(arg_tys[2], "third"); + + // Of the same length: + let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); + require!( + in_len == element_len1, + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "second", + in_len, + in_ty, + arg_tys[1], + element_len1 + ); + require!( + in_len == element_len2, + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "third", + in_len, + in_ty, + arg_tys[2], + element_len2 + ); + + // This counts how many pointers + fn ptr_count(t: Ty<'_>) -> usize { + match t.kind() { + ty::RawPtr(p) => 1 + ptr_count(p.ty), + _ => 0, + } + } + + // Non-ptr type + fn non_ptr(t: Ty<'_>) -> Ty<'_> { + match t.kind() { + ty::RawPtr(p) => non_ptr(p.ty), + _ => t, + } + } + + // The second argument must be a simd vector with an element type that's a pointer + // to the element type of the first argument + let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); + let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); + let (pointer_count, underlying_ty) = match element_ty1.kind() { + ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => { + (ptr_count(element_ty1), non_ptr(element_ty1)) + } + _ => { + require!( + false, + "expected element type `{}` of second argument `{}` \ + to be a pointer to the element type `{}` of the first \ + argument `{}`, found `{}` != `*mut {}`", + element_ty1, + arg_tys[1], + in_elem, + in_ty, + element_ty1, + in_elem + ); + unreachable!(); + } + }; + assert!(pointer_count > 0); + assert_eq!(pointer_count - 1, ptr_count(element_ty0)); + assert_eq!(underlying_ty, non_ptr(element_ty0)); + + // The element type of the third argument must be a signed integer type of any width: + match element_ty2.kind() { + ty::Int(_) => (), + _ => { + require!( + false, + "expected element type `{}` of third argument `{}` \ + be a signed integer type", + element_ty2, + arg_tys[2] + ); + } + } + + // Alignment of T, must be a constant integer value: + let alignment_ty = bx.type_i32(); + let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); + + // Truncate the mask vector to a vector of i1s: + let (mask, mask_ty) = { + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, in_len); + (bx.trunc(args[2].immediate(), i1xn), i1xn) + }; + + let ret_t = bx.type_void(); + + // Type of the vector of pointers: + let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); + let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx); + + // Type of the vector of elements: + let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); + let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx); + + let llvm_intrinsic = + format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); + let fn_ty = + bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = + bx.call(fn_ty, f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None); + return Ok(v); + } + + macro_rules! arith_red { + ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident, + $identity:expr) => { + if name == sym::$name { + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); + return match in_elem.kind() { + ty::Int(_) | ty::Uint(_) => { + let r = bx.$integer_reduce(args[0].immediate()); + if $ordered { + // if overflow occurs, the result is the + // mathematical result modulo 2^n: + Ok(bx.$op(args[1].immediate(), r)) + } else { + Ok(bx.$integer_reduce(args[0].immediate())) + } + } + ty::Float(f) => { + let acc = if $ordered { + // ordered arithmetic reductions take an accumulator + args[1].immediate() + } else { + // unordered arithmetic reductions use the identity accumulator + match f.bit_width() { + 32 => bx.const_real(bx.type_f32(), $identity), + 64 => bx.const_real(bx.type_f64(), $identity), + v => return_error!( + r#" +unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, + sym::$name, + in_ty, + in_elem, + v, + ret_ty + ), + } + }; + Ok(bx.$float_reduce(acc, args[0].immediate())) + } + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + sym::$name, + in_ty, + in_elem, + ret_ty + ), + }; + } + }; + } + + arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, 0.0); + arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); + arith_red!( + simd_reduce_add_unordered: vector_reduce_add, + vector_reduce_fadd_fast, + false, + add, + 0.0 + ); + arith_red!( + simd_reduce_mul_unordered: vector_reduce_mul, + vector_reduce_fmul_fast, + false, + mul, + 1.0 + ); + + macro_rules! minmax_red { + ($name:ident: $int_red:ident, $float_red:ident) => { + if name == sym::$name { + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); + return match in_elem.kind() { + ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)), + ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)), + ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())), + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + sym::$name, + in_ty, + in_elem, + ret_ty + ), + }; + } + }; + } + + minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin); + minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax); + + minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast); + minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast); + + macro_rules! bitwise_red { + ($name:ident : $red:ident, $boolean:expr) => { + if name == sym::$name { + let input = if !$boolean { + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); + args[0].immediate() + } else { + match in_elem.kind() { + ty::Int(_) | ty::Uint(_) => {} + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + sym::$name, + in_ty, + in_elem, + ret_ty + ), + } + + // boolean reductions operate on vectors of i1s: + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, in_len as u64); + bx.trunc(args[0].immediate(), i1xn) + }; + return match in_elem.kind() { + ty::Int(_) | ty::Uint(_) => { + let r = bx.$red(input); + Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) }) + } + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + sym::$name, + in_ty, + in_elem, + ret_ty + ), + }; + } + }; + } + + bitwise_red!(simd_reduce_and: vector_reduce_and, false); + bitwise_red!(simd_reduce_or: vector_reduce_or, false); + bitwise_red!(simd_reduce_xor: vector_reduce_xor, false); + bitwise_red!(simd_reduce_all: vector_reduce_and, true); + bitwise_red!(simd_reduce_any: vector_reduce_or, true); + + if name == sym::simd_cast || name == sym::simd_as { + require_simd!(ret_ty, "return"); + let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + "expected return type with length {} (same as input type `{}`), \ + found `{}` with length {}", + in_len, + in_ty, + ret_ty, + out_len + ); + // casting cares about nominal type, not just structural type + if in_elem == out_elem { + return Ok(args[0].immediate()); + } + + enum Style { + Float, + Int(/* is signed? */ bool), + Unsupported, + } + + let (in_style, in_width) = match in_elem.kind() { + // vectors of pointer-sized integers should've been + // disallowed before here, so this unwrap is safe. + ty::Int(i) => ( + Style::Int(true), + i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Uint(u) => ( + Style::Int(false), + u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Float(f) => (Style::Float, f.bit_width()), + _ => (Style::Unsupported, 0), + }; + let (out_style, out_width) = match out_elem.kind() { + ty::Int(i) => ( + Style::Int(true), + i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Uint(u) => ( + Style::Int(false), + u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Float(f) => (Style::Float, f.bit_width()), + _ => (Style::Unsupported, 0), + }; + + match (in_style, out_style) { + (Style::Int(in_is_signed), Style::Int(_)) => { + return Ok(match in_width.cmp(&out_width) { + Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty), + Ordering::Equal => args[0].immediate(), + Ordering::Less => { + if in_is_signed { + bx.sext(args[0].immediate(), llret_ty) + } else { + bx.zext(args[0].immediate(), llret_ty) + } + } + }); + } + (Style::Int(in_is_signed), Style::Float) => { + return Ok(if in_is_signed { + bx.sitofp(args[0].immediate(), llret_ty) + } else { + bx.uitofp(args[0].immediate(), llret_ty) + }); + } + (Style::Float, Style::Int(out_is_signed)) => { + return Ok(match (out_is_signed, name == sym::simd_as) { + (false, false) => bx.fptoui(args[0].immediate(), llret_ty), + (true, false) => bx.fptosi(args[0].immediate(), llret_ty), + (_, true) => bx.cast_float_to_int(out_is_signed, args[0].immediate(), llret_ty), + }); + } + (Style::Float, Style::Float) => { + return Ok(match in_width.cmp(&out_width) { + Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty), + Ordering::Equal => args[0].immediate(), + Ordering::Less => bx.fpext(args[0].immediate(), llret_ty), + }); + } + _ => { /* Unsupported. Fallthrough. */ } + } + require!( + false, + "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", + in_ty, + in_elem, + ret_ty, + out_elem + ); + } + macro_rules! arith_binary { + ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { + $(if name == sym::$name { + match in_elem.kind() { + $($(ty::$p(_))|* => { + return Ok(bx.$call(args[0].immediate(), args[1].immediate())) + })* + _ => {}, + } + require!(false, + "unsupported operation on `{}` with element `{}`", + in_ty, + in_elem) + })* + } + } + arith_binary! { + simd_add: Uint, Int => add, Float => fadd; + simd_sub: Uint, Int => sub, Float => fsub; + simd_mul: Uint, Int => mul, Float => fmul; + simd_div: Uint => udiv, Int => sdiv, Float => fdiv; + simd_rem: Uint => urem, Int => srem, Float => frem; + simd_shl: Uint, Int => shl; + simd_shr: Uint => lshr, Int => ashr; + simd_and: Uint, Int => and; + simd_or: Uint, Int => or; + simd_xor: Uint, Int => xor; + simd_fmax: Float => maxnum; + simd_fmin: Float => minnum; + + } + macro_rules! arith_unary { + ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { + $(if name == sym::$name { + match in_elem.kind() { + $($(ty::$p(_))|* => { + return Ok(bx.$call(args[0].immediate())) + })* + _ => {}, + } + require!(false, + "unsupported operation on `{}` with element `{}`", + in_ty, + in_elem) + })* + } + } + arith_unary! { + simd_neg: Int => neg, Float => fneg; + } + + if name == sym::simd_arith_offset { + // This also checks that the first operand is a ptr type. + let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| { + span_bug!(span, "must be called with a vector of pointer types as first argument") + }); + let layout = bx.layout_of(pointee.ty); + let ptrs = args[0].immediate(); + // The second argument must be a ptr-sized integer. + // (We don't care about the signedness, this is wrapping anyway.) + let (_offsets_len, offsets_elem) = arg_tys[1].simd_size_and_type(bx.tcx()); + if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) { + span_bug!( + span, + "must be called with a vector of pointer-sized integers as second argument" + ); + } + let offsets = args[1].immediate(); + + return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets])); + } + + if name == sym::simd_saturating_add || name == sym::simd_saturating_sub { + let lhs = args[0].immediate(); + let rhs = args[1].immediate(); + let is_add = name == sym::simd_saturating_add; + let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _; + let (signed, elem_width, elem_ty) = match *in_elem.kind() { + ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), + ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), + _ => { + return_error!( + "expected element type `{}` of vector type `{}` \ + to be a signed or unsigned integer type", + arg_tys[0].simd_size_and_type(bx.tcx()).1, + arg_tys[0] + ); + } + }; + let llvm_intrinsic = &format!( + "llvm.{}{}.sat.v{}i{}", + if signed { 's' } else { 'u' }, + if is_add { "add" } else { "sub" }, + in_len, + elem_width + ); + let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); + + let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = bx.call(fn_ty, f, &[lhs, rhs], None); + return Ok(v); + } + + span_bug!(span, "unknown SIMD intrinsic"); +} + +// Returns the width of an int Ty, and if it's signed or not +// Returns None if the type is not an integer +// FIXME: there’s multiple of this functions, investigate using some of the already existing +// stuffs. +fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> { + match ty.kind() { + ty::Int(t) => { + Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), true)) + } + ty::Uint(t) => { + Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), false)) + } + _ => None, + } +} diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs new file mode 100644 index 00000000000..636d689a34b --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -0,0 +1,442 @@ +//! The Rust compiler. +//! +//! # Note +//! +//! This API is completely unstable and subject to change. + +#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] +#![feature(hash_raw_entry)] +#![feature(let_chains)] +#![feature(let_else)] +#![feature(extern_types)] +#![feature(once_cell)] +#![feature(iter_intersperse)] +#![recursion_limit = "256"] +#![allow(rustc::potential_query_instability)] + +#[macro_use] +extern crate rustc_macros; + +use back::write::{create_informational_target_machine, create_target_machine}; + +pub use llvm_util::target_features; +use rustc_ast::expand::allocator::AllocatorKind; +use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; +use rustc_codegen_ssa::back::write::{ + CodegenContext, FatLTOInput, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, +}; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::ModuleCodegen; +use rustc_codegen_ssa::{CodegenResults, CompiledModule}; +use rustc_data_structures::fx::FxHashMap; +use rustc_errors::{ErrorGuaranteed, FatalError, Handler}; +use rustc_metadata::EncodedMetadata; +use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::ty::query::Providers; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; +use rustc_session::Session; +use rustc_span::symbol::Symbol; + +use std::any::Any; +use std::ffi::CStr; + +mod back { + pub mod archive; + pub mod lto; + mod profiling; + pub mod write; +} + +mod abi; +mod allocator; +mod asm; +mod attributes; +mod base; +mod builder; +mod callee; +mod common; +mod consts; +mod context; +mod coverageinfo; +mod debuginfo; +mod declare; +mod intrinsic; + +// The following is a work around that replaces `pub mod llvm;` and that fixes issue 53912. +#[path = "llvm/mod.rs"] +mod llvm_; +pub mod llvm { + pub use super::llvm_::*; +} + +mod llvm_util; +mod mono_item; +mod type_; +mod type_of; +mod va_arg; +mod value; + +#[derive(Clone)] +pub struct LlvmCodegenBackend(()); + +struct TimeTraceProfiler { + enabled: bool, +} + +impl TimeTraceProfiler { + fn new(enabled: bool) -> Self { + if enabled { + unsafe { llvm::LLVMTimeTraceProfilerInitialize() } + } + TimeTraceProfiler { enabled } + } +} + +impl Drop for TimeTraceProfiler { + fn drop(&mut self) { + if self.enabled { + unsafe { llvm::LLVMTimeTraceProfilerFinishThread() } + } + } +} + +impl ExtraBackendMethods for LlvmCodegenBackend { + fn codegen_allocator<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + module_name: &str, + kind: AllocatorKind, + has_alloc_error_handler: bool, + ) -> ModuleLlvm { + let mut module_llvm = ModuleLlvm::new_metadata(tcx, module_name); + unsafe { + allocator::codegen(tcx, &mut module_llvm, module_name, kind, has_alloc_error_handler); + } + module_llvm + } + fn compile_codegen_unit( + &self, + tcx: TyCtxt<'_>, + cgu_name: Symbol, + ) -> (ModuleCodegen, u64) { + base::compile_codegen_unit(tcx, cgu_name) + } + fn target_machine_factory( + &self, + sess: &Session, + optlvl: OptLevel, + target_features: &[String], + ) -> TargetMachineFactoryFn { + back::write::target_machine_factory(sess, optlvl, target_features) + } + fn target_cpu<'b>(&self, sess: &'b Session) -> &'b str { + llvm_util::target_cpu(sess) + } + fn tune_cpu<'b>(&self, sess: &'b Session) -> Option<&'b str> { + llvm_util::tune_cpu(sess) + } + + fn spawn_thread(time_trace: bool, f: F) -> std::thread::JoinHandle + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send + 'static, + { + std::thread::spawn(move || { + let _profiler = TimeTraceProfiler::new(time_trace); + f() + }) + } + + fn spawn_named_thread( + time_trace: bool, + name: String, + f: F, + ) -> std::io::Result> + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send + 'static, + { + std::thread::Builder::new().name(name).spawn(move || { + let _profiler = TimeTraceProfiler::new(time_trace); + f() + }) + } +} + +impl WriteBackendMethods for LlvmCodegenBackend { + type Module = ModuleLlvm; + type ModuleBuffer = back::lto::ModuleBuffer; + type Context = llvm::Context; + type TargetMachine = &'static mut llvm::TargetMachine; + type ThinData = back::lto::ThinData; + type ThinBuffer = back::lto::ThinBuffer; + fn print_pass_timings(&self) { + unsafe { + llvm::LLVMRustPrintPassTimings(); + } + } + fn run_link( + cgcx: &CodegenContext, + diag_handler: &Handler, + modules: Vec>, + ) -> Result, FatalError> { + back::write::link(cgcx, diag_handler, modules) + } + fn run_fat_lto( + cgcx: &CodegenContext, + modules: Vec>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + ) -> Result, FatalError> { + back::lto::run_fat(cgcx, modules, cached_modules) + } + fn run_thin_lto( + cgcx: &CodegenContext, + modules: Vec<(String, Self::ThinBuffer)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + ) -> Result<(Vec>, Vec), FatalError> { + back::lto::run_thin(cgcx, modules, cached_modules) + } + unsafe fn optimize( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: &ModuleCodegen, + config: &ModuleConfig, + ) -> Result<(), FatalError> { + back::write::optimize(cgcx, diag_handler, module, config) + } + fn optimize_fat( + cgcx: &CodegenContext, + module: &mut ModuleCodegen, + ) -> Result<(), FatalError> { + let diag_handler = cgcx.create_diag_handler(); + back::lto::run_pass_manager(cgcx, &diag_handler, module, false) + } + unsafe fn optimize_thin( + cgcx: &CodegenContext, + thin: ThinModule, + ) -> Result, FatalError> { + back::lto::optimize_thin_module(thin, cgcx) + } + unsafe fn codegen( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: ModuleCodegen, + config: &ModuleConfig, + ) -> Result { + back::write::codegen(cgcx, diag_handler, module, config) + } + fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer) { + back::lto::prepare_thin(module) + } + fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer) { + (module.name, back::lto::ModuleBuffer::new(module.module_llvm.llmod())) + } +} + +unsafe impl Send for LlvmCodegenBackend {} // Llvm is on a per-thread basis +unsafe impl Sync for LlvmCodegenBackend {} + +impl LlvmCodegenBackend { + pub fn new() -> Box { + Box::new(LlvmCodegenBackend(())) + } +} + +impl CodegenBackend for LlvmCodegenBackend { + fn init(&self, sess: &Session) { + llvm_util::init(sess); // Make sure llvm is inited + } + + fn provide(&self, providers: &mut Providers) { + providers.global_backend_features = + |tcx, ()| llvm_util::global_llvm_features(tcx.sess, true) + } + + fn print(&self, req: PrintRequest, sess: &Session) { + match req { + PrintRequest::RelocationModels => { + println!("Available relocation models:"); + for name in &[ + "static", + "pic", + "pie", + "dynamic-no-pic", + "ropi", + "rwpi", + "ropi-rwpi", + "default", + ] { + println!(" {}", name); + } + println!(); + } + PrintRequest::CodeModels => { + println!("Available code models:"); + for name in &["tiny", "small", "kernel", "medium", "large"] { + println!(" {}", name); + } + println!(); + } + PrintRequest::TlsModels => { + println!("Available TLS models:"); + for name in &["global-dynamic", "local-dynamic", "initial-exec", "local-exec"] { + println!(" {}", name); + } + println!(); + } + PrintRequest::StackProtectorStrategies => { + println!( + r#"Available stack protector strategies: + all + Generate stack canaries in all functions. + + strong + Generate stack canaries in a function if it either: + - has a local variable of `[T; N]` type, regardless of `T` and `N` + - takes the address of a local variable. + + (Note that a local variable being borrowed is not equivalent to its + address being taken: e.g. some borrows may be removed by optimization, + while by-value argument passing may be implemented with reference to a + local stack variable in the ABI.) + + basic + Generate stack canaries in functions with local variables of `[T; N]` + type, where `T` is byte-sized and `N` >= 8. + + none + Do not generate stack canaries. +"# + ); + } + req => llvm_util::print(req, sess), + } + } + + fn print_passes(&self) { + llvm_util::print_passes(); + } + + fn print_version(&self) { + llvm_util::print_version(); + } + + fn target_features(&self, sess: &Session, allow_unstable: bool) -> Vec { + target_features(sess, allow_unstable) + } + + fn codegen_crate<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + metadata: EncodedMetadata, + need_metadata_module: bool, + ) -> Box { + Box::new(rustc_codegen_ssa::base::codegen_crate( + LlvmCodegenBackend(()), + tcx, + crate::llvm_util::target_cpu(tcx.sess).to_string(), + metadata, + need_metadata_module, + )) + } + + fn join_codegen( + &self, + ongoing_codegen: Box, + sess: &Session, + outputs: &OutputFilenames, + ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed> { + let (codegen_results, work_products) = ongoing_codegen + .downcast::>() + .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box") + .join(sess); + + sess.time("llvm_dump_timing_file", || { + if sess.opts.unstable_opts.llvm_time_trace { + let file_name = outputs.with_extension("llvm_timings.json"); + llvm_util::time_trace_profiler_finish(&file_name); + } + }); + + Ok((codegen_results, work_products)) + } + + fn link( + &self, + sess: &Session, + codegen_results: CodegenResults, + outputs: &OutputFilenames, + ) -> Result<(), ErrorGuaranteed> { + use crate::back::archive::LlvmArchiveBuilderBuilder; + use rustc_codegen_ssa::back::link::link_binary; + + // Run the linker on any artifacts that resulted from the LLVM run. + // This should produce either a finished executable or library. + link_binary(sess, &LlvmArchiveBuilderBuilder, &codegen_results, outputs) + } +} + +pub struct ModuleLlvm { + llcx: &'static mut llvm::Context, + llmod_raw: *const llvm::Module, + tm: &'static mut llvm::TargetMachine, +} + +unsafe impl Send for ModuleLlvm {} +unsafe impl Sync for ModuleLlvm {} + +impl ModuleLlvm { + fn new(tcx: TyCtxt<'_>, mod_name: &str) -> Self { + unsafe { + let llcx = llvm::LLVMRustContextCreate(tcx.sess.fewer_names()); + let llmod_raw = context::create_module(tcx, llcx, mod_name) as *const _; + ModuleLlvm { llmod_raw, llcx, tm: create_target_machine(tcx, mod_name) } + } + } + + fn new_metadata(tcx: TyCtxt<'_>, mod_name: &str) -> Self { + unsafe { + let llcx = llvm::LLVMRustContextCreate(tcx.sess.fewer_names()); + let llmod_raw = context::create_module(tcx, llcx, mod_name) as *const _; + ModuleLlvm { llmod_raw, llcx, tm: create_informational_target_machine(tcx.sess) } + } + } + + fn parse( + cgcx: &CodegenContext, + name: &CStr, + buffer: &[u8], + handler: &Handler, + ) -> Result { + unsafe { + let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); + let llmod_raw = back::lto::parse_module(llcx, name, buffer, handler)?; + let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, name.to_str().unwrap()); + let tm = match (cgcx.tm_factory)(tm_factory_config) { + Ok(m) => m, + Err(e) => { + handler.struct_err(&e).emit(); + return Err(FatalError); + } + }; + + Ok(ModuleLlvm { llmod_raw, llcx, tm }) + } + } + + fn llmod(&self) -> &llvm::Module { + unsafe { &*self.llmod_raw } + } +} + +impl Drop for ModuleLlvm { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustDisposeTargetMachine(&mut *(self.tm as *mut _)); + llvm::LLVMContextDispose(&mut *(self.llcx as *mut _)); + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs new file mode 100644 index 00000000000..64db4f7462d --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs @@ -0,0 +1,105 @@ +//! A wrapper around LLVM's archive (.a) code + +use rustc_fs_util::path_to_c_string; +use std::path::Path; +use std::slice; +use std::str; + +pub struct ArchiveRO { + pub raw: &'static mut super::Archive, +} + +unsafe impl Send for ArchiveRO {} + +pub struct Iter<'a> { + raw: &'a mut super::ArchiveIterator<'a>, +} + +pub struct Child<'a> { + pub raw: &'a mut super::ArchiveChild<'a>, +} + +impl ArchiveRO { + /// Opens a static archive for read-only purposes. This is more optimized + /// than the `open` method because it uses LLVM's internal `Archive` class + /// rather than shelling out to `ar` for everything. + /// + /// If this archive is used with a mutable method, then an error will be + /// raised. + pub fn open(dst: &Path) -> Result { + unsafe { + let s = path_to_c_string(dst); + let ar = super::LLVMRustOpenArchive(s.as_ptr()).ok_or_else(|| { + super::last_error().unwrap_or_else(|| "failed to open archive".to_owned()) + })?; + Ok(ArchiveRO { raw: ar }) + } + } + + pub fn iter(&self) -> Iter<'_> { + unsafe { Iter { raw: super::LLVMRustArchiveIteratorNew(self.raw) } } + } +} + +impl Drop for ArchiveRO { + fn drop(&mut self) { + unsafe { + super::LLVMRustDestroyArchive(&mut *(self.raw as *mut _)); + } + } +} + +impl<'a> Iterator for Iter<'a> { + type Item = Result, String>; + + fn next(&mut self) -> Option, String>> { + unsafe { + match super::LLVMRustArchiveIteratorNext(self.raw) { + Some(raw) => Some(Ok(Child { raw })), + None => super::last_error().map(Err), + } + } + } +} + +impl<'a> Drop for Iter<'a> { + fn drop(&mut self) { + unsafe { + super::LLVMRustArchiveIteratorFree(&mut *(self.raw as *mut _)); + } + } +} + +impl<'a> Child<'a> { + pub fn name(&self) -> Option<&'a str> { + unsafe { + let mut name_len = 0; + let name_ptr = super::LLVMRustArchiveChildName(self.raw, &mut name_len); + if name_ptr.is_null() { + None + } else { + let name = slice::from_raw_parts(name_ptr as *const u8, name_len as usize); + str::from_utf8(name).ok().map(|s| s.trim()) + } + } + } + + pub fn data(&self) -> &'a [u8] { + unsafe { + let mut data_len = 0; + let data_ptr = super::LLVMRustArchiveChildData(self.raw, &mut data_len); + if data_ptr.is_null() { + panic!("failed to read data from archive child"); + } + slice::from_raw_parts(data_ptr as *const u8, data_len as usize) + } + } +} + +impl<'a> Drop for Child<'a> { + fn drop(&mut self) { + unsafe { + super::LLVMRustArchiveChildFree(&mut *(self.raw as *mut _)); + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs new file mode 100644 index 00000000000..45de284d22a --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -0,0 +1,213 @@ +//! LLVM diagnostic reports. + +pub use self::Diagnostic::*; +pub use self::OptimizationDiagnosticKind::*; + +use crate::value::Value; +use libc::c_uint; + +use super::{DiagnosticInfo, SMDiagnostic}; +use rustc_span::InnerSpan; + +#[derive(Copy, Clone)] +pub enum OptimizationDiagnosticKind { + OptimizationRemark, + OptimizationMissed, + OptimizationAnalysis, + OptimizationAnalysisFPCommute, + OptimizationAnalysisAliasing, + OptimizationFailure, + OptimizationRemarkOther, +} + +pub struct OptimizationDiagnostic<'ll> { + pub kind: OptimizationDiagnosticKind, + pub pass_name: String, + pub function: &'ll Value, + pub line: c_uint, + pub column: c_uint, + pub filename: String, + pub message: String, +} + +impl<'ll> OptimizationDiagnostic<'ll> { + unsafe fn unpack(kind: OptimizationDiagnosticKind, di: &'ll DiagnosticInfo) -> Self { + let mut function = None; + let mut line = 0; + let mut column = 0; + + let mut message = None; + let mut filename = None; + let pass_name = super::build_string(|pass_name| { + message = super::build_string(|message| { + filename = super::build_string(|filename| { + super::LLVMRustUnpackOptimizationDiagnostic( + di, + pass_name, + &mut function, + &mut line, + &mut column, + filename, + message, + ) + }) + .ok() + }) + .ok() + }) + .ok(); + + let mut filename = filename.unwrap_or_default(); + if filename.is_empty() { + filename.push_str(""); + } + + OptimizationDiagnostic { + kind, + pass_name: pass_name.expect("got a non-UTF8 pass name from LLVM"), + function: function.unwrap(), + line, + column, + filename, + message: message.expect("got a non-UTF8 OptimizationDiagnostic message from LLVM"), + } + } +} + +pub struct SrcMgrDiagnostic { + pub level: super::DiagnosticLevel, + pub message: String, + pub source: Option<(String, Vec)>, +} + +impl SrcMgrDiagnostic { + pub unsafe fn unpack(diag: &SMDiagnostic) -> SrcMgrDiagnostic { + // Recover the post-substitution assembly code from LLVM for better + // diagnostics. + let mut have_source = false; + let mut buffer = String::new(); + let mut level = super::DiagnosticLevel::Error; + let mut loc = 0; + let mut ranges = [0; 8]; + let mut num_ranges = ranges.len() / 2; + let message = super::build_string(|message| { + buffer = super::build_string(|buffer| { + have_source = super::LLVMRustUnpackSMDiagnostic( + diag, + message, + buffer, + &mut level, + &mut loc, + ranges.as_mut_ptr(), + &mut num_ranges, + ); + }) + .expect("non-UTF8 inline asm"); + }) + .expect("non-UTF8 SMDiagnostic"); + + SrcMgrDiagnostic { + message, + level, + source: have_source.then(|| { + let mut spans = vec![InnerSpan::new(loc as usize, loc as usize)]; + for i in 0..num_ranges { + spans.push(InnerSpan::new(ranges[i * 2] as usize, ranges[i * 2 + 1] as usize)); + } + (buffer, spans) + }), + } + } +} + +#[derive(Clone)] +pub struct InlineAsmDiagnostic { + pub level: super::DiagnosticLevel, + pub cookie: c_uint, + pub message: String, + pub source: Option<(String, Vec)>, +} + +impl InlineAsmDiagnostic { + unsafe fn unpackInlineAsm(di: &DiagnosticInfo) -> Self { + let mut cookie = 0; + let mut message = None; + let mut level = super::DiagnosticLevel::Error; + + super::LLVMRustUnpackInlineAsmDiagnostic(di, &mut level, &mut cookie, &mut message); + + InlineAsmDiagnostic { + level, + cookie, + message: super::twine_to_string(message.unwrap()), + source: None, + } + } + + unsafe fn unpackSrcMgr(di: &DiagnosticInfo) -> Self { + let mut cookie = 0; + let smdiag = SrcMgrDiagnostic::unpack(super::LLVMRustGetSMDiagnostic(di, &mut cookie)); + InlineAsmDiagnostic { + level: smdiag.level, + cookie, + message: smdiag.message, + source: smdiag.source, + } + } +} + +pub enum Diagnostic<'ll> { + Optimization(OptimizationDiagnostic<'ll>), + InlineAsm(InlineAsmDiagnostic), + PGO(&'ll DiagnosticInfo), + Linker(&'ll DiagnosticInfo), + Unsupported(&'ll DiagnosticInfo), + + /// LLVM has other types that we do not wrap here. + UnknownDiagnostic(&'ll DiagnosticInfo), +} + +impl<'ll> Diagnostic<'ll> { + pub unsafe fn unpack(di: &'ll DiagnosticInfo) -> Self { + use super::DiagnosticKind as Dk; + let kind = super::LLVMRustGetDiagInfoKind(di); + + match kind { + Dk::InlineAsm => InlineAsm(InlineAsmDiagnostic::unpackInlineAsm(di)), + + Dk::OptimizationRemark => { + Optimization(OptimizationDiagnostic::unpack(OptimizationRemark, di)) + } + Dk::OptimizationRemarkOther => { + Optimization(OptimizationDiagnostic::unpack(OptimizationRemarkOther, di)) + } + Dk::OptimizationRemarkMissed => { + Optimization(OptimizationDiagnostic::unpack(OptimizationMissed, di)) + } + + Dk::OptimizationRemarkAnalysis => { + Optimization(OptimizationDiagnostic::unpack(OptimizationAnalysis, di)) + } + + Dk::OptimizationRemarkAnalysisFPCommute => { + Optimization(OptimizationDiagnostic::unpack(OptimizationAnalysisFPCommute, di)) + } + + Dk::OptimizationRemarkAnalysisAliasing => { + Optimization(OptimizationDiagnostic::unpack(OptimizationAnalysisAliasing, di)) + } + + Dk::OptimizationFailure => { + Optimization(OptimizationDiagnostic::unpack(OptimizationFailure, di)) + } + + Dk::PGOProfile => PGO(di), + Dk::Linker => Linker(di), + Dk::Unsupported => Unsupported(di), + + Dk::SrcMgr => InlineAsm(InlineAsmDiagnostic::unpackSrcMgr(di)), + + _ => UnknownDiagnostic(di), + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs new file mode 100644 index 00000000000..3139f93bfef --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -0,0 +1,2547 @@ +#![allow(non_camel_case_types)] +#![allow(non_upper_case_globals)] + +use rustc_codegen_ssa::coverageinfo::map as coverage_map; + +use super::debuginfo::{ + DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, + DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DILocation, DINameSpace, + DISPFlags, DIScope, DISubprogram, DISubrange, DITemplateTypeParameter, DIType, DIVariable, + DebugEmissionKind, +}; + +use libc::{c_char, c_int, c_uint, size_t}; +use libc::{c_ulonglong, c_void}; + +use std::marker::PhantomData; + +use super::RustString; + +pub type Bool = c_uint; + +pub const True: Bool = 1 as Bool; +pub const False: Bool = 0 as Bool; + +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +#[allow(dead_code)] // Variants constructed by C++. +pub enum LLVMRustResult { + Success, + Failure, +} + +// Rust version of the C struct with the same name in rustc_llvm/llvm-wrapper/RustWrapper.cpp. +#[repr(C)] +pub struct LLVMRustCOFFShortExport { + pub name: *const c_char, + pub ordinal_present: bool, + // value of `ordinal` only important when `ordinal_present` is true + pub ordinal: u16, +} + +impl LLVMRustCOFFShortExport { + pub fn new(name: *const c_char, ordinal: Option) -> LLVMRustCOFFShortExport { + LLVMRustCOFFShortExport { + name, + ordinal_present: ordinal.is_some(), + ordinal: ordinal.unwrap_or(0), + } + } +} + +/// Translation of LLVM's MachineTypes enum, defined in llvm\include\llvm\BinaryFormat\COFF.h. +/// +/// We include only architectures supported on Windows. +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum LLVMMachineType { + AMD64 = 0x8664, + I386 = 0x14c, + ARM64 = 0xaa64, + ARM = 0x01c0, +} + +/// LLVM's Module::ModFlagBehavior, defined in llvm/include/llvm/IR/Module.h. +/// +/// When merging modules (e.g. during LTO), their metadata flags are combined. Conflicts are +/// resolved according to the merge behaviors specified here. Flags differing only in merge +/// behavior are still considered to be in conflict. +/// +/// In order for Rust-C LTO to work, we must specify behaviors compatible with Clang. Notably, +/// 'Error' and 'Warning' cannot be mixed for a given flag. +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum LLVMModFlagBehavior { + Error = 1, + Warning = 2, + Require = 3, + Override = 4, + Append = 5, + AppendUnique = 6, + Max = 7, +} + +// Consts for the LLVM CallConv type, pre-cast to usize. + +/// LLVM CallingConv::ID. Should we wrap this? +#[derive(Copy, Clone, PartialEq, Debug)] +#[repr(C)] +pub enum CallConv { + CCallConv = 0, + FastCallConv = 8, + ColdCallConv = 9, + X86StdcallCallConv = 64, + X86FastcallCallConv = 65, + ArmAapcsCallConv = 67, + Msp430Intr = 69, + X86_ThisCall = 70, + PtxKernel = 71, + X86_64_SysV = 78, + X86_64_Win64 = 79, + X86_VectorCall = 80, + X86_Intr = 83, + AvrNonBlockingInterrupt = 84, + AvrInterrupt = 85, + AmdGpuKernel = 91, +} + +/// LLVMRustLinkage +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum Linkage { + ExternalLinkage = 0, + AvailableExternallyLinkage = 1, + LinkOnceAnyLinkage = 2, + LinkOnceODRLinkage = 3, + WeakAnyLinkage = 4, + WeakODRLinkage = 5, + AppendingLinkage = 6, + InternalLinkage = 7, + PrivateLinkage = 8, + ExternalWeakLinkage = 9, + CommonLinkage = 10, +} + +// LLVMRustVisibility +#[repr(C)] +#[derive(Copy, Clone, PartialEq)] +pub enum Visibility { + Default = 0, + Hidden = 1, + Protected = 2, +} + +/// LLVMUnnamedAddr +#[repr(C)] +pub enum UnnamedAddr { + No, + Local, + Global, +} + +/// LLVMDLLStorageClass +#[derive(Copy, Clone)] +#[repr(C)] +pub enum DLLStorageClass { + #[allow(dead_code)] + Default = 0, + DllImport = 1, // Function to be imported from DLL. + #[allow(dead_code)] + DllExport = 2, // Function to be accessible from DLL. +} + +/// Matches LLVMRustAttribute in LLVMWrapper.h +/// Semantically a subset of the C++ enum llvm::Attribute::AttrKind, +/// though it is not ABI compatible (since it's a C++ enum) +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub enum AttributeKind { + AlwaysInline = 0, + ByVal = 1, + Cold = 2, + InlineHint = 3, + MinSize = 4, + Naked = 5, + NoAlias = 6, + NoCapture = 7, + NoInline = 8, + NonNull = 9, + NoRedZone = 10, + NoReturn = 11, + NoUnwind = 12, + OptimizeForSize = 13, + ReadOnly = 14, + SExt = 15, + StructRet = 16, + UWTable = 17, + ZExt = 18, + InReg = 19, + SanitizeThread = 20, + SanitizeAddress = 21, + SanitizeMemory = 22, + NonLazyBind = 23, + OptimizeNone = 24, + ReturnsTwice = 25, + ReadNone = 26, + InaccessibleMemOnly = 27, + SanitizeHWAddress = 28, + WillReturn = 29, + StackProtectReq = 30, + StackProtectStrong = 31, + StackProtect = 32, + NoUndef = 33, + SanitizeMemTag = 34, + NoCfCheck = 35, + ShadowCallStack = 36, + AllocSize = 37, + AllocatedPointer = 38, + AllocAlign = 39, +} + +/// LLVMIntPredicate +#[derive(Copy, Clone)] +#[repr(C)] +pub enum IntPredicate { + IntEQ = 32, + IntNE = 33, + IntUGT = 34, + IntUGE = 35, + IntULT = 36, + IntULE = 37, + IntSGT = 38, + IntSGE = 39, + IntSLT = 40, + IntSLE = 41, +} + +impl IntPredicate { + pub fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self { + match intpre { + rustc_codegen_ssa::common::IntPredicate::IntEQ => IntPredicate::IntEQ, + rustc_codegen_ssa::common::IntPredicate::IntNE => IntPredicate::IntNE, + rustc_codegen_ssa::common::IntPredicate::IntUGT => IntPredicate::IntUGT, + rustc_codegen_ssa::common::IntPredicate::IntUGE => IntPredicate::IntUGE, + rustc_codegen_ssa::common::IntPredicate::IntULT => IntPredicate::IntULT, + rustc_codegen_ssa::common::IntPredicate::IntULE => IntPredicate::IntULE, + rustc_codegen_ssa::common::IntPredicate::IntSGT => IntPredicate::IntSGT, + rustc_codegen_ssa::common::IntPredicate::IntSGE => IntPredicate::IntSGE, + rustc_codegen_ssa::common::IntPredicate::IntSLT => IntPredicate::IntSLT, + rustc_codegen_ssa::common::IntPredicate::IntSLE => IntPredicate::IntSLE, + } + } +} + +/// LLVMRealPredicate +#[derive(Copy, Clone)] +#[repr(C)] +pub enum RealPredicate { + RealPredicateFalse = 0, + RealOEQ = 1, + RealOGT = 2, + RealOGE = 3, + RealOLT = 4, + RealOLE = 5, + RealONE = 6, + RealORD = 7, + RealUNO = 8, + RealUEQ = 9, + RealUGT = 10, + RealUGE = 11, + RealULT = 12, + RealULE = 13, + RealUNE = 14, + RealPredicateTrue = 15, +} + +impl RealPredicate { + pub fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self { + match realp { + rustc_codegen_ssa::common::RealPredicate::RealPredicateFalse => { + RealPredicate::RealPredicateFalse + } + rustc_codegen_ssa::common::RealPredicate::RealOEQ => RealPredicate::RealOEQ, + rustc_codegen_ssa::common::RealPredicate::RealOGT => RealPredicate::RealOGT, + rustc_codegen_ssa::common::RealPredicate::RealOGE => RealPredicate::RealOGE, + rustc_codegen_ssa::common::RealPredicate::RealOLT => RealPredicate::RealOLT, + rustc_codegen_ssa::common::RealPredicate::RealOLE => RealPredicate::RealOLE, + rustc_codegen_ssa::common::RealPredicate::RealONE => RealPredicate::RealONE, + rustc_codegen_ssa::common::RealPredicate::RealORD => RealPredicate::RealORD, + rustc_codegen_ssa::common::RealPredicate::RealUNO => RealPredicate::RealUNO, + rustc_codegen_ssa::common::RealPredicate::RealUEQ => RealPredicate::RealUEQ, + rustc_codegen_ssa::common::RealPredicate::RealUGT => RealPredicate::RealUGT, + rustc_codegen_ssa::common::RealPredicate::RealUGE => RealPredicate::RealUGE, + rustc_codegen_ssa::common::RealPredicate::RealULT => RealPredicate::RealULT, + rustc_codegen_ssa::common::RealPredicate::RealULE => RealPredicate::RealULE, + rustc_codegen_ssa::common::RealPredicate::RealUNE => RealPredicate::RealUNE, + rustc_codegen_ssa::common::RealPredicate::RealPredicateTrue => { + RealPredicate::RealPredicateTrue + } + } + } +} + +/// LLVMTypeKind +#[derive(Copy, Clone, PartialEq, Debug)] +#[repr(C)] +pub enum TypeKind { + Void = 0, + Half = 1, + Float = 2, + Double = 3, + X86_FP80 = 4, + FP128 = 5, + PPC_FP128 = 6, + Label = 7, + Integer = 8, + Function = 9, + Struct = 10, + Array = 11, + Pointer = 12, + Vector = 13, + Metadata = 14, + X86_MMX = 15, + Token = 16, + ScalableVector = 17, + BFloat = 18, + X86_AMX = 19, +} + +impl TypeKind { + pub fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind { + match self { + TypeKind::Void => rustc_codegen_ssa::common::TypeKind::Void, + TypeKind::Half => rustc_codegen_ssa::common::TypeKind::Half, + TypeKind::Float => rustc_codegen_ssa::common::TypeKind::Float, + TypeKind::Double => rustc_codegen_ssa::common::TypeKind::Double, + TypeKind::X86_FP80 => rustc_codegen_ssa::common::TypeKind::X86_FP80, + TypeKind::FP128 => rustc_codegen_ssa::common::TypeKind::FP128, + TypeKind::PPC_FP128 => rustc_codegen_ssa::common::TypeKind::PPC_FP128, + TypeKind::Label => rustc_codegen_ssa::common::TypeKind::Label, + TypeKind::Integer => rustc_codegen_ssa::common::TypeKind::Integer, + TypeKind::Function => rustc_codegen_ssa::common::TypeKind::Function, + TypeKind::Struct => rustc_codegen_ssa::common::TypeKind::Struct, + TypeKind::Array => rustc_codegen_ssa::common::TypeKind::Array, + TypeKind::Pointer => rustc_codegen_ssa::common::TypeKind::Pointer, + TypeKind::Vector => rustc_codegen_ssa::common::TypeKind::Vector, + TypeKind::Metadata => rustc_codegen_ssa::common::TypeKind::Metadata, + TypeKind::X86_MMX => rustc_codegen_ssa::common::TypeKind::X86_MMX, + TypeKind::Token => rustc_codegen_ssa::common::TypeKind::Token, + TypeKind::ScalableVector => rustc_codegen_ssa::common::TypeKind::ScalableVector, + TypeKind::BFloat => rustc_codegen_ssa::common::TypeKind::BFloat, + TypeKind::X86_AMX => rustc_codegen_ssa::common::TypeKind::X86_AMX, + } + } +} + +/// LLVMAtomicRmwBinOp +#[derive(Copy, Clone)] +#[repr(C)] +pub enum AtomicRmwBinOp { + AtomicXchg = 0, + AtomicAdd = 1, + AtomicSub = 2, + AtomicAnd = 3, + AtomicNand = 4, + AtomicOr = 5, + AtomicXor = 6, + AtomicMax = 7, + AtomicMin = 8, + AtomicUMax = 9, + AtomicUMin = 10, +} + +impl AtomicRmwBinOp { + pub fn from_generic(op: rustc_codegen_ssa::common::AtomicRmwBinOp) -> Self { + match op { + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg => AtomicRmwBinOp::AtomicXchg, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicAdd => AtomicRmwBinOp::AtomicAdd, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicSub => AtomicRmwBinOp::AtomicSub, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicAnd => AtomicRmwBinOp::AtomicAnd, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicNand => AtomicRmwBinOp::AtomicNand, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicOr => AtomicRmwBinOp::AtomicOr, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXor => AtomicRmwBinOp::AtomicXor, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicMax => AtomicRmwBinOp::AtomicMax, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicMin => AtomicRmwBinOp::AtomicMin, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicUMax => AtomicRmwBinOp::AtomicUMax, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicUMin => AtomicRmwBinOp::AtomicUMin, + } + } +} + +/// LLVMAtomicOrdering +#[derive(Copy, Clone)] +#[repr(C)] +pub enum AtomicOrdering { + #[allow(dead_code)] + NotAtomic = 0, + Unordered = 1, + Monotonic = 2, + // Consume = 3, // Not specified yet. + Acquire = 4, + Release = 5, + AcquireRelease = 6, + SequentiallyConsistent = 7, +} + +impl AtomicOrdering { + pub fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { + match ao { + rustc_codegen_ssa::common::AtomicOrdering::Unordered => AtomicOrdering::Unordered, + rustc_codegen_ssa::common::AtomicOrdering::Relaxed => AtomicOrdering::Monotonic, + rustc_codegen_ssa::common::AtomicOrdering::Acquire => AtomicOrdering::Acquire, + rustc_codegen_ssa::common::AtomicOrdering::Release => AtomicOrdering::Release, + rustc_codegen_ssa::common::AtomicOrdering::AcquireRelease => { + AtomicOrdering::AcquireRelease + } + rustc_codegen_ssa::common::AtomicOrdering::SequentiallyConsistent => { + AtomicOrdering::SequentiallyConsistent + } + } + } +} + +/// LLVMRustSynchronizationScope +#[derive(Copy, Clone)] +#[repr(C)] +pub enum SynchronizationScope { + SingleThread, + CrossThread, +} + +impl SynchronizationScope { + pub fn from_generic(sc: rustc_codegen_ssa::common::SynchronizationScope) -> Self { + match sc { + rustc_codegen_ssa::common::SynchronizationScope::SingleThread => { + SynchronizationScope::SingleThread + } + rustc_codegen_ssa::common::SynchronizationScope::CrossThread => { + SynchronizationScope::CrossThread + } + } + } +} + +/// LLVMRustFileType +#[derive(Copy, Clone)] +#[repr(C)] +pub enum FileType { + AssemblyFile, + ObjectFile, +} + +/// LLVMMetadataType +#[derive(Copy, Clone)] +#[repr(C)] +pub enum MetadataType { + MD_dbg = 0, + MD_tbaa = 1, + MD_prof = 2, + MD_fpmath = 3, + MD_range = 4, + MD_tbaa_struct = 5, + MD_invariant_load = 6, + MD_alias_scope = 7, + MD_noalias = 8, + MD_nontemporal = 9, + MD_mem_parallel_loop_access = 10, + MD_nonnull = 11, + MD_align = 17, + MD_type = 19, + MD_vcall_visibility = 28, + MD_noundef = 29, +} + +/// LLVMRustAsmDialect +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum AsmDialect { + Att, + Intel, +} + +/// LLVMRustCodeGenOptLevel +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum CodeGenOptLevel { + None, + Less, + Default, + Aggressive, +} + +/// LLVMRustPassBuilderOptLevel +#[repr(C)] +pub enum PassBuilderOptLevel { + O0, + O1, + O2, + O3, + Os, + Oz, +} + +/// LLVMRustOptStage +#[derive(PartialEq)] +#[repr(C)] +pub enum OptStage { + PreLinkNoLTO, + PreLinkThinLTO, + PreLinkFatLTO, + ThinLTO, + FatLTO, +} + +/// LLVMRustSanitizerOptions +#[repr(C)] +pub struct SanitizerOptions { + pub sanitize_address: bool, + pub sanitize_address_recover: bool, + pub sanitize_memory: bool, + pub sanitize_memory_recover: bool, + pub sanitize_memory_track_origins: c_int, + pub sanitize_thread: bool, + pub sanitize_hwaddress: bool, + pub sanitize_hwaddress_recover: bool, +} + +/// LLVMRelocMode +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum RelocModel { + Static, + PIC, + DynamicNoPic, + ROPI, + RWPI, + ROPI_RWPI, +} + +/// LLVMRustCodeModel +#[derive(Copy, Clone)] +#[repr(C)] +pub enum CodeModel { + Tiny, + Small, + Kernel, + Medium, + Large, + None, +} + +/// LLVMRustDiagnosticKind +#[derive(Copy, Clone)] +#[repr(C)] +#[allow(dead_code)] // Variants constructed by C++. +pub enum DiagnosticKind { + Other, + InlineAsm, + StackSize, + DebugMetadataVersion, + SampleProfile, + OptimizationRemark, + OptimizationRemarkMissed, + OptimizationRemarkAnalysis, + OptimizationRemarkAnalysisFPCommute, + OptimizationRemarkAnalysisAliasing, + OptimizationRemarkOther, + OptimizationFailure, + PGOProfile, + Linker, + Unsupported, + SrcMgr, +} + +/// LLVMRustDiagnosticLevel +#[derive(Copy, Clone)] +#[repr(C)] +#[allow(dead_code)] // Variants constructed by C++. +pub enum DiagnosticLevel { + Error, + Warning, + Note, + Remark, +} + +/// LLVMRustArchiveKind +#[derive(Copy, Clone)] +#[repr(C)] +pub enum ArchiveKind { + K_GNU, + K_BSD, + K_DARWIN, + K_COFF, +} + +// LLVMRustThinLTOData +extern "C" { + pub type ThinLTOData; +} + +// LLVMRustThinLTOBuffer +extern "C" { + pub type ThinLTOBuffer; +} + +/// LLVMRustThinLTOModule +#[repr(C)] +pub struct ThinLTOModule { + pub identifier: *const c_char, + pub data: *const u8, + pub len: usize, +} + +/// LLVMThreadLocalMode +#[derive(Copy, Clone)] +#[repr(C)] +pub enum ThreadLocalMode { + NotThreadLocal, + GeneralDynamic, + LocalDynamic, + InitialExec, + LocalExec, +} + +/// LLVMRustChecksumKind +#[derive(Copy, Clone)] +#[repr(C)] +pub enum ChecksumKind { + None, + MD5, + SHA1, + SHA256, +} + +extern "C" { + type Opaque; +} +#[repr(C)] +struct InvariantOpaque<'a> { + _marker: PhantomData<&'a mut &'a ()>, + _opaque: Opaque, +} + +// Opaque pointer types +extern "C" { + pub type Module; +} +extern "C" { + pub type Context; +} +extern "C" { + pub type Type; +} +extern "C" { + pub type Value; +} +extern "C" { + pub type ConstantInt; +} +extern "C" { + pub type Attribute; +} +extern "C" { + pub type Metadata; +} +extern "C" { + pub type BasicBlock; +} +#[repr(C)] +pub struct Builder<'a>(InvariantOpaque<'a>); +#[repr(C)] +pub struct PassManager<'a>(InvariantOpaque<'a>); +extern "C" { + pub type PassManagerBuilder; +} +extern "C" { + pub type Pass; +} +extern "C" { + pub type TargetMachine; +} +extern "C" { + pub type Archive; +} +#[repr(C)] +pub struct ArchiveIterator<'a>(InvariantOpaque<'a>); +#[repr(C)] +pub struct ArchiveChild<'a>(InvariantOpaque<'a>); +extern "C" { + pub type Twine; +} +extern "C" { + pub type DiagnosticInfo; +} +extern "C" { + pub type SMDiagnostic; +} +#[repr(C)] +pub struct RustArchiveMember<'a>(InvariantOpaque<'a>); +#[repr(C)] +pub struct OperandBundleDef<'a>(InvariantOpaque<'a>); +#[repr(C)] +pub struct Linker<'a>(InvariantOpaque<'a>); + +extern "C" { + pub type DiagnosticHandler; +} + +pub type DiagnosticHandlerTy = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void); +pub type InlineAsmDiagHandlerTy = unsafe extern "C" fn(&SMDiagnostic, *const c_void, c_uint); + +pub mod coverageinfo { + use super::coverage_map; + + /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L209-L230) + #[derive(Copy, Clone, Debug)] + #[repr(C)] + pub enum RegionKind { + /// A CodeRegion associates some code with a counter + CodeRegion = 0, + + /// An ExpansionRegion represents a file expansion region that associates + /// a source range with the expansion of a virtual source file, such as + /// for a macro instantiation or #include file. + ExpansionRegion = 1, + + /// A SkippedRegion represents a source range with code that was skipped + /// by a preprocessor or similar means. + SkippedRegion = 2, + + /// A GapRegion is like a CodeRegion, but its count is only set as the + /// line execution count when its the only region in the line. + GapRegion = 3, + + /// A BranchRegion represents leaf-level boolean expressions and is + /// associated with two counters, each representing the number of times the + /// expression evaluates to true or false. + BranchRegion = 4, + } + + /// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the + /// coverage map, in accordance with the + /// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). + /// The struct composes fields representing the `Counter` type and value(s) (injected counter + /// ID, or expression type and operands), the source file (an indirect index into a "filenames + /// array", encoded separately), and source location (start and end positions of the represented + /// code region). + /// + /// Matches LLVMRustCounterMappingRegion. + #[derive(Copy, Clone, Debug)] + #[repr(C)] + pub struct CounterMappingRegion { + /// The counter type and type-dependent counter data, if any. + counter: coverage_map::Counter, + + /// If the `RegionKind` is a `BranchRegion`, this represents the counter + /// for the false branch of the region. + false_counter: coverage_map::Counter, + + /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the + /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes + /// that, in turn, are used to look up the filename for this region. + file_id: u32, + + /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find + /// the mapping regions created as a result of macro expansion, by checking if their file id + /// matches the expanded file id. + expanded_file_id: u32, + + /// 1-based starting line of the mapping region. + start_line: u32, + + /// 1-based starting column of the mapping region. + start_col: u32, + + /// 1-based ending line of the mapping region. + end_line: u32, + + /// 1-based ending column of the mapping region. If the high bit is set, the current + /// mapping region is a gap area. + end_col: u32, + + kind: RegionKind, + } + + impl CounterMappingRegion { + pub(crate) fn code_region( + counter: coverage_map::Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter: coverage_map::Counter::zero(), + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::CodeRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn branch_region( + counter: coverage_map::Counter, + false_counter: coverage_map::Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::BranchRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn expansion_region( + file_id: u32, + expanded_file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter: coverage_map::Counter::zero(), + false_counter: coverage_map::Counter::zero(), + file_id, + expanded_file_id, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::ExpansionRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn skipped_region( + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter: coverage_map::Counter::zero(), + false_counter: coverage_map::Counter::zero(), + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::SkippedRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn gap_region( + counter: coverage_map::Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter: coverage_map::Counter::zero(), + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col: (1_u32 << 31) | end_col, + kind: RegionKind::GapRegion, + } + } + } +} + +pub mod debuginfo { + use super::{InvariantOpaque, Metadata}; + use bitflags::bitflags; + + #[repr(C)] + pub struct DIBuilder<'a>(InvariantOpaque<'a>); + + pub type DIDescriptor = Metadata; + pub type DILocation = Metadata; + pub type DIScope = DIDescriptor; + pub type DIFile = DIScope; + pub type DILexicalBlock = DIScope; + pub type DISubprogram = DIScope; + pub type DINameSpace = DIScope; + pub type DIType = DIDescriptor; + pub type DIBasicType = DIType; + pub type DIDerivedType = DIType; + pub type DICompositeType = DIDerivedType; + pub type DIVariable = DIDescriptor; + pub type DIGlobalVariableExpression = DIDescriptor; + pub type DIArray = DIDescriptor; + pub type DISubrange = DIDescriptor; + pub type DIEnumerator = DIDescriptor; + pub type DITemplateTypeParameter = DIDescriptor; + + // These values **must** match with LLVMRustDIFlags!! + bitflags! { + #[repr(transparent)] + #[derive(Default)] + pub struct DIFlags: u32 { + const FlagZero = 0; + const FlagPrivate = 1; + const FlagProtected = 2; + const FlagPublic = 3; + const FlagFwdDecl = (1 << 2); + const FlagAppleBlock = (1 << 3); + const FlagBlockByrefStruct = (1 << 4); + const FlagVirtual = (1 << 5); + const FlagArtificial = (1 << 6); + const FlagExplicit = (1 << 7); + const FlagPrototyped = (1 << 8); + const FlagObjcClassComplete = (1 << 9); + const FlagObjectPointer = (1 << 10); + const FlagVector = (1 << 11); + const FlagStaticMember = (1 << 12); + const FlagLValueReference = (1 << 13); + const FlagRValueReference = (1 << 14); + const FlagExternalTypeRef = (1 << 15); + const FlagIntroducedVirtual = (1 << 18); + const FlagBitField = (1 << 19); + const FlagNoReturn = (1 << 20); + } + } + + // These values **must** match with LLVMRustDISPFlags!! + bitflags! { + #[repr(transparent)] + #[derive(Default)] + pub struct DISPFlags: u32 { + const SPFlagZero = 0; + const SPFlagVirtual = 1; + const SPFlagPureVirtual = 2; + const SPFlagLocalToUnit = (1 << 2); + const SPFlagDefinition = (1 << 3); + const SPFlagOptimized = (1 << 4); + const SPFlagMainSubprogram = (1 << 5); + } + } + + /// LLVMRustDebugEmissionKind + #[derive(Copy, Clone)] + #[repr(C)] + pub enum DebugEmissionKind { + NoDebug, + FullDebug, + LineTablesOnly, + } + + impl DebugEmissionKind { + pub fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { + use rustc_session::config::DebugInfo; + match kind { + DebugInfo::None => DebugEmissionKind::NoDebug, + DebugInfo::Limited => DebugEmissionKind::LineTablesOnly, + DebugInfo::Full => DebugEmissionKind::FullDebug, + } + } + } +} + +use bitflags::bitflags; +// These values **must** match with LLVMRustAllocKindFlags +bitflags! { + #[repr(transparent)] + #[derive(Default)] + pub struct AllocKindFlags : u64 { + const Unknown = 0; + const Alloc = 1; + const Realloc = 1 << 1; + const Free = 1 << 2; + const Uninitialized = 1 << 3; + const Zeroed = 1 << 4; + const Aligned = 1 << 5; + } +} + +extern "C" { + pub type ModuleBuffer; +} + +pub type SelfProfileBeforePassCallback = + unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); +pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void); + +extern "C" { + pub fn LLVMRustInstallFatalErrorHandler(); + pub fn LLVMRustDisableSystemDialogsOnCrash(); + + // Create and destroy contexts. + pub fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; + pub fn LLVMContextDispose(C: &'static mut Context); + pub fn LLVMGetMDKindIDInContext(C: &Context, Name: *const c_char, SLen: c_uint) -> c_uint; + + // Create modules. + pub fn LLVMModuleCreateWithNameInContext(ModuleID: *const c_char, C: &Context) -> &Module; + pub fn LLVMGetModuleContext(M: &Module) -> &Context; + pub fn LLVMCloneModule(M: &Module) -> &Module; + + /// Data layout. See Module::getDataLayout. + pub fn LLVMGetDataLayoutStr(M: &Module) -> *const c_char; + pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); + + /// See Module::setModuleInlineAsm. + pub fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t); + + /// See llvm::LLVMTypeKind::getTypeID. + pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; + + // Operations on integer types + pub fn LLVMInt1TypeInContext(C: &Context) -> &Type; + pub fn LLVMInt8TypeInContext(C: &Context) -> &Type; + pub fn LLVMInt16TypeInContext(C: &Context) -> &Type; + pub fn LLVMInt32TypeInContext(C: &Context) -> &Type; + pub fn LLVMInt64TypeInContext(C: &Context) -> &Type; + pub fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type; + + pub fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; + + // Operations on real types + pub fn LLVMFloatTypeInContext(C: &Context) -> &Type; + pub fn LLVMDoubleTypeInContext(C: &Context) -> &Type; + + // Operations on function types + pub fn LLVMFunctionType<'a>( + ReturnType: &'a Type, + ParamTypes: *const &'a Type, + ParamCount: c_uint, + IsVarArg: Bool, + ) -> &'a Type; + pub fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint; + pub fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type); + + // Operations on struct types + pub fn LLVMStructTypeInContext<'a>( + C: &'a Context, + ElementTypes: *const &'a Type, + ElementCount: c_uint, + Packed: Bool, + ) -> &'a Type; + + // Operations on array, pointer, and vector types (sequence types) + pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type; + pub fn LLVMPointerType(ElementType: &Type, AddressSpace: c_uint) -> &Type; + pub fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; + + pub fn LLVMGetElementType(Ty: &Type) -> &Type; + pub fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint; + + // Operations on other types + pub fn LLVMVoidTypeInContext(C: &Context) -> &Type; + pub fn LLVMRustMetadataTypeInContext(C: &Context) -> &Type; + + // Operations on all values + pub fn LLVMTypeOf(Val: &Value) -> &Type; + pub fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char; + pub fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); + pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); + pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); + pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; + + // Operations on constants of any type + pub fn LLVMConstNull(Ty: &Type) -> &Value; + pub fn LLVMGetUndef(Ty: &Type) -> &Value; + + // Operations on metadata + pub fn LLVMMDStringInContext(C: &Context, Str: *const c_char, SLen: c_uint) -> &Value; + pub fn LLVMMDNodeInContext<'a>( + C: &'a Context, + Vals: *const &'a Value, + Count: c_uint, + ) -> &'a Value; + pub fn LLVMMDNodeInContext2<'a>( + C: &'a Context, + Vals: *const &'a Metadata, + Count: size_t, + ) -> &'a Metadata; + pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value); + + // Operations on scalar constants + pub fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; + pub fn LLVMConstIntOfArbitraryPrecision(IntTy: &Type, Wn: c_uint, Ws: *const u64) -> &Value; + pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; + pub fn LLVMConstIntGetZExtValue(ConstantVal: &ConstantInt) -> c_ulonglong; + pub fn LLVMRustConstInt128Get( + ConstantVal: &ConstantInt, + SExt: bool, + high: &mut u64, + low: &mut u64, + ) -> bool; + + // Operations on composite constants + pub fn LLVMConstStringInContext( + C: &Context, + Str: *const c_char, + Length: c_uint, + DontNullTerminate: Bool, + ) -> &Value; + pub fn LLVMConstStructInContext<'a>( + C: &'a Context, + ConstantVals: *const &'a Value, + Count: c_uint, + Packed: Bool, + ) -> &'a Value; + + pub fn LLVMConstArray<'a>( + ElementTy: &'a Type, + ConstantVals: *const &'a Value, + Length: c_uint, + ) -> &'a Value; + pub fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; + + // Constant expressions + pub fn LLVMRustConstInBoundsGEP2<'a>( + ty: &'a Type, + ConstantVal: &'a Value, + ConstantIndices: *const &'a Value, + NumIndices: c_uint, + ) -> &'a Value; + pub fn LLVMConstZExt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; + + // Operations on global variables, functions, and aliases (globals) + pub fn LLVMIsDeclaration(Global: &Value) -> Bool; + pub fn LLVMRustGetLinkage(Global: &Value) -> Linkage; + pub fn LLVMRustSetLinkage(Global: &Value, RustLinkage: Linkage); + pub fn LLVMSetSection(Global: &Value, Section: *const c_char); + pub fn LLVMRustGetVisibility(Global: &Value) -> Visibility; + pub fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility); + pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); + pub fn LLVMGetAlignment(Global: &Value) -> c_uint; + pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); + pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); + + // Operations on global variables + pub fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; + pub fn LLVMAddGlobal<'a>(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value; + pub fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>; + pub fn LLVMRustGetOrInsertGlobal<'a>( + M: &'a Module, + Name: *const c_char, + NameLen: size_t, + T: &'a Type, + ) -> &'a Value; + pub fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value; + pub fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>; + pub fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>; + pub fn LLVMDeleteGlobal(GlobalVar: &Value); + pub fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; + pub fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); + pub fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; + pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); + pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; + pub fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); + pub fn LLVMRustGetNamedValue( + M: &Module, + Name: *const c_char, + NameLen: size_t, + ) -> Option<&Value>; + pub fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); + + // Operations on attributes + pub fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute; + pub fn LLVMCreateStringAttribute( + C: &Context, + Name: *const c_char, + NameLen: c_uint, + Value: *const c_char, + ValueLen: c_uint, + ) -> &Attribute; + pub fn LLVMRustCreateAlignmentAttr(C: &Context, bytes: u64) -> &Attribute; + pub fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute; + pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; + pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; + pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; + pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; + + // Operations on functions + pub fn LLVMRustGetOrInsertFunction<'a>( + M: &'a Module, + Name: *const c_char, + NameLen: size_t, + FunctionTy: &'a Type, + ) -> &'a Value; + pub fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); + pub fn LLVMRustAddFunctionAttributes<'a>( + Fn: &'a Value, + index: c_uint, + Attrs: *const &'a Attribute, + AttrsLen: size_t, + ); + + // Operations on parameters + pub fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; + pub fn LLVMCountParams(Fn: &Value) -> c_uint; + pub fn LLVMGetParam(Fn: &Value, Index: c_uint) -> &Value; + + // Operations on basic blocks + pub fn LLVMGetBasicBlockParent(BB: &BasicBlock) -> &Value; + pub fn LLVMAppendBasicBlockInContext<'a>( + C: &'a Context, + Fn: &'a Value, + Name: *const c_char, + ) -> &'a BasicBlock; + + // Operations on instructions + pub fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; + pub fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; + + // Operations on call sites + pub fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); + pub fn LLVMRustAddCallSiteAttributes<'a>( + Instr: &'a Value, + index: c_uint, + Attrs: *const &'a Attribute, + AttrsLen: size_t, + ); + + // Operations on load/store instructions (only) + pub fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool); + + // Operations on phi nodes + pub fn LLVMAddIncoming<'a>( + PhiNode: &'a Value, + IncomingValues: *const &'a Value, + IncomingBlocks: *const &'a BasicBlock, + Count: c_uint, + ); + + // Instruction builders + pub fn LLVMCreateBuilderInContext(C: &Context) -> &mut Builder<'_>; + pub fn LLVMPositionBuilderAtEnd<'a>(Builder: &Builder<'a>, Block: &'a BasicBlock); + pub fn LLVMGetInsertBlock<'a>(Builder: &Builder<'a>) -> &'a BasicBlock; + pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); + + // Metadata + pub fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: &'a Value); + + // Terminators + pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; + pub fn LLVMBuildRet<'a>(B: &Builder<'a>, V: &'a Value) -> &'a Value; + pub fn LLVMBuildBr<'a>(B: &Builder<'a>, Dest: &'a BasicBlock) -> &'a Value; + pub fn LLVMBuildCondBr<'a>( + B: &Builder<'a>, + If: &'a Value, + Then: &'a BasicBlock, + Else: &'a BasicBlock, + ) -> &'a Value; + pub fn LLVMBuildSwitch<'a>( + B: &Builder<'a>, + V: &'a Value, + Else: &'a BasicBlock, + NumCases: c_uint, + ) -> &'a Value; + pub fn LLVMRustBuildInvoke<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + Then: &'a BasicBlock, + Catch: &'a BasicBlock, + Bundle: Option<&OperandBundleDef<'a>>, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildLandingPad<'a>( + B: &Builder<'a>, + Ty: &'a Type, + PersFn: Option<&'a Value>, + NumClauses: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; + pub fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; + + pub fn LLVMRustBuildCleanupPad<'a>( + B: &Builder<'a>, + ParentPad: Option<&'a Value>, + ArgCnt: c_uint, + Args: *const &'a Value, + Name: *const c_char, + ) -> Option<&'a Value>; + pub fn LLVMRustBuildCleanupRet<'a>( + B: &Builder<'a>, + CleanupPad: &'a Value, + UnwindBB: Option<&'a BasicBlock>, + ) -> Option<&'a Value>; + pub fn LLVMRustBuildCatchPad<'a>( + B: &Builder<'a>, + ParentPad: &'a Value, + ArgCnt: c_uint, + Args: *const &'a Value, + Name: *const c_char, + ) -> Option<&'a Value>; + pub fn LLVMRustBuildCatchRet<'a>( + B: &Builder<'a>, + Pad: &'a Value, + BB: &'a BasicBlock, + ) -> Option<&'a Value>; + pub fn LLVMRustBuildCatchSwitch<'a>( + Builder: &Builder<'a>, + ParentPad: Option<&'a Value>, + BB: Option<&'a BasicBlock>, + NumHandlers: c_uint, + Name: *const c_char, + ) -> Option<&'a Value>; + pub fn LLVMRustAddHandler<'a>(CatchSwitch: &'a Value, Handler: &'a BasicBlock); + pub fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); + + // Add a case to the switch instruction + pub fn LLVMAddCase<'a>(Switch: &'a Value, OnVal: &'a Value, Dest: &'a BasicBlock); + + // Add a clause to the landing pad instruction + pub fn LLVMAddClause<'a>(LandingPad: &'a Value, ClauseVal: &'a Value); + + // Set the cleanup on a landing pad instruction + pub fn LLVMSetCleanup(LandingPad: &Value, Val: Bool); + + // Arithmetic + pub fn LLVMBuildAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildUDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildExactUDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildSDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildExactSDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildURem<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildSRem<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFRem<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildShl<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildLShr<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildAShr<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNSWAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNUWAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNSWSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNUWSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNSWMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNUWMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildAnd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildOr<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildXor<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; + pub fn LLVMBuildFNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; + pub fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; + pub fn LLVMRustSetFastMath(Instr: &Value); + + // Memory + pub fn LLVMBuildAlloca<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; + pub fn LLVMBuildArrayAlloca<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Val: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildLoad2<'a>( + B: &Builder<'a>, + Ty: &'a Type, + PointerVal: &'a Value, + Name: *const c_char, + ) -> &'a Value; + + pub fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value; + + pub fn LLVMBuildGEP2<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Pointer: &'a Value, + Indices: *const &'a Value, + NumIndices: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildInBoundsGEP2<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Pointer: &'a Value, + Indices: *const &'a Value, + NumIndices: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildStructGEP2<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Pointer: &'a Value, + Idx: c_uint, + Name: *const c_char, + ) -> &'a Value; + + // Casts + pub fn LLVMBuildTrunc<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildZExt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildSExt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFPToUI<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFPToSI<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildUIToFP<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildSIToFP<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFPTrunc<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFPExt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildPtrToInt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildIntToPtr<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildBitCast<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildPointerCast<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMRustBuildIntCast<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + IsSigned: bool, + ) -> &'a Value; + + // Comparisons + pub fn LLVMBuildICmp<'a>( + B: &Builder<'a>, + Op: c_uint, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFCmp<'a>( + B: &Builder<'a>, + Op: c_uint, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + + // Miscellaneous instructions + pub fn LLVMBuildPhi<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; + pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; + pub fn LLVMRustBuildCall<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + Bundle: Option<&OperandBundleDef<'a>>, + ) -> &'a Value; + pub fn LLVMRustBuildMemCpy<'a>( + B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Src: &'a Value, + SrcAlign: c_uint, + Size: &'a Value, + IsVolatile: bool, + ) -> &'a Value; + pub fn LLVMRustBuildMemMove<'a>( + B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Src: &'a Value, + SrcAlign: c_uint, + Size: &'a Value, + IsVolatile: bool, + ) -> &'a Value; + pub fn LLVMRustBuildMemSet<'a>( + B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Val: &'a Value, + Size: &'a Value, + IsVolatile: bool, + ) -> &'a Value; + pub fn LLVMBuildSelect<'a>( + B: &Builder<'a>, + If: &'a Value, + Then: &'a Value, + Else: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildVAArg<'a>( + B: &Builder<'a>, + list: &'a Value, + Ty: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildExtractElement<'a>( + B: &Builder<'a>, + VecVal: &'a Value, + Index: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildInsertElement<'a>( + B: &Builder<'a>, + VecVal: &'a Value, + EltVal: &'a Value, + Index: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildShuffleVector<'a>( + B: &Builder<'a>, + V1: &'a Value, + V2: &'a Value, + Mask: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildExtractValue<'a>( + B: &Builder<'a>, + AggVal: &'a Value, + Index: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildInsertValue<'a>( + B: &Builder<'a>, + AggVal: &'a Value, + EltVal: &'a Value, + Index: c_uint, + Name: *const c_char, + ) -> &'a Value; + + pub fn LLVMRustBuildVectorReduceFAdd<'a>( + B: &Builder<'a>, + Acc: &'a Value, + Src: &'a Value, + ) -> &'a Value; + pub fn LLVMRustBuildVectorReduceFMul<'a>( + B: &Builder<'a>, + Acc: &'a Value, + Src: &'a Value, + ) -> &'a Value; + pub fn LLVMRustBuildVectorReduceAdd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub fn LLVMRustBuildVectorReduceMul<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub fn LLVMRustBuildVectorReduceAnd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub fn LLVMRustBuildVectorReduceOr<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub fn LLVMRustBuildVectorReduceXor<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub fn LLVMRustBuildVectorReduceMin<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsSigned: bool, + ) -> &'a Value; + pub fn LLVMRustBuildVectorReduceMax<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsSigned: bool, + ) -> &'a Value; + pub fn LLVMRustBuildVectorReduceFMin<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsNaN: bool, + ) -> &'a Value; + pub fn LLVMRustBuildVectorReduceFMax<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsNaN: bool, + ) -> &'a Value; + + pub fn LLVMRustBuildMinNum<'a>(B: &Builder<'a>, LHS: &'a Value, LHS: &'a Value) -> &'a Value; + pub fn LLVMRustBuildMaxNum<'a>(B: &Builder<'a>, LHS: &'a Value, LHS: &'a Value) -> &'a Value; + + // Atomic Operations + pub fn LLVMRustBuildAtomicLoad<'a>( + B: &Builder<'a>, + ElementType: &'a Type, + PointerVal: &'a Value, + Name: *const c_char, + Order: AtomicOrdering, + ) -> &'a Value; + + pub fn LLVMRustBuildAtomicStore<'a>( + B: &Builder<'a>, + Val: &'a Value, + Ptr: &'a Value, + Order: AtomicOrdering, + ) -> &'a Value; + + pub fn LLVMRustBuildAtomicCmpXchg<'a>( + B: &Builder<'a>, + LHS: &'a Value, + CMP: &'a Value, + RHS: &'a Value, + Order: AtomicOrdering, + FailureOrder: AtomicOrdering, + Weak: Bool, + ) -> &'a Value; + + pub fn LLVMBuildAtomicRMW<'a>( + B: &Builder<'a>, + Op: AtomicRmwBinOp, + LHS: &'a Value, + RHS: &'a Value, + Order: AtomicOrdering, + SingleThreaded: Bool, + ) -> &'a Value; + + pub fn LLVMRustBuildAtomicFence( + B: &Builder<'_>, + Order: AtomicOrdering, + Scope: SynchronizationScope, + ); + + /// Writes a module to the specified path. Returns 0 on success. + pub fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int; + + /// Creates a pass manager. + pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; + + /// Creates a function-by-function pass manager + pub fn LLVMCreateFunctionPassManagerForModule(M: &Module) -> &mut PassManager<'_>; + + /// Disposes a pass manager. + pub fn LLVMDisposePassManager<'a>(PM: &'a mut PassManager<'a>); + + /// Runs a pass manager on a module. + pub fn LLVMRunPassManager<'a>(PM: &PassManager<'a>, M: &'a Module) -> Bool; + + pub fn LLVMInitializePasses(); + + pub fn LLVMTimeTraceProfilerInitialize(); + + pub fn LLVMTimeTraceProfilerFinishThread(); + + pub fn LLVMTimeTraceProfilerFinish(FileName: *const c_char); + + pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); + + pub fn LLVMRustPassManagerBuilderCreate() -> &'static mut PassManagerBuilder; + pub fn LLVMRustPassManagerBuilderDispose(PMB: &'static mut PassManagerBuilder); + pub fn LLVMRustPassManagerBuilderUseInlinerWithThreshold( + PMB: &PassManagerBuilder, + threshold: c_uint, + ); + pub fn LLVMRustPassManagerBuilderPopulateModulePassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + ); + + pub fn LLVMRustPassManagerBuilderPopulateFunctionPassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + ); + pub fn LLVMRustPassManagerBuilderPopulateLTOPassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + Internalize: Bool, + RunInliner: Bool, + ); + pub fn LLVMRustPassManagerBuilderPopulateThinLTOPassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + ); + + pub fn LLVMGetHostCPUFeatures() -> *mut c_char; + + pub fn LLVMDisposeMessage(message: *mut c_char); + + pub fn LLVMIsMultithreaded() -> Bool; + + /// Returns a string describing the last error caused by an LLVMRust* call. + pub fn LLVMRustGetLastError() -> *const c_char; + + /// Print the pass timings since static dtors aren't picking them up. + pub fn LLVMRustPrintPassTimings(); + + pub fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type; + + pub fn LLVMStructSetBody<'a>( + StructTy: &'a Type, + ElementTypes: *const &'a Type, + ElementCount: c_uint, + Packed: Bool, + ); + + /// Prepares inline assembly. + pub fn LLVMRustInlineAsm( + Ty: &Type, + AsmString: *const c_char, + AsmStringLen: size_t, + Constraints: *const c_char, + ConstraintsLen: size_t, + SideEffects: Bool, + AlignStack: Bool, + Dialect: AsmDialect, + CanThrow: Bool, + ) -> &Value; + pub fn LLVMRustInlineAsmVerify( + Ty: &Type, + Constraints: *const c_char, + ConstraintsLen: size_t, + ) -> bool; + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( + Filenames: *const *const c_char, + FilenamesLen: size_t, + BufferOut: &RustString, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMappingToBuffer( + VirtualFileMappingIDs: *const c_uint, + NumVirtualFileMappingIDs: c_uint, + Expressions: *const coverage_map::CounterExpression, + NumExpressions: c_uint, + MappingRegions: *const coverageinfo::CounterMappingRegion, + NumMappingRegions: c_uint, + BufferOut: &RustString, + ); + + pub fn LLVMRustCoverageCreatePGOFuncNameVar(F: &Value, FuncName: *const c_char) -> &Value; + pub fn LLVMRustCoverageHashCString(StrVal: *const c_char) -> u64; + pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); + + pub fn LLVMRustCoverageMappingVersion() -> u32; + pub fn LLVMRustDebugMetadataVersion() -> u32; + pub fn LLVMRustVersionMajor() -> u32; + pub fn LLVMRustVersionMinor() -> u32; + pub fn LLVMRustVersionPatch() -> u32; + + /// Add LLVM module flags. + /// + /// In order for Rust-C LTO to work, module flags must be compatible with Clang. What + /// "compatible" means depends on the merge behaviors involved. + pub fn LLVMRustAddModuleFlag( + M: &Module, + merge_behavior: LLVMModFlagBehavior, + name: *const c_char, + value: u32, + ); + pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; + + pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; + + pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; + + pub fn LLVMRustDIBuilderDispose<'a>(Builder: &'a mut DIBuilder<'a>); + + pub fn LLVMRustDIBuilderFinalize(Builder: &DIBuilder<'_>); + + pub fn LLVMRustDIBuilderCreateCompileUnit<'a>( + Builder: &DIBuilder<'a>, + Lang: c_uint, + File: &'a DIFile, + Producer: *const c_char, + ProducerLen: size_t, + isOptimized: bool, + Flags: *const c_char, + RuntimeVer: c_uint, + SplitName: *const c_char, + SplitNameLen: size_t, + kind: DebugEmissionKind, + DWOId: u64, + SplitDebugInlining: bool, + ) -> &'a DIDescriptor; + + pub fn LLVMRustDIBuilderCreateFile<'a>( + Builder: &DIBuilder<'a>, + Filename: *const c_char, + FilenameLen: size_t, + Directory: *const c_char, + DirectoryLen: size_t, + CSKind: ChecksumKind, + Checksum: *const c_char, + ChecksumLen: size_t, + ) -> &'a DIFile; + + pub fn LLVMRustDIBuilderCreateSubroutineType<'a>( + Builder: &DIBuilder<'a>, + ParameterTypes: &'a DIArray, + ) -> &'a DICompositeType; + + pub fn LLVMRustDIBuilderCreateFunction<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + ScopeLine: c_uint, + Flags: DIFlags, + SPFlags: DISPFlags, + MaybeFn: Option<&'a Value>, + TParam: &'a DIArray, + Decl: Option<&'a DIDescriptor>, + ) -> &'a DISubprogram; + + pub fn LLVMRustDIBuilderCreateBasicType<'a>( + Builder: &DIBuilder<'a>, + Name: *const c_char, + NameLen: size_t, + SizeInBits: u64, + Encoding: c_uint, + ) -> &'a DIBasicType; + + pub fn LLVMRustDIBuilderCreateTypedef<'a>( + Builder: &DIBuilder<'a>, + Type: &'a DIBasicType, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Scope: Option<&'a DIScope>, + ) -> &'a DIDerivedType; + + pub fn LLVMRustDIBuilderCreatePointerType<'a>( + Builder: &DIBuilder<'a>, + PointeeTy: &'a DIType, + SizeInBits: u64, + AlignInBits: u32, + AddressSpace: c_uint, + Name: *const c_char, + NameLen: size_t, + ) -> &'a DIDerivedType; + + pub fn LLVMRustDIBuilderCreateStructType<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIDescriptor>, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + DerivedFrom: Option<&'a DIType>, + Elements: &'a DIArray, + RunTimeLang: c_uint, + VTableHolder: Option<&'a DIType>, + UniqueId: *const c_char, + UniqueIdLen: size_t, + ) -> &'a DICompositeType; + + pub fn LLVMRustDIBuilderCreateMemberType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + SizeInBits: u64, + AlignInBits: u32, + OffsetInBits: u64, + Flags: DIFlags, + Ty: &'a DIType, + ) -> &'a DIDerivedType; + + pub fn LLVMRustDIBuilderCreateVariantMemberType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + OffsetInBits: u64, + Discriminant: Option<&'a Value>, + Flags: DIFlags, + Ty: &'a DIType, + ) -> &'a DIType; + + pub fn LLVMRustDIBuilderCreateLexicalBlock<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + File: &'a DIFile, + Line: c_uint, + Col: c_uint, + ) -> &'a DILexicalBlock; + + pub fn LLVMRustDIBuilderCreateLexicalBlockFile<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + File: &'a DIFile, + ) -> &'a DILexicalBlock; + + pub fn LLVMRustDIBuilderCreateStaticVariable<'a>( + Builder: &DIBuilder<'a>, + Context: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + isLocalToUnit: bool, + Val: &'a Value, + Decl: Option<&'a DIDescriptor>, + AlignInBits: u32, + ) -> &'a DIGlobalVariableExpression; + + pub fn LLVMRustDIBuilderCreateVariable<'a>( + Builder: &DIBuilder<'a>, + Tag: c_uint, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + AlwaysPreserve: bool, + Flags: DIFlags, + ArgNo: c_uint, + AlignInBits: u32, + ) -> &'a DIVariable; + + pub fn LLVMRustDIBuilderCreateArrayType<'a>( + Builder: &DIBuilder<'a>, + Size: u64, + AlignInBits: u32, + Ty: &'a DIType, + Subscripts: &'a DIArray, + ) -> &'a DIType; + + pub fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( + Builder: &DIBuilder<'a>, + Lo: i64, + Count: i64, + ) -> &'a DISubrange; + + pub fn LLVMRustDIBuilderGetOrCreateArray<'a>( + Builder: &DIBuilder<'a>, + Ptr: *const Option<&'a DIDescriptor>, + Count: c_uint, + ) -> &'a DIArray; + + pub fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( + Builder: &DIBuilder<'a>, + Val: &'a Value, + VarInfo: &'a DIVariable, + AddrOps: *const u64, + AddrOpsCount: c_uint, + DL: &'a DILocation, + InsertAtEnd: &'a BasicBlock, + ) -> &'a Value; + + pub fn LLVMRustDIBuilderCreateEnumerator<'a>( + Builder: &DIBuilder<'a>, + Name: *const c_char, + NameLen: size_t, + Value: i64, + IsUnsigned: bool, + ) -> &'a DIEnumerator; + + pub fn LLVMRustDIBuilderCreateEnumerationType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Elements: &'a DIArray, + ClassType: &'a DIType, + IsScoped: bool, + ) -> &'a DIType; + + pub fn LLVMRustDIBuilderCreateUnionType<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + Elements: Option<&'a DIArray>, + RunTimeLang: c_uint, + UniqueId: *const c_char, + UniqueIdLen: size_t, + ) -> &'a DIType; + + pub fn LLVMRustDIBuilderCreateVariantPart<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + Discriminator: Option<&'a DIDerivedType>, + Elements: &'a DIArray, + UniqueId: *const c_char, + UniqueIdLen: size_t, + ) -> &'a DIDerivedType; + + pub fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); + + pub fn LLVMRustDIBuilderCreateTemplateTypeParameter<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + Ty: &'a DIType, + ) -> &'a DITemplateTypeParameter; + + pub fn LLVMRustDIBuilderCreateNameSpace<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + ExportSymbols: bool, + ) -> &'a DINameSpace; + + pub fn LLVMRustDICompositeTypeReplaceArrays<'a>( + Builder: &DIBuilder<'a>, + CompositeType: &'a DIType, + Elements: Option<&'a DIArray>, + Params: Option<&'a DIArray>, + ); + + pub fn LLVMRustDIBuilderCreateDebugLocation<'a>( + Line: c_uint, + Column: c_uint, + Scope: &'a DIScope, + InlinedAt: Option<&'a DILocation>, + ) -> &'a DILocation; + pub fn LLVMRustDIBuilderCreateOpDeref() -> u64; + pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> u64; + + #[allow(improper_ctypes)] + pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); + #[allow(improper_ctypes)] + pub fn LLVMRustWriteValueToString(value_ref: &Value, s: &RustString); + + pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; + + pub fn LLVMRustFindAndCreatePass(Pass: *const c_char) -> Option<&'static mut Pass>; + pub fn LLVMRustCreateAddressSanitizerFunctionPass(Recover: bool) -> &'static mut Pass; + pub fn LLVMRustCreateModuleAddressSanitizerPass(Recover: bool) -> &'static mut Pass; + pub fn LLVMRustCreateMemorySanitizerPass( + TrackOrigins: c_int, + Recover: bool, + ) -> &'static mut Pass; + pub fn LLVMRustCreateThreadSanitizerPass() -> &'static mut Pass; + pub fn LLVMRustCreateHWAddressSanitizerPass(Recover: bool) -> &'static mut Pass; + pub fn LLVMRustAddPass(PM: &PassManager<'_>, Pass: &'static mut Pass); + pub fn LLVMRustAddLastExtensionPasses( + PMB: &PassManagerBuilder, + Passes: *const &'static mut Pass, + NumPasses: size_t, + ); + + pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; + + pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine); + pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; + pub fn LLVMRustGetTargetFeature( + T: &TargetMachine, + Index: size_t, + Feature: &mut *const c_char, + Desc: &mut *const c_char, + ); + + pub fn LLVMRustGetHostCPUName(len: *mut usize) -> *const c_char; + pub fn LLVMRustCreateTargetMachine( + Triple: *const c_char, + CPU: *const c_char, + Features: *const c_char, + Abi: *const c_char, + Model: CodeModel, + Reloc: RelocModel, + Level: CodeGenOptLevel, + UseSoftFP: bool, + FunctionSections: bool, + DataSections: bool, + UniqueSectionNames: bool, + TrapUnreachable: bool, + Singlethread: bool, + AsmComments: bool, + EmitStackSizeSection: bool, + RelaxELFRelocations: bool, + UseInitArray: bool, + SplitDwarfFile: *const c_char, + ) -> Option<&'static mut TargetMachine>; + pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine); + pub fn LLVMRustAddBuilderLibraryInfo<'a>( + PMB: &'a PassManagerBuilder, + M: &'a Module, + DisableSimplifyLibCalls: bool, + ); + pub fn LLVMRustConfigurePassManagerBuilder( + PMB: &PassManagerBuilder, + OptLevel: CodeGenOptLevel, + MergeFunctions: bool, + SLPVectorize: bool, + LoopVectorize: bool, + PrepareForThinLTO: bool, + PGOGenPath: *const c_char, + PGOUsePath: *const c_char, + PGOSampleUsePath: *const c_char, + SizeLevel: c_int, + ); + pub fn LLVMRustAddLibraryInfo<'a>( + PM: &PassManager<'a>, + M: &'a Module, + DisableSimplifyLibCalls: bool, + ); + pub fn LLVMRustRunFunctionPassManager<'a>(PM: &PassManager<'a>, M: &'a Module); + pub fn LLVMRustWriteOutputFile<'a>( + T: &'a TargetMachine, + PM: &PassManager<'a>, + M: &'a Module, + Output: *const c_char, + DwoOutput: *const c_char, + FileType: FileType, + ) -> LLVMRustResult; + pub fn LLVMRustOptimizeWithNewPassManager<'a>( + M: &'a Module, + TM: &'a TargetMachine, + OptLevel: PassBuilderOptLevel, + OptStage: OptStage, + NoPrepopulatePasses: bool, + VerifyIR: bool, + UseThinLTOBuffers: bool, + MergeFunctions: bool, + UnrollLoops: bool, + SLPVectorize: bool, + LoopVectorize: bool, + DisableSimplifyLibCalls: bool, + EmitLifetimeMarkers: bool, + SanitizerOptions: Option<&SanitizerOptions>, + PGOGenPath: *const c_char, + PGOUsePath: *const c_char, + InstrumentCoverage: bool, + InstrumentGCOV: bool, + PGOSampleUsePath: *const c_char, + DebugInfoForProfiling: bool, + llvm_selfprofiler: *mut c_void, + begin_callback: SelfProfileBeforePassCallback, + end_callback: SelfProfileAfterPassCallback, + ExtraPasses: *const c_char, + ExtraPassesLen: size_t, + LLVMPlugins: *const c_char, + LLVMPluginsLen: size_t, + ) -> LLVMRustResult; + pub fn LLVMRustPrintModule( + M: &Module, + Output: *const c_char, + Demangle: extern "C" fn(*const c_char, size_t, *mut c_char, size_t) -> size_t, + ) -> LLVMRustResult; + pub fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); + pub fn LLVMRustPrintPasses(); + pub fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); + pub fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool); + pub fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); + + pub fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; + pub fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; + pub fn LLVMRustArchiveIteratorNext<'a>( + AIR: &ArchiveIterator<'a>, + ) -> Option<&'a mut ArchiveChild<'a>>; + pub fn LLVMRustArchiveChildName(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; + pub fn LLVMRustArchiveChildData(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; + pub fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); + pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); + pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); + + #[allow(improper_ctypes)] + pub fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); + + #[allow(improper_ctypes)] + pub fn LLVMRustUnpackOptimizationDiagnostic<'a>( + DI: &'a DiagnosticInfo, + pass_name_out: &RustString, + function_out: &mut Option<&'a Value>, + loc_line_out: &mut c_uint, + loc_column_out: &mut c_uint, + loc_filename_out: &RustString, + message_out: &RustString, + ); + + pub fn LLVMRustUnpackInlineAsmDiagnostic<'a>( + DI: &'a DiagnosticInfo, + level_out: &mut DiagnosticLevel, + cookie_out: &mut c_uint, + message_out: &mut Option<&'a Twine>, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustWriteDiagnosticInfoToString(DI: &DiagnosticInfo, s: &RustString); + pub fn LLVMRustGetDiagInfoKind(DI: &DiagnosticInfo) -> DiagnosticKind; + + pub fn LLVMRustGetSMDiagnostic<'a>( + DI: &'a DiagnosticInfo, + cookie_out: &mut c_uint, + ) -> &'a SMDiagnostic; + + pub fn LLVMRustSetInlineAsmDiagnosticHandler( + C: &Context, + H: InlineAsmDiagHandlerTy, + CX: *mut c_void, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustUnpackSMDiagnostic( + d: &SMDiagnostic, + message_out: &RustString, + buffer_out: &RustString, + level_out: &mut DiagnosticLevel, + loc_out: &mut c_uint, + ranges_out: *mut c_uint, + num_ranges: &mut usize, + ) -> bool; + + pub fn LLVMRustWriteArchive( + Dst: *const c_char, + NumMembers: size_t, + Members: *const &RustArchiveMember<'_>, + WriteSymbtab: bool, + Kind: ArchiveKind, + ) -> LLVMRustResult; + pub fn LLVMRustArchiveMemberNew<'a>( + Filename: *const c_char, + Name: *const c_char, + Child: Option<&ArchiveChild<'a>>, + ) -> &'a mut RustArchiveMember<'a>; + pub fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); + + pub fn LLVMRustWriteImportLibrary( + ImportName: *const c_char, + Path: *const c_char, + Exports: *const LLVMRustCOFFShortExport, + NumExports: usize, + Machine: u16, + MinGW: bool, + ) -> LLVMRustResult; + + pub fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); + + pub fn LLVMRustBuildOperandBundleDef<'a>( + Name: *const c_char, + Inputs: *const &'a Value, + NumInputs: c_uint, + ) -> &'a mut OperandBundleDef<'a>; + pub fn LLVMRustFreeOperandBundleDef<'a>(Bundle: &'a mut OperandBundleDef<'a>); + + pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); + + pub fn LLVMRustSetComdat<'a>(M: &'a Module, V: &'a Value, Name: *const c_char, NameLen: size_t); + pub fn LLVMRustSetModulePICLevel(M: &Module); + pub fn LLVMRustSetModulePIELevel(M: &Module); + pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); + pub fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer; + pub fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8; + pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; + pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); + pub fn LLVMRustModuleCost(M: &Module) -> u64; + + pub fn LLVMRustThinLTOBufferCreate(M: &Module, is_thin: bool) -> &'static mut ThinLTOBuffer; + pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); + pub fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; + pub fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; + pub fn LLVMRustCreateThinLTOData( + Modules: *const ThinLTOModule, + NumModules: c_uint, + PreservedSymbols: *const *const c_char, + PreservedSymbolsLen: c_uint, + ) -> Option<&'static mut ThinLTOData>; + pub fn LLVMRustPrepareThinLTORename( + Data: &ThinLTOData, + Module: &Module, + Target: &TargetMachine, + ) -> bool; + pub fn LLVMRustPrepareThinLTOResolveWeak(Data: &ThinLTOData, Module: &Module) -> bool; + pub fn LLVMRustPrepareThinLTOInternalize(Data: &ThinLTOData, Module: &Module) -> bool; + pub fn LLVMRustPrepareThinLTOImport( + Data: &ThinLTOData, + Module: &Module, + Target: &TargetMachine, + ) -> bool; + pub fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); + pub fn LLVMRustParseBitcodeForLTO( + Context: &Context, + Data: *const u8, + len: usize, + Identifier: *const c_char, + ) -> Option<&Module>; + pub fn LLVMRustGetBitcodeSliceFromObjectData( + Data: *const u8, + len: usize, + out_len: &mut usize, + ) -> *const u8; + pub fn LLVMRustThinLTOGetDICompileUnit( + M: &Module, + CU1: &mut *mut c_void, + CU2: &mut *mut c_void, + ); + pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); + + pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; + pub fn LLVMRustLinkerAdd( + linker: &Linker<'_>, + bytecode: *const c_char, + bytecode_len: usize, + ) -> bool; + pub fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>); + #[allow(improper_ctypes)] + pub fn LLVMRustComputeLTOCacheKey( + key_out: &RustString, + mod_id: *const c_char, + data: &ThinLTOData, + ); + + pub fn LLVMRustContextGetDiagnosticHandler(Context: &Context) -> Option<&DiagnosticHandler>; + pub fn LLVMRustContextSetDiagnosticHandler( + context: &Context, + diagnostic_handler: Option<&DiagnosticHandler>, + ); + pub fn LLVMRustContextConfigureDiagnosticHandler( + context: &Context, + diagnostic_handler_callback: DiagnosticHandlerTy, + diagnostic_handler_context: *mut c_void, + remark_all_passes: bool, + remark_passes: *const *const c_char, + remark_passes_len: usize, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); + + pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs new file mode 100644 index 00000000000..6602a4ab863 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -0,0 +1,318 @@ +#![allow(non_snake_case)] + +pub use self::AtomicRmwBinOp::*; +pub use self::CallConv::*; +pub use self::CodeGenOptSize::*; +pub use self::IntPredicate::*; +pub use self::Linkage::*; +pub use self::MetadataType::*; +pub use self::RealPredicate::*; + +use libc::c_uint; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_llvm::RustString; +use std::cell::RefCell; +use std::ffi::{CStr, CString}; +use std::str::FromStr; +use std::string::FromUtf8Error; + +pub mod archive_ro; +pub mod diagnostic; +mod ffi; + +pub use self::ffi::*; + +impl LLVMRustResult { + pub fn into_result(self) -> Result<(), ()> { + match self { + LLVMRustResult::Success => Ok(()), + LLVMRustResult::Failure => Err(()), + } + } +} + +pub fn AddFunctionAttributes<'ll>(llfn: &'ll Value, idx: AttributePlace, attrs: &[&'ll Attribute]) { + unsafe { + LLVMRustAddFunctionAttributes(llfn, idx.as_uint(), attrs.as_ptr(), attrs.len()); + } +} + +pub fn AddCallSiteAttributes<'ll>( + callsite: &'ll Value, + idx: AttributePlace, + attrs: &[&'ll Attribute], +) { + unsafe { + LLVMRustAddCallSiteAttributes(callsite, idx.as_uint(), attrs.as_ptr(), attrs.len()); + } +} + +pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &str, value: &str) -> &'ll Attribute { + unsafe { + LLVMCreateStringAttribute( + llcx, + attr.as_ptr().cast(), + attr.len().try_into().unwrap(), + value.as_ptr().cast(), + value.len().try_into().unwrap(), + ) + } +} + +pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { + unsafe { + LLVMCreateStringAttribute( + llcx, + attr.as_ptr().cast(), + attr.len().try_into().unwrap(), + std::ptr::null(), + 0, + ) + } +} + +pub fn CreateAlignmentAttr(llcx: &Context, bytes: u64) -> &Attribute { + unsafe { LLVMRustCreateAlignmentAttr(llcx, bytes) } +} + +pub fn CreateDereferenceableAttr(llcx: &Context, bytes: u64) -> &Attribute { + unsafe { LLVMRustCreateDereferenceableAttr(llcx, bytes) } +} + +pub fn CreateDereferenceableOrNullAttr(llcx: &Context, bytes: u64) -> &Attribute { + unsafe { LLVMRustCreateDereferenceableOrNullAttr(llcx, bytes) } +} + +pub fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { + unsafe { LLVMRustCreateByValAttr(llcx, ty) } +} + +pub fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { + unsafe { LLVMRustCreateStructRetAttr(llcx, ty) } +} + +pub fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { + unsafe { LLVMRustCreateUWTableAttr(llcx, async_) } +} + +pub fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute { + unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) } +} + +pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute { + unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } +} + +#[derive(Copy, Clone)] +pub enum AttributePlace { + ReturnValue, + Argument(u32), + Function, +} + +impl AttributePlace { + pub fn as_uint(self) -> c_uint { + match self { + AttributePlace::ReturnValue => 0, + AttributePlace::Argument(i) => 1 + i, + AttributePlace::Function => !0, + } + } +} + +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum CodeGenOptSize { + CodeGenOptSizeNone = 0, + CodeGenOptSizeDefault = 1, + CodeGenOptSizeAggressive = 2, +} + +impl FromStr for ArchiveKind { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "gnu" => Ok(ArchiveKind::K_GNU), + "bsd" => Ok(ArchiveKind::K_BSD), + "darwin" => Ok(ArchiveKind::K_DARWIN), + "coff" => Ok(ArchiveKind::K_COFF), + _ => Err(()), + } + } +} + +pub fn SetInstructionCallConv(instr: &Value, cc: CallConv) { + unsafe { + LLVMSetInstructionCallConv(instr, cc as c_uint); + } +} +pub fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { + unsafe { + LLVMSetFunctionCallConv(fn_, cc as c_uint); + } +} + +// Externally visible symbols that might appear in multiple codegen units need to appear in +// their own comdat section so that the duplicates can be discarded at link time. This can for +// example happen for generics when using multiple codegen units. This function simply uses the +// value's name as the comdat value to make sure that it is in a 1-to-1 relationship to the +// function. +// For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52 +pub fn SetUniqueComdat(llmod: &Module, val: &Value) { + unsafe { + let name = get_value_name(val); + LLVMRustSetComdat(llmod, val, name.as_ptr().cast(), name.len()); + } +} + +pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { + unsafe { + LLVMSetUnnamedAddress(global, unnamed); + } +} + +pub fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { + unsafe { + LLVMSetThreadLocalMode(global, mode); + } +} + +impl AttributeKind { + /// Create an LLVM Attribute with no associated value. + pub fn create_attr(self, llcx: &Context) -> &Attribute { + unsafe { LLVMRustCreateAttrNoValue(llcx, self) } + } +} + +pub fn set_section(llglobal: &Value, section_name: &str) { + let section_name_cstr = CString::new(section_name).expect("unexpected CString error"); + unsafe { + LLVMSetSection(llglobal, section_name_cstr.as_ptr()); + } +} + +pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name: &str) -> &'a Value { + let name_cstr = CString::new(name).expect("unexpected CString error"); + unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) } +} + +pub fn set_initializer(llglobal: &Value, constant_val: &Value) { + unsafe { + LLVMSetInitializer(llglobal, constant_val); + } +} + +pub fn set_global_constant(llglobal: &Value, is_constant: bool) { + unsafe { + LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); + } +} + +pub fn set_linkage(llglobal: &Value, linkage: Linkage) { + unsafe { + LLVMRustSetLinkage(llglobal, linkage); + } +} + +pub fn set_visibility(llglobal: &Value, visibility: Visibility) { + unsafe { + LLVMRustSetVisibility(llglobal, visibility); + } +} + +pub fn set_alignment(llglobal: &Value, bytes: usize) { + unsafe { + ffi::LLVMSetAlignment(llglobal, bytes as c_uint); + } +} + +pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &str) { + unsafe { + LLVMRustSetComdat(llmod, llglobal, name.as_ptr().cast(), name.len()); + } +} + +/// Safe wrapper around `LLVMGetParam`, because segfaults are no fun. +pub fn get_param(llfn: &Value, index: c_uint) -> &Value { + unsafe { + assert!( + index < LLVMCountParams(llfn), + "out of bounds argument access: {} out of {} arguments", + index, + LLVMCountParams(llfn) + ); + LLVMGetParam(llfn, index) + } +} + +/// Safe wrapper for `LLVMGetValueName2` into a byte slice +pub fn get_value_name(value: &Value) -> &[u8] { + unsafe { + let mut len = 0; + let data = LLVMGetValueName2(value, &mut len); + std::slice::from_raw_parts(data.cast(), len) + } +} + +/// Safe wrapper for `LLVMSetValueName2` from a byte slice +pub fn set_value_name(value: &Value, name: &[u8]) { + unsafe { + let data = name.as_ptr().cast(); + LLVMSetValueName2(value, data, name.len()); + } +} + +pub fn build_string(f: impl FnOnce(&RustString)) -> Result { + let sr = RustString { bytes: RefCell::new(Vec::new()) }; + f(&sr); + String::from_utf8(sr.bytes.into_inner()) +} + +pub fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec { + let sr = RustString { bytes: RefCell::new(Vec::new()) }; + f(&sr); + sr.bytes.into_inner() +} + +pub fn twine_to_string(tr: &Twine) -> String { + unsafe { + build_string(|s| LLVMRustWriteTwineToString(tr, s)).expect("got a non-UTF8 Twine from LLVM") + } +} + +pub fn last_error() -> Option { + unsafe { + let cstr = LLVMRustGetLastError(); + if cstr.is_null() { + None + } else { + let err = CStr::from_ptr(cstr).to_bytes(); + let err = String::from_utf8_lossy(err).to_string(); + libc::free(cstr as *mut _); + Some(err) + } + } +} + +pub struct OperandBundleDef<'a> { + pub raw: &'a mut ffi::OperandBundleDef<'a>, +} + +impl<'a> OperandBundleDef<'a> { + pub fn new(name: &str, vals: &[&'a Value]) -> Self { + let name = SmallCStr::new(name); + let def = unsafe { + LLVMRustBuildOperandBundleDef(name.as_ptr(), vals.as_ptr(), vals.len() as c_uint) + }; + OperandBundleDef { raw: def } + } +} + +impl Drop for OperandBundleDef<'_> { + fn drop(&mut self) { + unsafe { + LLVMRustFreeOperandBundleDef(&mut *(self.raw as *mut _)); + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs new file mode 100644 index 00000000000..a0a640473eb --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -0,0 +1,562 @@ +use crate::back::write::create_informational_target_machine; +use crate::{llvm, llvm_util}; +use libc::c_int; +use libloading::Library; +use rustc_codegen_ssa::target_features::{ + supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES, +}; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_fs_util::path_to_c_string; +use rustc_middle::bug; +use rustc_session::config::PrintRequest; +use rustc_session::Session; +use rustc_span::symbol::Symbol; +use rustc_target::spec::{MergeFunctions, PanicStrategy}; +use smallvec::{smallvec, SmallVec}; +use std::ffi::{CStr, CString}; +use tracing::debug; + +use std::mem; +use std::path::Path; +use std::ptr; +use std::slice; +use std::str; +use std::sync::Once; + +static INIT: Once = Once::new(); + +pub(crate) fn init(sess: &Session) { + unsafe { + // Before we touch LLVM, make sure that multithreading is enabled. + if llvm::LLVMIsMultithreaded() != 1 { + bug!("LLVM compiled without support for threads"); + } + INIT.call_once(|| { + configure_llvm(sess); + }); + } +} + +fn require_inited() { + if !INIT.is_completed() { + bug!("LLVM is not initialized"); + } +} + +unsafe fn configure_llvm(sess: &Session) { + let n_args = sess.opts.cg.llvm_args.len() + sess.target.llvm_args.len(); + let mut llvm_c_strs = Vec::with_capacity(n_args + 1); + let mut llvm_args = Vec::with_capacity(n_args + 1); + + llvm::LLVMRustInstallFatalErrorHandler(); + // On Windows, an LLVM assertion will open an Abort/Retry/Ignore dialog + // box for the purpose of launching a debugger. However, on CI this will + // cause it to hang until it times out, which can take several hours. + if std::env::var_os("CI").is_some() { + llvm::LLVMRustDisableSystemDialogsOnCrash(); + } + + fn llvm_arg_to_arg_name(full_arg: &str) -> &str { + full_arg.trim().split(|c: char| c == '=' || c.is_whitespace()).next().unwrap_or("") + } + + let cg_opts = sess.opts.cg.llvm_args.iter().map(AsRef::as_ref); + let tg_opts = sess.target.llvm_args.iter().map(AsRef::as_ref); + let sess_args = cg_opts.chain(tg_opts); + + let user_specified_args: FxHashSet<_> = + sess_args.clone().map(|s| llvm_arg_to_arg_name(s)).filter(|s| !s.is_empty()).collect(); + + { + // This adds the given argument to LLVM. Unless `force` is true + // user specified arguments are *not* overridden. + let mut add = |arg: &str, force: bool| { + if force || !user_specified_args.contains(llvm_arg_to_arg_name(arg)) { + let s = CString::new(arg).unwrap(); + llvm_args.push(s.as_ptr()); + llvm_c_strs.push(s); + } + }; + // Set the llvm "program name" to make usage and invalid argument messages more clear. + add("rustc -Cllvm-args=\"...\" with", true); + if sess.time_llvm_passes() { + add("-time-passes", false); + } + if sess.print_llvm_passes() { + add("-debug-pass=Structure", false); + } + if sess.target.generate_arange_section + && !sess.opts.unstable_opts.no_generate_arange_section + { + add("-generate-arange-section", false); + } + + // Disable the machine outliner by default in LLVM versions 11 and LLVM + // version 12, where it leads to miscompilation. + // + // Ref: + // - https://github.com/rust-lang/rust/issues/85351 + // - https://reviews.llvm.org/D103167 + if llvm_util::get_version() < (13, 0, 0) { + add("-enable-machine-outliner=never", false); + } + + match sess.opts.unstable_opts.merge_functions.unwrap_or(sess.target.merge_functions) { + MergeFunctions::Disabled | MergeFunctions::Trampolines => {} + MergeFunctions::Aliases => { + add("-mergefunc-use-aliases", false); + } + } + + if sess.target.os == "emscripten" && sess.panic_strategy() == PanicStrategy::Unwind { + add("-enable-emscripten-cxx-exceptions", false); + } + + // HACK(eddyb) LLVM inserts `llvm.assume` calls to preserve align attributes + // during inlining. Unfortunately these may block other optimizations. + add("-preserve-alignment-assumptions-during-inlining=false", false); + + // Use non-zero `import-instr-limit` multiplier for cold callsites. + add("-import-cold-multiplier=0.1", false); + + for arg in sess_args { + add(&(*arg), true); + } + } + + if sess.opts.unstable_opts.llvm_time_trace { + llvm::LLVMTimeTraceProfilerInitialize(); + } + + llvm::LLVMInitializePasses(); + + // Use the legacy plugin registration if we don't use the new pass manager + if !should_use_new_llvm_pass_manager( + &sess.opts.unstable_opts.new_llvm_pass_manager, + &sess.target.arch, + ) { + // Register LLVM plugins by loading them into the compiler process. + for plugin in &sess.opts.unstable_opts.llvm_plugins { + let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); + debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); + + // Intentionally leak the dynamic library. We can't ever unload it + // since the library can make things that will live arbitrarily long. + mem::forget(lib); + } + } + + rustc_llvm::initialize_available_targets(); + + llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()); +} + +pub fn time_trace_profiler_finish(file_name: &Path) { + unsafe { + let file_name = path_to_c_string(file_name); + llvm::LLVMTimeTraceProfilerFinish(file_name.as_ptr()); + } +} + +// WARNING: the features after applying `to_llvm_features` must be known +// to LLVM or the feature detection code will walk past the end of the feature +// array, leading to crashes. +// +// To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def +// where the * matches the architecture's name +// Beware to not use the llvm github project for this, but check the git submodule +// found in src/llvm-project +// Though note that Rust can also be build with an external precompiled version of LLVM +// which might lead to failures if the oldest tested / supported LLVM version +// doesn't yet support the relevant intrinsics +pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> { + let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; + match (arch, s) { + ("x86", "sse4.2") => { + if get_version() >= (14, 0, 0) { + smallvec!["sse4.2", "crc32"] + } else { + smallvec!["sse4.2"] + } + } + ("x86", "pclmulqdq") => smallvec!["pclmul"], + ("x86", "rdrand") => smallvec!["rdrnd"], + ("x86", "bmi1") => smallvec!["bmi"], + ("x86", "cmpxchg16b") => smallvec!["cx16"], + ("x86", "avx512vaes") => smallvec!["vaes"], + ("x86", "avx512gfni") => smallvec!["gfni"], + ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"], + ("aarch64", "rcpc2") => smallvec!["rcpc-immo"], + ("aarch64", "dpb") => smallvec!["ccpp"], + ("aarch64", "dpb2") => smallvec!["ccdp"], + ("aarch64", "frintts") => smallvec!["fptoint"], + ("aarch64", "fcma") => smallvec!["complxnum"], + ("aarch64", "pmuv3") => smallvec!["perfmon"], + ("aarch64", "paca") => smallvec!["pauth"], + ("aarch64", "pacg") => smallvec!["pauth"], + // Rust ties fp and neon together. In LLVM neon implicitly enables fp, + // but we manually enable neon when a feature only implicitly enables fp + ("aarch64", "f32mm") => smallvec!["f32mm", "neon"], + ("aarch64", "f64mm") => smallvec!["f64mm", "neon"], + ("aarch64", "fhm") => smallvec!["fp16fml", "neon"], + ("aarch64", "fp16") => smallvec!["fullfp16", "neon"], + ("aarch64", "jsconv") => smallvec!["jsconv", "neon"], + ("aarch64", "sve") => smallvec!["sve", "neon"], + ("aarch64", "sve2") => smallvec!["sve2", "neon"], + ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"], + ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"], + ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"], + ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"], + (_, s) => smallvec![s], + } +} + +// Given a map from target_features to whether they are enabled or disabled, +// ensure only valid combinations are allowed. +pub fn check_tied_features( + sess: &Session, + features: &FxHashMap<&str, bool>, +) -> Option<&'static [&'static str]> { + if !features.is_empty() { + for tied in tied_target_features(sess) { + // Tied features must be set to the same value, or not set at all + let mut tied_iter = tied.iter(); + let enabled = features.get(tied_iter.next().unwrap()); + if tied_iter.any(|f| enabled != features.get(f)) { + return Some(tied); + } + } + } + return None; +} + +// Used to generate cfg variables and apply features +// Must express features in the way Rust understands them +pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec { + let target_machine = create_informational_target_machine(sess); + let mut features: Vec = supported_target_features(sess) + .iter() + .filter_map(|&(feature, gate)| { + if sess.is_nightly_build() || allow_unstable || gate.is_none() { + Some(feature) + } else { + None + } + }) + .filter(|feature| { + // check that all features in a given smallvec are enabled + for llvm_feature in to_llvm_features(sess, feature) { + let cstr = SmallCStr::new(llvm_feature); + if !unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { + return false; + } + } + true + }) + .map(|feature| Symbol::intern(feature)) + .collect(); + + // LLVM 14 changed the ABI for i128 arguments to __float/__fix builtins on Win64 + // (see https://reviews.llvm.org/D110413). This unstable target feature is intended for use + // by compiler-builtins, to export the builtins with the expected, LLVM-version-dependent ABI. + // The target feature can be dropped once we no longer support older LLVM versions. + if sess.is_nightly_build() && get_version() >= (14, 0, 0) { + features.push(Symbol::intern("llvm14-builtins-abi")); + } + features +} + +pub fn print_version() { + let (major, minor, patch) = get_version(); + println!("LLVM version: {}.{}.{}", major, minor, patch); +} + +pub fn get_version() -> (u32, u32, u32) { + // Can be called without initializing LLVM + unsafe { + (llvm::LLVMRustVersionMajor(), llvm::LLVMRustVersionMinor(), llvm::LLVMRustVersionPatch()) + } +} + +pub fn print_passes() { + // Can be called without initializing LLVM + unsafe { + llvm::LLVMRustPrintPasses(); + } +} + +fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> { + let len = unsafe { llvm::LLVMRustGetTargetFeaturesCount(tm) }; + let mut ret = Vec::with_capacity(len); + for i in 0..len { + unsafe { + let mut feature = ptr::null(); + let mut desc = ptr::null(); + llvm::LLVMRustGetTargetFeature(tm, i, &mut feature, &mut desc); + if feature.is_null() || desc.is_null() { + bug!("LLVM returned a `null` target feature string"); + } + let feature = CStr::from_ptr(feature).to_str().unwrap_or_else(|e| { + bug!("LLVM returned a non-utf8 feature string: {}", e); + }); + let desc = CStr::from_ptr(desc).to_str().unwrap_or_else(|e| { + bug!("LLVM returned a non-utf8 feature string: {}", e); + }); + ret.push((feature, desc)); + } + } + ret +} + +fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { + let mut target_features = llvm_target_features(tm); + let mut rustc_target_features = supported_target_features(sess) + .iter() + .filter_map(|(feature, _gate)| { + for llvm_feature in to_llvm_features(sess, *feature) { + // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. + match target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok().map( + |index| { + let (_f, desc) = target_features.remove(index); + (*feature, desc) + }, + ) { + Some(v) => return Some(v), + None => {} + } + } + None + }) + .collect::>(); + rustc_target_features.extend_from_slice(&[( + "crt-static", + "Enables C Run-time Libraries to be statically linked", + )]); + let max_feature_len = target_features + .iter() + .chain(rustc_target_features.iter()) + .map(|(feature, _desc)| feature.len()) + .max() + .unwrap_or(0); + + println!("Features supported by rustc for this target:"); + for (feature, desc) in &rustc_target_features { + println!(" {1:0$} - {2}.", max_feature_len, feature, desc); + } + println!("\nCode-generation features supported by LLVM for this target:"); + for (feature, desc) in &target_features { + println!(" {1:0$} - {2}.", max_feature_len, feature, desc); + } + if target_features.is_empty() { + println!(" Target features listing is not supported by this LLVM version."); + } + println!("\nUse +feature to enable a feature, or -feature to disable it."); + println!("For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n"); + println!("Code-generation features cannot be used in cfg or #[target_feature],"); + println!("and may be renamed or removed in a future version of LLVM or rustc.\n"); +} + +pub(crate) fn print(req: PrintRequest, sess: &Session) { + require_inited(); + let tm = create_informational_target_machine(sess); + match req { + PrintRequest::TargetCPUs => unsafe { llvm::LLVMRustPrintTargetCPUs(tm) }, + PrintRequest::TargetFeatures => print_target_features(sess, tm), + _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), + } +} + +fn handle_native(name: &str) -> &str { + if name != "native" { + return name; + } + + unsafe { + let mut len = 0; + let ptr = llvm::LLVMRustGetHostCPUName(&mut len); + str::from_utf8(slice::from_raw_parts(ptr as *const u8, len)).unwrap() + } +} + +pub fn target_cpu(sess: &Session) -> &str { + match sess.opts.cg.target_cpu { + Some(ref name) => handle_native(name), + None => handle_native(sess.target.cpu.as_ref()), + } +} + +/// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`, +/// `--target` and similar). +pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec { + // Features that come earlier are overridden by conflicting features later in the string. + // Typically we'll want more explicit settings to override the implicit ones, so: + // + // * Features from -Ctarget-cpu=*; are overridden by [^1] + // * Features implied by --target; are overridden by + // * Features from -Ctarget-feature; are overridden by + // * function specific features. + // + // [^1]: target-cpu=native is handled here, other target-cpu values are handled implicitly + // through LLVM TargetMachine implementation. + // + // FIXME(nagisa): it isn't clear what's the best interaction between features implied by + // `-Ctarget-cpu` and `--target` are. On one hand, you'd expect CLI arguments to always + // override anything that's implicit, so e.g. when there's no `--target` flag, features implied + // the host target are overridden by `-Ctarget-cpu=*`. On the other hand, what about when both + // `--target` and `-Ctarget-cpu=*` are specified? Both then imply some target features and both + // flags are specified by the user on the CLI. It isn't as clear-cut which order of precedence + // should be taken in cases like these. + let mut features = vec![]; + + // -Ctarget-cpu=native + match sess.opts.cg.target_cpu { + Some(ref s) if s == "native" => { + let features_string = unsafe { + let ptr = llvm::LLVMGetHostCPUFeatures(); + let features_string = if !ptr.is_null() { + CStr::from_ptr(ptr) + .to_str() + .unwrap_or_else(|e| { + bug!("LLVM returned a non-utf8 features string: {}", e); + }) + .to_owned() + } else { + bug!("could not allocate host CPU features, LLVM returned a `null` string"); + }; + + llvm::LLVMDisposeMessage(ptr); + + features_string + }; + features.extend(features_string.split(',').map(String::from)); + } + Some(_) | None => {} + }; + + // Features implied by an implicit or explicit `--target`. + features.extend( + sess.target + .features + .split(',') + .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) + .map(String::from), + ); + + // -Ctarget-features + let supported_features = supported_target_features(sess); + let mut featsmap = FxHashMap::default(); + let feats = sess + .opts + .cg + .target_feature + .split(',') + .filter_map(|s| { + let enable_disable = match s.chars().next() { + None => return None, + Some(c @ '+' | c @ '-') => c, + Some(_) => { + if diagnostics { + let mut diag = sess.struct_warn(&format!( + "unknown feature specified for `-Ctarget-feature`: `{}`", + s + )); + diag.note("features must begin with a `+` to enable or `-` to disable it"); + diag.emit(); + } + return None; + } + }; + + let feature = backend_feature_name(s)?; + // Warn against use of LLVM specific feature names on the CLI. + if diagnostics && !supported_features.iter().any(|&(v, _)| v == feature) { + let rust_feature = supported_features.iter().find_map(|&(rust_feature, _)| { + let llvm_features = to_llvm_features(sess, rust_feature); + if llvm_features.contains(&feature) && !llvm_features.contains(&rust_feature) { + Some(rust_feature) + } else { + None + } + }); + let mut diag = sess.struct_warn(&format!( + "unknown feature specified for `-Ctarget-feature`: `{}`", + feature + )); + diag.note("it is still passed through to the codegen backend"); + if let Some(rust_feature) = rust_feature { + diag.help(&format!("you might have meant: `{}`", rust_feature)); + } else { + diag.note("consider filing a feature request"); + } + diag.emit(); + } + + if diagnostics { + // FIXME(nagisa): figure out how to not allocate a full hashset here. + featsmap.insert(feature, enable_disable == '+'); + } + + // rustc-specific features do not get passed down to LLVM… + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } + // ... otherwise though we run through `to_llvm_features` when + // passing requests down to LLVM. This means that all in-language + // features also work on the command line instead of having two + // different names when the LLVM name and the Rust name differ. + Some( + to_llvm_features(sess, feature) + .into_iter() + .map(move |f| format!("{}{}", enable_disable, f)), + ) + }) + .flatten(); + features.extend(feats); + + if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { + sess.err(&format!( + "target features {} must all be enabled or disabled together", + f.join(", ") + )); + } + + features +} + +/// Returns a feature name for the given `+feature` or `-feature` string. +/// +/// Only allows features that are backend specific (i.e. not [`RUSTC_SPECIFIC_FEATURES`].) +fn backend_feature_name(s: &str) -> Option<&str> { + // features must start with a `+` or `-`. + let feature = s.strip_prefix(&['+', '-'][..]).unwrap_or_else(|| { + bug!("target feature `{}` must begin with a `+` or `-`", s); + }); + // Rustc-specific feature requests like `+crt-static` or `-crt-static` + // are not passed down to LLVM. + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } + Some(feature) +} + +pub fn tune_cpu(sess: &Session) -> Option<&str> { + let name = sess.opts.unstable_opts.tune_cpu.as_ref()?; + Some(handle_native(name)) +} + +pub(crate) fn should_use_new_llvm_pass_manager(user_opt: &Option, target_arch: &str) -> bool { + // The new pass manager is enabled by default for LLVM >= 13. + // This matches Clang, which also enables it since Clang 13. + + // Since LLVM 15, the legacy pass manager is no longer supported. + if llvm_util::get_version() >= (15, 0, 0) { + return true; + } + + // There are some perf issues with the new pass manager when targeting + // s390x with LLVM 13, so enable the new pass manager only with LLVM 14. + // See https://github.com/rust-lang/rust/issues/89609. + let min_version = if target_arch == "s390x" { 14 } else { 13 }; + user_opt.unwrap_or_else(|| llvm_util::get_version() >= (min_version, 0, 0)) +} diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs new file mode 100644 index 00000000000..6e94284852f --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -0,0 +1,150 @@ +use crate::attributes; +use crate::base; +use crate::context::CodegenCx; +use crate::llvm; +use crate::type_of::LayoutLlvmExt; +use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +pub use rustc_middle::mir::mono::MonoItem; +use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; +use rustc_middle::ty::{self, Instance, TypeVisitable}; +use rustc_session::config::CrateType; +use rustc_target::spec::RelocModel; +use tracing::debug; + +impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { + fn predefine_static( + &self, + def_id: DefId, + linkage: Linkage, + visibility: Visibility, + symbol_name: &str, + ) { + let instance = Instance::mono(self.tcx, def_id); + let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + let llty = self.layout_of(ty).llvm_type(self); + + let g = self.define_global(symbol_name, llty).unwrap_or_else(|| { + self.sess().span_fatal( + self.tcx.def_span(def_id), + &format!("symbol `{}` is already defined", symbol_name), + ) + }); + + unsafe { + llvm::LLVMRustSetLinkage(g, base::linkage_to_llvm(linkage)); + llvm::LLVMRustSetVisibility(g, base::visibility_to_llvm(visibility)); + if self.should_assume_dso_local(g, false) { + llvm::LLVMRustSetDSOLocal(g, true); + } + } + + self.instances.borrow_mut().insert(instance, g); + } + + fn predefine_fn( + &self, + instance: Instance<'tcx>, + linkage: Linkage, + visibility: Visibility, + symbol_name: &str, + ) { + assert!(!instance.substs.needs_infer()); + + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + let lldecl = self.declare_fn(symbol_name, fn_abi); + unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; + let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); + base::set_link_section(lldecl, attrs); + if linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR { + llvm::SetUniqueComdat(self.llmod, lldecl); + } + + // If we're compiling the compiler-builtins crate, e.g., the equivalent of + // compiler-rt, then we want to implicitly compile everything with hidden + // visibility as we're going to link this object all over the place but + // don't want the symbols to get exported. + if linkage != Linkage::Internal + && linkage != Linkage::Private + && self.tcx.is_compiler_builtins(LOCAL_CRATE) + { + unsafe { + llvm::LLVMRustSetVisibility(lldecl, llvm::Visibility::Hidden); + } + } else { + unsafe { + llvm::LLVMRustSetVisibility(lldecl, base::visibility_to_llvm(visibility)); + } + } + + debug!("predefine_fn: instance = {:?}", instance); + + attributes::from_fn_attrs(self, lldecl, instance); + + unsafe { + if self.should_assume_dso_local(lldecl, false) { + llvm::LLVMRustSetDSOLocal(lldecl, true); + } + } + + self.instances.borrow_mut().insert(instance, lldecl); + } +} + +impl CodegenCx<'_, '_> { + /// Whether a definition or declaration can be assumed to be local to a group of + /// libraries that form a single DSO or executable. + pub(crate) unsafe fn should_assume_dso_local( + &self, + llval: &llvm::Value, + is_declaration: bool, + ) -> bool { + let linkage = llvm::LLVMRustGetLinkage(llval); + let visibility = llvm::LLVMRustGetVisibility(llval); + + if matches!(linkage, llvm::Linkage::InternalLinkage | llvm::Linkage::PrivateLinkage) { + return true; + } + + if visibility != llvm::Visibility::Default && linkage != llvm::Linkage::ExternalWeakLinkage + { + return true; + } + + // Symbols from executables can't really be imported any further. + let all_exe = self.tcx.sess.crate_types().iter().all(|ty| *ty == CrateType::Executable); + let is_declaration_for_linker = + is_declaration || linkage == llvm::Linkage::AvailableExternallyLinkage; + if all_exe && !is_declaration_for_linker { + return true; + } + + // PowerPC64 prefers TOC indirection to avoid copy relocations. + if matches!(&*self.tcx.sess.target.arch, "powerpc64" | "powerpc64le") { + return false; + } + + // Thread-local variables generally don't support copy relocations. + let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) + .map(|v| llvm::LLVMIsThreadLocal(v) == llvm::True) + .unwrap_or(false); + if is_thread_local_var { + return false; + } + + // Match clang by only supporting COFF and ELF for now. + if self.tcx.sess.target.is_like_osx { + return false; + } + + // Static relocation model should force copy relocations everywhere. + if self.tcx.sess.relocation_model() == RelocModel::Static { + return true; + } + + // With pie relocation model calls of functions defined in the translation + // unit can use copy relocations. + self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration + } +} diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs new file mode 100644 index 00000000000..eeb38d4ecf5 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -0,0 +1,319 @@ +pub use crate::llvm::Type; + +use crate::abi::{FnAbiLlvmExt, LlvmType}; +use crate::common; +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm::{Bool, False, True}; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; +use rustc_codegen_ssa::common::TypeKind; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_middle::bug; +use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::ty::{self, Ty}; +use rustc_target::abi::call::{CastTarget, FnAbi, Reg}; +use rustc_target::abi::{AddressSpace, Align, Integer, Size}; + +use std::fmt; +use std::ptr; + +use libc::{c_char, c_uint}; + +impl PartialEq for Type { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self, other) + } +} + +impl fmt::Debug for Type { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str( + &llvm::build_string(|s| unsafe { + llvm::LLVMRustWriteTypeToString(self, s); + }) + .expect("non-UTF8 type description from LLVM"), + ) + } +} + +impl<'ll> CodegenCx<'ll, '_> { + pub(crate) fn type_named_struct(&self, name: &str) -> &'ll Type { + let name = SmallCStr::new(name); + unsafe { llvm::LLVMStructCreateNamed(self.llcx, name.as_ptr()) } + } + + pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { + unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) } + } + + pub(crate) fn type_void(&self) -> &'ll Type { + unsafe { llvm::LLVMVoidTypeInContext(self.llcx) } + } + + pub(crate) fn type_metadata(&self) -> &'ll Type { + unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } + } + + ///x Creates an integer type with the given number of bits, e.g., i24 + pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type { + unsafe { llvm::LLVMIntTypeInContext(self.llcx, num_bits as c_uint) } + } + + pub(crate) fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { + unsafe { llvm::LLVMVectorType(ty, len as c_uint) } + } + + pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { + unsafe { + let n_args = llvm::LLVMCountParamTypes(ty) as usize; + let mut args = Vec::with_capacity(n_args); + llvm::LLVMGetParamTypes(ty, args.as_mut_ptr()); + args.set_len(n_args); + args + } + } + + pub(crate) fn type_bool(&self) -> &'ll Type { + self.type_i8() + } + + pub(crate) fn type_int_from_ty(&self, t: ty::IntTy) -> &'ll Type { + match t { + ty::IntTy::Isize => self.type_isize(), + ty::IntTy::I8 => self.type_i8(), + ty::IntTy::I16 => self.type_i16(), + ty::IntTy::I32 => self.type_i32(), + ty::IntTy::I64 => self.type_i64(), + ty::IntTy::I128 => self.type_i128(), + } + } + + pub(crate) fn type_uint_from_ty(&self, t: ty::UintTy) -> &'ll Type { + match t { + ty::UintTy::Usize => self.type_isize(), + ty::UintTy::U8 => self.type_i8(), + ty::UintTy::U16 => self.type_i16(), + ty::UintTy::U32 => self.type_i32(), + ty::UintTy::U64 => self.type_i64(), + ty::UintTy::U128 => self.type_i128(), + } + } + + pub(crate) fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { + match t { + ty::FloatTy::F32 => self.type_f32(), + ty::FloatTy::F64 => self.type_f64(), + } + } + + pub(crate) fn type_pointee_for_align(&self, align: Align) -> &'ll Type { + // FIXME(eddyb) We could find a better approximation if ity.align < align. + let ity = Integer::approximate_align(self, align); + self.type_from_integer(ity) + } + + /// Return a LLVM type that has at most the required alignment, + /// and exactly the required size, as a best-effort padding array. + pub(crate) fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { + let unit = Integer::approximate_align(self, align); + let size = size.bytes(); + let unit_size = unit.size().bytes(); + assert_eq!(size % unit_size, 0); + self.type_array(self.type_from_integer(unit), size / unit_size) + } + + pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } + } + + pub(crate) fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { + unsafe { llvm::LLVMRustArrayType(ty, len) } + } +} + +impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn type_i1(&self) -> &'ll Type { + unsafe { llvm::LLVMInt1TypeInContext(self.llcx) } + } + + fn type_i8(&self) -> &'ll Type { + unsafe { llvm::LLVMInt8TypeInContext(self.llcx) } + } + + fn type_i16(&self) -> &'ll Type { + unsafe { llvm::LLVMInt16TypeInContext(self.llcx) } + } + + fn type_i32(&self) -> &'ll Type { + unsafe { llvm::LLVMInt32TypeInContext(self.llcx) } + } + + fn type_i64(&self) -> &'ll Type { + unsafe { llvm::LLVMInt64TypeInContext(self.llcx) } + } + + fn type_i128(&self) -> &'ll Type { + unsafe { llvm::LLVMIntTypeInContext(self.llcx, 128) } + } + + fn type_isize(&self) -> &'ll Type { + self.isize_ty + } + + fn type_f32(&self) -> &'ll Type { + unsafe { llvm::LLVMFloatTypeInContext(self.llcx) } + } + + fn type_f64(&self) -> &'ll Type { + unsafe { llvm::LLVMDoubleTypeInContext(self.llcx) } + } + + fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) } + } + + fn type_struct(&self, els: &[&'ll Type], packed: bool) -> &'ll Type { + unsafe { + llvm::LLVMStructTypeInContext( + self.llcx, + els.as_ptr(), + els.len() as c_uint, + packed as Bool, + ) + } + } + + fn type_kind(&self, ty: &'ll Type) -> TypeKind { + unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() } + } + + fn type_ptr_to(&self, ty: &'ll Type) -> &'ll Type { + assert_ne!( + self.type_kind(ty), + TypeKind::Function, + "don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead or explicitly specify an address space if it makes sense" + ); + ty.ptr_to(AddressSpace::DATA) + } + + fn type_ptr_to_ext(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type { + ty.ptr_to(address_space) + } + + fn element_type(&self, ty: &'ll Type) -> &'ll Type { + match self.type_kind(ty) { + TypeKind::Array | TypeKind::Vector => unsafe { llvm::LLVMGetElementType(ty) }, + TypeKind::Pointer => bug!("element_type is not supported for opaque pointers"), + other => bug!("element_type called on unsupported type {:?}", other), + } + } + + fn vector_length(&self, ty: &'ll Type) -> usize { + unsafe { llvm::LLVMGetVectorSize(ty) as usize } + } + + fn float_width(&self, ty: &'ll Type) -> usize { + match self.type_kind(ty) { + TypeKind::Float => 32, + TypeKind::Double => 64, + TypeKind::X86_FP80 => 80, + TypeKind::FP128 | TypeKind::PPC_FP128 => 128, + _ => bug!("llvm_float_width called on a non-float type"), + } + } + + fn int_width(&self, ty: &'ll Type) -> u64 { + unsafe { llvm::LLVMGetIntTypeWidth(ty) as u64 } + } + + fn val_ty(&self, v: &'ll Value) -> &'ll Type { + common::val_ty(v) + } +} + +impl Type { + pub fn i8_llcx(llcx: &llvm::Context) -> &Type { + unsafe { llvm::LLVMInt8TypeInContext(llcx) } + } + + // Creates an integer type with the given number of bits, e.g., i24 + pub fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { + unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) } + } + + pub fn i8p_llcx(llcx: &llvm::Context) -> &Type { + Type::i8_llcx(llcx).ptr_to(AddressSpace::DATA) + } + + fn ptr_to(&self, address_space: AddressSpace) -> &Type { + unsafe { llvm::LLVMPointerType(self, address_space.0) } + } +} + +impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type { + layout.llvm_type(self) + } + fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type { + layout.immediate_llvm_type(self) + } + fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool { + layout.is_llvm_immediate() + } + fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool { + layout.is_llvm_scalar_pair() + } + fn backend_field_index(&self, layout: TyAndLayout<'tcx>, index: usize) -> u64 { + layout.llvm_field_index(self, index) + } + fn scalar_pair_element_backend_type( + &self, + layout: TyAndLayout<'tcx>, + index: usize, + immediate: bool, + ) -> &'ll Type { + layout.scalar_pair_element_llvm_type(self, index, immediate) + } + fn cast_backend_type(&self, ty: &CastTarget) -> &'ll Type { + ty.llvm_type(self) + } + fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + fn_abi.llvm_type(self) + } + fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + fn_abi.ptr_to_llvm_type(self) + } + fn reg_backend_type(&self, ty: &Reg) -> &'ll Type { + ty.llvm_type(self) + } +} + +impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn set_type_metadata(&self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMGlobalSetMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + + fn typeid_metadata(&self, typeid: String) -> &'ll Value { + unsafe { + llvm::LLVMMDStringInContext( + self.llcx, + typeid.as_ptr() as *const c_char, + typeid.len() as c_uint, + ) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs new file mode 100644 index 00000000000..9f0e6c80b19 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -0,0 +1,418 @@ +use crate::common::*; +use crate::context::TypeLowering; +use crate::llvm_util::get_version; +use crate::type_::Type; +use rustc_codegen_ssa::traits::*; +use rustc_middle::bug; +use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; +use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; +use rustc_middle::ty::{self, Ty, TypeVisitable}; +use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape}; +use rustc_target::abi::{Int, Pointer, F32, F64}; +use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants}; +use smallvec::{smallvec, SmallVec}; +use tracing::debug; + +use std::fmt::Write; + +fn uncached_llvm_type<'a, 'tcx>( + cx: &CodegenCx<'a, 'tcx>, + layout: TyAndLayout<'tcx>, + defer: &mut Option<(&'a Type, TyAndLayout<'tcx>)>, + field_remapping: &mut Option>, +) -> &'a Type { + match layout.abi { + Abi::Scalar(_) => bug!("handled elsewhere"), + Abi::Vector { element, count } => { + let element = layout.scalar_llvm_type_at(cx, element, Size::ZERO); + return cx.type_vector(element, count); + } + Abi::ScalarPair(..) => { + return cx.type_struct( + &[ + layout.scalar_pair_element_llvm_type(cx, 0, false), + layout.scalar_pair_element_llvm_type(cx, 1, false), + ], + false, + ); + } + Abi::Uninhabited | Abi::Aggregate { .. } => {} + } + + let name = match layout.ty.kind() { + // FIXME(eddyb) producing readable type names for trait objects can result + // in problematically distinct types due to HRTB and subtyping (see #47638). + // ty::Dynamic(..) | + ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str + // For performance reasons we use names only when emitting LLVM IR. Unless we are on + // LLVM < 14, where the use of unnamed types resulted in various issues, e.g., #76213, + // #79564, and #79246. + if get_version() < (14, 0, 0) || !cx.sess().fewer_names() => + { + let mut name = with_no_visible_paths!(with_no_trimmed_paths!(layout.ty.to_string())); + if let (&ty::Adt(def, _), &Variants::Single { index }) = + (layout.ty.kind(), &layout.variants) + { + if def.is_enum() && !def.variants().is_empty() { + write!(&mut name, "::{}", def.variant(index).name).unwrap(); + } + } + if let (&ty::Generator(_, _, _), &Variants::Single { index }) = + (layout.ty.kind(), &layout.variants) + { + write!(&mut name, "::{}", ty::GeneratorSubsts::variant_name(index)).unwrap(); + } + Some(name) + } + // Use identified structure types for ADT. Due to pointee types in LLVM IR their definition + // might be recursive. Other cases are non-recursive and we can use literal structure types. + ty::Adt(..) => Some(String::new()), + _ => None, + }; + + match layout.fields { + FieldsShape::Primitive | FieldsShape::Union(_) => { + let fill = cx.type_padding_filler(layout.size, layout.align.abi); + let packed = false; + match name { + None => cx.type_struct(&[fill], packed), + Some(ref name) => { + let llty = cx.type_named_struct(name); + cx.set_struct_body(llty, &[fill], packed); + llty + } + } + } + FieldsShape::Array { count, .. } => cx.type_array(layout.field(cx, 0).llvm_type(cx), count), + FieldsShape::Arbitrary { .. } => match name { + None => { + let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); + *field_remapping = new_field_remapping; + cx.type_struct(&llfields, packed) + } + Some(ref name) => { + let llty = cx.type_named_struct(name); + *defer = Some((llty, layout)); + llty + } + }, + } +} + +fn struct_llfields<'a, 'tcx>( + cx: &CodegenCx<'a, 'tcx>, + layout: TyAndLayout<'tcx>, +) -> (Vec<&'a Type>, bool, Option>) { + debug!("struct_llfields: {:#?}", layout); + let field_count = layout.fields.count(); + + let mut packed = false; + let mut offset = Size::ZERO; + let mut prev_effective_align = layout.align.abi; + let mut result: Vec<_> = Vec::with_capacity(1 + field_count * 2); + let mut field_remapping = smallvec![0; field_count]; + for i in layout.fields.index_by_increasing_offset() { + let target_offset = layout.fields.offset(i as usize); + let field = layout.field(cx, i); + let effective_field_align = + layout.align.abi.min(field.align.abi).restrict_for_offset(target_offset); + packed |= effective_field_align < field.align.abi; + + debug!( + "struct_llfields: {}: {:?} offset: {:?} target_offset: {:?} \ + effective_field_align: {}", + i, + field, + offset, + target_offset, + effective_field_align.bytes() + ); + assert!(target_offset >= offset); + let padding = target_offset - offset; + if padding != Size::ZERO { + let padding_align = prev_effective_align.min(effective_field_align); + assert_eq!(offset.align_to(padding_align) + padding, target_offset); + result.push(cx.type_padding_filler(padding, padding_align)); + debug!(" padding before: {:?}", padding); + } + field_remapping[i] = result.len() as u32; + result.push(field.llvm_type(cx)); + offset = target_offset + field.size; + prev_effective_align = effective_field_align; + } + let padding_used = result.len() > field_count; + if !layout.is_unsized() && field_count > 0 { + if offset > layout.size { + bug!("layout: {:#?} stride: {:?} offset: {:?}", layout, layout.size, offset); + } + let padding = layout.size - offset; + if padding != Size::ZERO { + let padding_align = prev_effective_align; + assert_eq!(offset.align_to(padding_align) + padding, layout.size); + debug!( + "struct_llfields: pad_bytes: {:?} offset: {:?} stride: {:?}", + padding, offset, layout.size + ); + result.push(cx.type_padding_filler(padding, padding_align)); + } + } else { + debug!("struct_llfields: offset: {:?} stride: {:?}", offset, layout.size); + } + let field_remapping = if padding_used { Some(field_remapping) } else { None }; + (result, packed, field_remapping) +} + +impl<'a, 'tcx> CodegenCx<'a, 'tcx> { + pub fn align_of(&self, ty: Ty<'tcx>) -> Align { + self.layout_of(ty).align.abi + } + + pub fn size_of(&self, ty: Ty<'tcx>) -> Size { + self.layout_of(ty).size + } + + pub fn size_and_align_of(&self, ty: Ty<'tcx>) -> (Size, Align) { + let layout = self.layout_of(ty); + (layout.size, layout.align.abi) + } +} + +pub trait LayoutLlvmExt<'tcx> { + fn is_llvm_immediate(&self) -> bool; + fn is_llvm_scalar_pair(&self) -> bool; + fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; + fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; + fn scalar_llvm_type_at<'a>( + &self, + cx: &CodegenCx<'a, 'tcx>, + scalar: Scalar, + offset: Size, + ) -> &'a Type; + fn scalar_pair_element_llvm_type<'a>( + &self, + cx: &CodegenCx<'a, 'tcx>, + index: usize, + immediate: bool, + ) -> &'a Type; + fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64; + fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option; +} + +impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { + fn is_llvm_immediate(&self) -> bool { + match self.abi { + Abi::Scalar(_) | Abi::Vector { .. } => true, + Abi::ScalarPair(..) => false, + Abi::Uninhabited | Abi::Aggregate { .. } => self.is_zst(), + } + } + + fn is_llvm_scalar_pair(&self) -> bool { + match self.abi { + Abi::ScalarPair(..) => true, + Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false, + } + } + + /// Gets the LLVM type corresponding to a Rust type, i.e., `rustc_middle::ty::Ty`. + /// The pointee type of the pointer in `PlaceRef` is always this type. + /// For sized types, it is also the right LLVM type for an `alloca` + /// containing a value of that type, and most immediates (except `bool`). + /// Unsized types, however, are represented by a "minimal unit", e.g. + /// `[T]` becomes `T`, while `str` and `Trait` turn into `i8` - this + /// is useful for indexing slices, as `&[T]`'s data pointer is `T*`. + /// If the type is an unsized struct, the regular layout is generated, + /// with the inner-most trailing unsized field using the "minimal unit" + /// of that field's type - this is useful for taking the address of + /// that field and ensuring the struct has the right alignment. + fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { + if let Abi::Scalar(scalar) = self.abi { + // Use a different cache for scalars because pointers to DSTs + // can be either fat or thin (data pointers of fat pointers). + if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { + return llty; + } + let llty = match *self.ty.kind() { + ty::Ref(_, ty, _) | ty::RawPtr(ty::TypeAndMut { ty, .. }) => { + cx.type_ptr_to(cx.layout_of(ty).llvm_type(cx)) + } + ty::Adt(def, _) if def.is_box() => { + cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).llvm_type(cx)) + } + ty::FnPtr(sig) => { + cx.fn_ptr_backend_type(cx.fn_abi_of_fn_ptr(sig, ty::List::empty())) + } + _ => self.scalar_llvm_type_at(cx, scalar, Size::ZERO), + }; + cx.scalar_lltypes.borrow_mut().insert(self.ty, llty); + return llty; + } + + // Check the cache. + let variant_index = match self.variants { + Variants::Single { index } => Some(index), + _ => None, + }; + if let Some(llty) = cx.type_lowering.borrow().get(&(self.ty, variant_index)) { + return llty.lltype; + } + + debug!("llvm_type({:#?})", self); + + assert!(!self.ty.has_escaping_bound_vars(), "{:?} has escaping bound vars", self.ty); + + // Make sure lifetimes are erased, to avoid generating distinct LLVM + // types for Rust types that only differ in the choice of lifetimes. + let normal_ty = cx.tcx.erase_regions(self.ty); + + let mut defer = None; + let mut field_remapping = None; + let llty = if self.ty != normal_ty { + let mut layout = cx.layout_of(normal_ty); + if let Some(v) = variant_index { + layout = layout.for_variant(cx, v); + } + layout.llvm_type(cx) + } else { + uncached_llvm_type(cx, *self, &mut defer, &mut field_remapping) + }; + debug!("--> mapped {:#?} to llty={:?}", self, llty); + + cx.type_lowering + .borrow_mut() + .insert((self.ty, variant_index), TypeLowering { lltype: llty, field_remapping }); + + if let Some((llty, layout)) = defer { + let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); + cx.set_struct_body(llty, &llfields, packed); + cx.type_lowering + .borrow_mut() + .get_mut(&(self.ty, variant_index)) + .unwrap() + .field_remapping = new_field_remapping; + } + llty + } + + fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { + if let Abi::Scalar(scalar) = self.abi { + if scalar.is_bool() { + return cx.type_i1(); + } + } + self.llvm_type(cx) + } + + fn scalar_llvm_type_at<'a>( + &self, + cx: &CodegenCx<'a, 'tcx>, + scalar: Scalar, + offset: Size, + ) -> &'a Type { + match scalar.primitive() { + Int(i, _) => cx.type_from_integer(i), + F32 => cx.type_f32(), + F64 => cx.type_f64(), + Pointer => { + // If we know the alignment, pick something better than i8. + let (pointee, address_space) = + if let Some(pointee) = self.pointee_info_at(cx, offset) { + (cx.type_pointee_for_align(pointee.align), pointee.address_space) + } else { + (cx.type_i8(), AddressSpace::DATA) + }; + cx.type_ptr_to_ext(pointee, address_space) + } + } + } + + fn scalar_pair_element_llvm_type<'a>( + &self, + cx: &CodegenCx<'a, 'tcx>, + index: usize, + immediate: bool, + ) -> &'a Type { + // HACK(eddyb) special-case fat pointers until LLVM removes + // pointee types, to avoid bitcasting every `OperandRef::deref`. + match self.ty.kind() { + ty::Ref(..) | ty::RawPtr(_) => { + return self.field(cx, index).llvm_type(cx); + } + // only wide pointer boxes are handled as pointers + // thin pointer boxes with scalar allocators are handled by the general logic below + ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { + let ptr_ty = cx.tcx.mk_mut_ptr(self.ty.boxed_ty()); + return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); + } + _ => {} + } + + let Abi::ScalarPair(a, b) = self.abi else { + bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self); + }; + let scalar = [a, b][index]; + + // Make sure to return the same type `immediate_llvm_type` would when + // dealing with an immediate pair. This means that `(bool, bool)` is + // effectively represented as `{i8, i8}` in memory and two `i1`s as an + // immediate, just like `bool` is typically `i8` in memory and only `i1` + // when immediate. We need to load/store `bool` as `i8` to avoid + // crippling LLVM optimizations or triggering other LLVM bugs with `i1`. + if immediate && scalar.is_bool() { + return cx.type_i1(); + } + + let offset = if index == 0 { Size::ZERO } else { a.size(cx).align_to(b.align(cx).abi) }; + self.scalar_llvm_type_at(cx, scalar, offset) + } + + fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64 { + match self.abi { + Abi::Scalar(_) | Abi::ScalarPair(..) => { + bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self) + } + _ => {} + } + match self.fields { + FieldsShape::Primitive | FieldsShape::Union(_) => { + bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self) + } + + FieldsShape::Array { .. } => index as u64, + + FieldsShape::Arbitrary { .. } => { + let variant_index = match self.variants { + Variants::Single { index } => Some(index), + _ => None, + }; + + // Look up llvm field if indexes do not match memory order due to padding. If + // `field_remapping` is `None` no padding was used and the llvm field index + // matches the memory index. + match cx.type_lowering.borrow().get(&(self.ty, variant_index)) { + Some(TypeLowering { field_remapping: Some(ref remap), .. }) => { + remap[index] as u64 + } + Some(_) => self.fields.memory_index(index) as u64, + None => { + bug!("TyAndLayout::llvm_field_index({:?}): type info not found", self) + } + } + } + } + } + + // FIXME(eddyb) this having the same name as `TyAndLayout::pointee_info_at` + // (the inherent method, which is lacking this caching logic) can result in + // the uncached version being called - not wrong, but potentially inefficient. + fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option { + if let Some(&pointee) = cx.pointee_infos.borrow().get(&(self.ty, offset)) { + return pointee; + } + + let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset); + + cx.pointee_infos.borrow_mut().insert((self.ty, offset), result); + result + } +} diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs new file mode 100644 index 00000000000..ceb3d5a84ab --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -0,0 +1,214 @@ +use crate::builder::Builder; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; +use rustc_codegen_ssa::mir::operand::OperandRef; +use rustc_codegen_ssa::{ + common::IntPredicate, + traits::{BaseTypeMethods, BuilderMethods, ConstMethods, DerivedTypeMethods}, +}; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::Ty; +use rustc_target::abi::{Align, Endian, HasDataLayout, Size}; + +fn round_pointer_up_to_alignment<'ll>( + bx: &mut Builder<'_, 'll, '_>, + addr: &'ll Value, + align: Align, + ptr_ty: &'ll Type, +) -> &'ll Value { + let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize()); + ptr_as_int = bx.add(ptr_as_int, bx.cx().const_i32(align.bytes() as i32 - 1)); + ptr_as_int = bx.and(ptr_as_int, bx.cx().const_i32(-(align.bytes() as i32))); + bx.inttoptr(ptr_as_int, ptr_ty) +} + +fn emit_direct_ptr_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + llty: &'ll Type, + size: Size, + align: Align, + slot_size: Align, + allow_higher_align: bool, +) -> (&'ll Value, Align) { + let va_list_ty = bx.type_i8p(); + let va_list_ptr_ty = bx.type_ptr_to(va_list_ty); + let va_list_addr = if list.layout.llvm_type(bx.cx) != va_list_ptr_ty { + bx.bitcast(list.immediate(), va_list_ptr_ty) + } else { + list.immediate() + }; + + let ptr = bx.load(va_list_ty, va_list_addr, bx.tcx().data_layout.pointer_align.abi); + + let (addr, addr_align) = if allow_higher_align && align > slot_size { + (round_pointer_up_to_alignment(bx, ptr, align, bx.cx().type_i8p()), align) + } else { + (ptr, slot_size) + }; + + let aligned_size = size.align_to(slot_size).bytes() as i32; + let full_direct_size = bx.cx().const_i32(aligned_size); + let next = bx.inbounds_gep(bx.type_i8(), addr, &[full_direct_size]); + bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi); + + if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big { + let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32); + let adjusted = bx.inbounds_gep(bx.type_i8(), addr, &[adjusted_size]); + (bx.bitcast(adjusted, bx.cx().type_ptr_to(llty)), addr_align) + } else { + (bx.bitcast(addr, bx.cx().type_ptr_to(llty)), addr_align) + } +} + +fn emit_ptr_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, + indirect: bool, + slot_size: Align, + allow_higher_align: bool, +) -> &'ll Value { + let layout = bx.cx.layout_of(target_ty); + let (llty, size, align) = if indirect { + ( + bx.cx.layout_of(bx.cx.tcx.mk_imm_ptr(target_ty)).llvm_type(bx.cx), + bx.cx.data_layout().pointer_size, + bx.cx.data_layout().pointer_align, + ) + } else { + (layout.llvm_type(bx.cx), layout.size, layout.align) + }; + let (addr, addr_align) = + emit_direct_ptr_va_arg(bx, list, llty, size, align.abi, slot_size, allow_higher_align); + if indirect { + let tmp_ret = bx.load(llty, addr, addr_align); + bx.load(bx.cx.layout_of(target_ty).llvm_type(bx.cx), tmp_ret, align.abi) + } else { + bx.load(llty, addr, addr_align) + } +} + +fn emit_aapcs_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Implementation of the AAPCS64 calling convention for va_args see + // https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst + let va_list_addr = list.immediate(); + let va_list_layout = list.deref(bx.cx).layout; + let va_list_ty = va_list_layout.llvm_type(bx); + let layout = bx.cx.layout_of(target_ty); + + let maybe_reg = bx.append_sibling_block("va_arg.maybe_reg"); + let in_reg = bx.append_sibling_block("va_arg.in_reg"); + let on_stack = bx.append_sibling_block("va_arg.on_stack"); + let end = bx.append_sibling_block("va_arg.end"); + let zero = bx.const_i32(0); + let offset_align = Align::from_bytes(4).unwrap(); + + let gr_type = target_ty.is_any_ptr() || target_ty.is_integral(); + let (reg_off, reg_top_index, slot_size) = if gr_type { + let gr_offs = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); + let nreg = (layout.size.bytes() + 7) / 8; + (gr_offs, va_list_layout.llvm_field_index(bx.cx, 1), nreg * 8) + } else { + let vr_off = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 4)); + let nreg = (layout.size.bytes() + 15) / 16; + (vr_off, va_list_layout.llvm_field_index(bx.cx, 2), nreg * 16) + }; + + // if the offset >= 0 then the value will be on the stack + let mut reg_off_v = bx.load(bx.type_i32(), reg_off, offset_align); + let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero); + bx.cond_br(use_stack, on_stack, maybe_reg); + + // The value at this point might be in a register, but there is a chance that + // it could be on the stack so we have to update the offset and then check + // the offset again. + + bx.switch_to_block(maybe_reg); + if gr_type && layout.align.abi.bytes() > 8 { + reg_off_v = bx.add(reg_off_v, bx.const_i32(15)); + reg_off_v = bx.and(reg_off_v, bx.const_i32(-16)); + } + let new_reg_off_v = bx.add(reg_off_v, bx.const_i32(slot_size as i32)); + + bx.store(new_reg_off_v, reg_off, offset_align); + + // Check to see if we have overflowed the registers as a result of this. + // If we have then we need to use the stack for this value + let use_stack = bx.icmp(IntPredicate::IntSGT, new_reg_off_v, zero); + bx.cond_br(use_stack, on_stack, in_reg); + + bx.switch_to_block(in_reg); + let top_type = bx.type_i8p(); + let top = bx.struct_gep(va_list_ty, va_list_addr, reg_top_index); + let top = bx.load(top_type, top, bx.tcx().data_layout.pointer_align.abi); + + // reg_value = *(@top + reg_off_v); + let mut reg_addr = bx.gep(bx.type_i8(), top, &[reg_off_v]); + if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size { + // On big-endian systems the value is right-aligned in its slot. + let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32); + reg_addr = bx.gep(bx.type_i8(), reg_addr, &[offset]); + } + let reg_type = layout.llvm_type(bx); + let reg_addr = bx.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type)); + let reg_value = bx.load(reg_type, reg_addr, layout.align.abi); + bx.br(end); + + // On Stack block + bx.switch_to_block(on_stack); + let stack_value = + emit_ptr_va_arg(bx, list, target_ty, false, Align::from_bytes(8).unwrap(), true); + bx.br(end); + + bx.switch_to_block(end); + let val = + bx.phi(layout.immediate_llvm_type(bx), &[reg_value, stack_value], &[in_reg, on_stack]); + + val +} + +pub(super) fn emit_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + addr: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Determine the va_arg implementation to use. The LLVM va_arg instruction + // is lacking in some instances, so we should only use it as a fallback. + let target = &bx.cx.tcx.sess.target; + let arch = &bx.cx.tcx.sess.target.arch; + match &**arch { + // Windows x86 + "x86" if target.is_like_windows => { + emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), false) + } + // Generic x86 + "x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true), + // Windows AArch64 + "aarch64" if target.is_like_windows => { + emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false) + } + // macOS / iOS AArch64 + "aarch64" if target.is_like_osx => { + emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true) + } + "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), + // Windows x86_64 + "x86_64" if target.is_like_windows => { + let target_ty_size = bx.cx.size_of(target_ty).bytes(); + let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two(); + emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false) + } + // For all other architecture/OS combinations fall back to using + // the LLVM va_arg instruction. + // https://llvm.org/docs/LangRef.html#va-arg-instruction + _ => bx.va_arg(addr.immediate(), bx.cx.layout_of(target_ty).llvm_type(bx.cx)), + } +} diff --git a/compiler/rustc_codegen_llvm/src/value.rs b/compiler/rustc_codegen_llvm/src/value.rs new file mode 100644 index 00000000000..1338a229566 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/value.rs @@ -0,0 +1,32 @@ +pub use crate::llvm::Value; + +use crate::llvm; + +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::ptr; + +impl PartialEq for Value { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self, other) + } +} + +impl Eq for Value {} + +impl Hash for Value { + fn hash(&self, hasher: &mut H) { + (self as *const Self).hash(hasher); + } +} + +impl fmt::Debug for Value { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str( + &llvm::build_string(|s| unsafe { + llvm::LLVMRustWriteValueToString(self, s); + }) + .expect("non-UTF8 value description from LLVM"), + ) + } +} From 09ac73987104a3ba1091bbb364bb2061a456860d Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 29 Nov 2023 17:11:08 +0000 Subject: [PATCH 07/70] add rustc_codegen_ssa --- compiler/rustc_codegen_ssa/Cargo.toml | 48 + compiler/rustc_codegen_ssa/README.md | 3 + .../rustc_codegen_ssa/src/back/archive.rs | 69 + .../rustc_codegen_ssa/src/back/command.rs | 178 ++ compiler/rustc_codegen_ssa/src/back/link.rs | 2800 +++++++++++++++++ compiler/rustc_codegen_ssa/src/back/linker.rs | 1788 +++++++++++ compiler/rustc_codegen_ssa/src/back/lto.rs | 104 + .../rustc_codegen_ssa/src/back/metadata.rs | 314 ++ compiler/rustc_codegen_ssa/src/back/mod.rs | 9 + compiler/rustc_codegen_ssa/src/back/rpath.rs | 114 + .../rustc_codegen_ssa/src/back/rpath/tests.rs | 72 + .../src/back/symbol_export.rs | 590 ++++ compiler/rustc_codegen_ssa/src/back/write.rs | 2015 ++++++++++++ compiler/rustc_codegen_ssa/src/base.rs | 961 ++++++ compiler/rustc_codegen_ssa/src/common.rs | 223 ++ .../rustc_codegen_ssa/src/coverageinfo/ffi.rs | 85 + .../rustc_codegen_ssa/src/coverageinfo/map.rs | 347 ++ .../rustc_codegen_ssa/src/coverageinfo/mod.rs | 2 + .../rustc_codegen_ssa/src/debuginfo/mod.rs | 34 + .../src/debuginfo/type_names.rs | 821 +++++ compiler/rustc_codegen_ssa/src/glue.rs | 123 + compiler/rustc_codegen_ssa/src/lib.rs | 244 ++ compiler/rustc_codegen_ssa/src/meth.rs | 116 + compiler/rustc_codegen_ssa/src/mir/analyze.rs | 368 +++ compiler/rustc_codegen_ssa/src/mir/block.rs | 1654 ++++++++++ .../rustc_codegen_ssa/src/mir/constant.rs | 90 + .../rustc_codegen_ssa/src/mir/coverageinfo.rs | 55 + .../rustc_codegen_ssa/src/mir/debuginfo.rs | 418 +++ .../rustc_codegen_ssa/src/mir/intrinsic.rs | 636 ++++ compiler/rustc_codegen_ssa/src/mir/mod.rs | 410 +++ compiler/rustc_codegen_ssa/src/mir/operand.rs | 461 +++ compiler/rustc_codegen_ssa/src/mir/place.rs | 549 ++++ compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 729 +++++ .../rustc_codegen_ssa/src/mir/statement.rs | 102 + compiler/rustc_codegen_ssa/src/mono_item.rs | 147 + .../rustc_codegen_ssa/src/target_features.rs | 308 ++ compiler/rustc_codegen_ssa/src/traits/abi.rs | 8 + compiler/rustc_codegen_ssa/src/traits/asm.rs | 66 + .../rustc_codegen_ssa/src/traits/backend.rs | 161 + .../rustc_codegen_ssa/src/traits/builder.rs | 481 +++ .../rustc_codegen_ssa/src/traits/consts.rs | 41 + .../src/traits/coverageinfo.rs | 57 + .../rustc_codegen_ssa/src/traits/debuginfo.rs | 79 + .../rustc_codegen_ssa/src/traits/declare.rs | 21 + .../rustc_codegen_ssa/src/traits/intrinsic.rs | 39 + compiler/rustc_codegen_ssa/src/traits/misc.rs | 26 + compiler/rustc_codegen_ssa/src/traits/mod.rs | 102 + .../rustc_codegen_ssa/src/traits/statics.rs | 24 + .../rustc_codegen_ssa/src/traits/type_.rs | 151 + .../rustc_codegen_ssa/src/traits/write.rs | 68 + 50 files changed, 18311 insertions(+) create mode 100644 compiler/rustc_codegen_ssa/Cargo.toml create mode 100644 compiler/rustc_codegen_ssa/README.md create mode 100644 compiler/rustc_codegen_ssa/src/back/archive.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/command.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/link.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/linker.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/lto.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/metadata.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/mod.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/rpath.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/rpath/tests.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/symbol_export.rs create mode 100644 compiler/rustc_codegen_ssa/src/back/write.rs create mode 100644 compiler/rustc_codegen_ssa/src/base.rs create mode 100644 compiler/rustc_codegen_ssa/src/common.rs create mode 100644 compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs create mode 100644 compiler/rustc_codegen_ssa/src/coverageinfo/map.rs create mode 100644 compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs create mode 100644 compiler/rustc_codegen_ssa/src/debuginfo/mod.rs create mode 100644 compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs create mode 100644 compiler/rustc_codegen_ssa/src/glue.rs create mode 100644 compiler/rustc_codegen_ssa/src/lib.rs create mode 100644 compiler/rustc_codegen_ssa/src/meth.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/analyze.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/block.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/constant.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/debuginfo.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/intrinsic.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/mod.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/operand.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/place.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/rvalue.rs create mode 100644 compiler/rustc_codegen_ssa/src/mir/statement.rs create mode 100644 compiler/rustc_codegen_ssa/src/mono_item.rs create mode 100644 compiler/rustc_codegen_ssa/src/target_features.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/abi.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/asm.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/backend.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/builder.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/consts.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/debuginfo.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/declare.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/intrinsic.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/misc.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/mod.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/statics.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/type_.rs create mode 100644 compiler/rustc_codegen_ssa/src/traits/write.rs diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml new file mode 100644 index 00000000000..46d6344dbb2 --- /dev/null +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "rustc_codegen_ssa" +version = "0.0.0" +edition = "2021" + +[lib] +test = false + +[dependencies] +bitflags = "1.2.1" +cc = "1.0.69" +itertools = "0.10.1" +tracing = "0.1" +libc = "0.2.50" +jobserver = "0.1.22" +tempfile = "3.2" +thorin-dwp = "0.3" +pathdiff = "0.2.0" +serde_json = "1.0.59" +snap = "1" +smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } +regex = "1.4" + +rustc_serialize = { path = "../rustc_serialize" } +rustc_arena = { path = "../rustc_arena" } +rustc_ast = { path = "../rustc_ast" } +rustc_span = { path = "../rustc_span" } +rustc_middle = { path = "../rustc_middle" } +rustc_apfloat = { path = "../rustc_apfloat" } +rustc_attr = { path = "../rustc_attr" } +rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } +rustc_data_structures = { path = "../rustc_data_structures" } +rustc_errors = { path = "../rustc_errors" } +rustc_fs_util = { path = "../rustc_fs_util" } +rustc_hir = { path = "../rustc_hir" } +rustc_incremental = { path = "../rustc_incremental" } +rustc_index = { path = "../rustc_index" } +rustc_macros = { path = "../rustc_macros" } +rustc_metadata = { path = "../rustc_metadata" } +rustc_query_system = { path = "../rustc_query_system" } +rustc_target = { path = "../rustc_target" } +rustc_session = { path = "../rustc_session" } +rustc_const_eval = { path = "../rustc_const_eval" } + +[dependencies.object] +version = "0.29.0" +default-features = false +features = ["read_core", "elf", "macho", "pe", "unaligned", "archive", "write"] diff --git a/compiler/rustc_codegen_ssa/README.md b/compiler/rustc_codegen_ssa/README.md new file mode 100644 index 00000000000..7b770187b75 --- /dev/null +++ b/compiler/rustc_codegen_ssa/README.md @@ -0,0 +1,3 @@ +Please read the rustc-dev-guide chapter on [Backend Agnostic Codegen][bac]. + +[bac]: https://rustc-dev-guide.rust-lang.org/backend/backend-agnostic.html diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs new file mode 100644 index 00000000000..0d2aa483d3d --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/archive.rs @@ -0,0 +1,69 @@ +use rustc_session::cstore::DllImport; +use rustc_session::Session; + +use std::io; +use std::path::{Path, PathBuf}; + +pub(super) fn find_library( + name: &str, + verbatim: bool, + search_paths: &[PathBuf], + sess: &Session, +) -> PathBuf { + // On Windows, static libraries sometimes show up as libfoo.a and other + // times show up as foo.lib + let oslibname = if verbatim { + name.to_string() + } else { + format!("{}{}{}", sess.target.staticlib_prefix, name, sess.target.staticlib_suffix) + }; + let unixlibname = format!("lib{}.a", name); + + for path in search_paths { + debug!("looking for {} inside {:?}", name, path); + let test = path.join(&oslibname); + if test.exists() { + return test; + } + if oslibname != unixlibname { + let test = path.join(&unixlibname); + if test.exists() { + return test; + } + } + } + sess.fatal(&format!( + "could not find native static library `{}`, \ + perhaps an -L flag is missing?", + name + )); +} + +pub trait ArchiveBuilderBuilder { + fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box + 'a>; + + /// Creates a DLL Import Library . + /// and returns the path on disk to that import library. + /// This functions doesn't take `self` so that it can be called from + /// `linker_with_args`, which is specialized on `ArchiveBuilder` but + /// doesn't take or create an instance of that type. + fn create_dll_import_lib( + &self, + sess: &Session, + lib_name: &str, + dll_imports: &[DllImport], + tmpdir: &Path, + ) -> PathBuf; +} + +pub trait ArchiveBuilder<'a> { + fn add_file(&mut self, path: &Path); + + fn add_archive( + &mut self, + archive: &Path, + skip: Box bool + 'static>, + ) -> io::Result<()>; + + fn build(self: Box, output: &Path) -> bool; +} diff --git a/compiler/rustc_codegen_ssa/src/back/command.rs b/compiler/rustc_codegen_ssa/src/back/command.rs new file mode 100644 index 00000000000..9b0ba34135c --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/command.rs @@ -0,0 +1,178 @@ +//! A thin wrapper around `Command` in the standard library which allows us to +//! read the arguments that are built up. + +use std::ffi::{OsStr, OsString}; +use std::fmt; +use std::io; +use std::mem; +use std::process::{self, Output}; + +use rustc_target::spec::LldFlavor; + +#[derive(Clone)] +pub struct Command { + program: Program, + args: Vec, + env: Vec<(OsString, OsString)>, + env_remove: Vec, +} + +#[derive(Clone)] +enum Program { + Normal(OsString), + CmdBatScript(OsString), + Lld(OsString, LldFlavor), +} + +impl Command { + pub fn new>(program: P) -> Command { + Command::_new(Program::Normal(program.as_ref().to_owned())) + } + + pub fn bat_script>(program: P) -> Command { + Command::_new(Program::CmdBatScript(program.as_ref().to_owned())) + } + + pub fn lld>(program: P, flavor: LldFlavor) -> Command { + Command::_new(Program::Lld(program.as_ref().to_owned(), flavor)) + } + + fn _new(program: Program) -> Command { + Command { program, args: Vec::new(), env: Vec::new(), env_remove: Vec::new() } + } + + pub fn arg>(&mut self, arg: P) -> &mut Command { + self._arg(arg.as_ref()); + self + } + + pub fn args(&mut self, args: I) -> &mut Command + where + I: IntoIterator>, + { + for arg in args { + self._arg(arg.as_ref()); + } + self + } + + fn _arg(&mut self, arg: &OsStr) { + self.args.push(arg.to_owned()); + } + + pub fn env(&mut self, key: K, value: V) -> &mut Command + where + K: AsRef, + V: AsRef, + { + self._env(key.as_ref(), value.as_ref()); + self + } + + fn _env(&mut self, key: &OsStr, value: &OsStr) { + self.env.push((key.to_owned(), value.to_owned())); + } + + pub fn env_remove(&mut self, key: K) -> &mut Command + where + K: AsRef, + { + self._env_remove(key.as_ref()); + self + } + + fn _env_remove(&mut self, key: &OsStr) { + self.env_remove.push(key.to_owned()); + } + + pub fn output(&mut self) -> io::Result { + self.command().output() + } + + pub fn command(&self) -> process::Command { + let mut ret = match self.program { + Program::Normal(ref p) => process::Command::new(p), + Program::CmdBatScript(ref p) => { + let mut c = process::Command::new("cmd"); + c.arg("/c").arg(p); + c + } + Program::Lld(ref p, flavor) => { + let mut c = process::Command::new(p); + c.arg("-flavor").arg(flavor.as_str()); + if let LldFlavor::Wasm = flavor { + // LLVM expects host-specific formatting for @file + // arguments, but we always generate posix formatted files + // at this time. Indicate as such. + c.arg("--rsp-quoting=posix"); + } + c + } + }; + ret.args(&self.args); + ret.envs(self.env.clone()); + for k in &self.env_remove { + ret.env_remove(k); + } + ret + } + + // extensions + + pub fn get_args(&self) -> &[OsString] { + &self.args + } + + pub fn take_args(&mut self) -> Vec { + mem::take(&mut self.args) + } + + /// Returns a `true` if we're pretty sure that this'll blow OS spawn limits, + /// or `false` if we should attempt to spawn and see what the OS says. + pub fn very_likely_to_exceed_some_spawn_limit(&self) -> bool { + // We mostly only care about Windows in this method, on Unix the limits + // can be gargantuan anyway so we're pretty unlikely to hit them + if cfg!(unix) { + return false; + } + + // Right now LLD doesn't support the `@` syntax of passing an argument + // through files, so regardless of the platform we try to go to the OS + // on this one. + if let Program::Lld(..) = self.program { + return false; + } + + // Ok so on Windows to spawn a process is 32,768 characters in its + // command line [1]. Unfortunately we don't actually have access to that + // as it's calculated just before spawning. Instead we perform a + // poor-man's guess as to how long our command line will be. We're + // assuming here that we don't have to escape every character... + // + // Turns out though that `cmd.exe` has even smaller limits, 8192 + // characters [2]. Linkers can often be batch scripts (for example + // Emscripten, Gecko's current build system) which means that we're + // running through batch scripts. These linkers often just forward + // arguments elsewhere (and maybe tack on more), so if we blow 8192 + // bytes we'll typically cause them to blow as well. + // + // Basically as a result just perform an inflated estimate of what our + // command line will look like and test if it's > 8192 (we actually + // test against 6k to artificially inflate our estimate). If all else + // fails we'll fall back to the normal unix logic of testing the OS + // error code if we fail to spawn and automatically re-spawning the + // linker with smaller arguments. + // + // [1]: https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessa + // [2]: https://devblogs.microsoft.com/oldnewthing/?p=41553 + + let estimated_command_line_len = self.args.iter().map(|a| a.len()).sum::(); + estimated_command_line_len > 1024 * 6 + } +} + +impl fmt::Debug for Command { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.command().fmt(f) + } +} diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs new file mode 100644 index 00000000000..63207803e32 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -0,0 +1,2800 @@ +use rustc_arena::TypedArena; +use rustc_ast::CRATE_NODE_ID; +use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; +use rustc_data_structures::memmap::Mmap; +use rustc_data_structures::temp_dir::MaybeTempDir; +use rustc_errors::{ErrorGuaranteed, Handler}; +use rustc_fs_util::fix_windows_verbatim_for_gcc; +use rustc_hir::def_id::CrateNum; +use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME}; +use rustc_middle::middle::dependency_format::Linkage; +use rustc_middle::middle::exported_symbols::SymbolExportKind; +use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip}; +use rustc_session::config::{OutputFilenames, OutputType, PrintRequest, SplitDwarfKind}; +use rustc_session::cstore::DllImport; +use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename}; +use rustc_session::search_paths::PathKind; +use rustc_session::utils::NativeLibKind; +/// For all the linkers we support, and information they might +/// need out of the shared crate context before we get rid of it. +use rustc_session::{filesearch, Session}; +use rustc_span::symbol::Symbol; +use rustc_span::DebuggerVisualizerFile; +use rustc_target::spec::crt_objects::{CrtObjects, CrtObjectsFallback}; +use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor, SplitDebuginfo}; +use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, SanitizerSet, Target}; + +use super::archive::{find_library, ArchiveBuilder, ArchiveBuilderBuilder}; +use super::command::Command; +use super::linker::{self, Linker}; +use super::metadata::{create_rmeta_file, MetadataPosition}; +use super::rpath::{self, RPathConfig}; +use crate::{looks_like_rust_object_file, CodegenResults, CompiledModule, CrateInfo, NativeLib}; + +use cc::windows_registry; +use regex::Regex; +use tempfile::Builder as TempFileBuilder; + +use std::borrow::Borrow; +use std::cell::OnceCell; +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::fs::{File, OpenOptions}; +use std::io::{BufWriter, Write}; +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::process::{ExitStatus, Output, Stdio}; +use std::{ascii, char, env, fmt, fs, io, mem, str}; + +pub fn ensure_removed(diag_handler: &Handler, path: &Path) { + if let Err(e) = fs::remove_file(path) { + if e.kind() != io::ErrorKind::NotFound { + diag_handler.err(&format!("failed to remove {}: {}", path.display(), e)); + } + } +} + +/// Performs the linkage portion of the compilation phase. This will generate all +/// of the requested outputs for this compilation session. +pub fn link_binary<'a>( + sess: &'a Session, + archive_builder_builder: &dyn ArchiveBuilderBuilder, + codegen_results: &CodegenResults, + outputs: &OutputFilenames, +) -> Result<(), ErrorGuaranteed> { + let _timer = sess.timer("link_binary"); + let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata); + for &crate_type in sess.crate_types().iter() { + // Ignore executable crates if we have -Z no-codegen, as they will error. + if (sess.opts.unstable_opts.no_codegen || !sess.opts.output_types.should_codegen()) + && !output_metadata + && crate_type == CrateType::Executable + { + continue; + } + + if invalid_output_for_target(sess, crate_type) { + bug!( + "invalid output type `{:?}` for target os `{}`", + crate_type, + sess.opts.target_triple + ); + } + + sess.time("link_binary_check_files_are_writeable", || { + for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) { + check_file_is_writeable(obj, sess); + } + }); + + if outputs.outputs.should_link() { + let tmpdir = TempFileBuilder::new() + .prefix("rustc") + .tempdir() + .unwrap_or_else(|err| sess.fatal(&format!("couldn't create a temp dir: {}", err))); + let path = MaybeTempDir::new(tmpdir, sess.opts.cg.save_temps); + let out_filename = out_filename( + sess, + crate_type, + outputs, + codegen_results.crate_info.local_crate_name.as_str(), + ); + match crate_type { + CrateType::Rlib => { + let _timer = sess.timer("link_rlib"); + info!("preparing rlib to {:?}", out_filename); + link_rlib( + sess, + archive_builder_builder, + codegen_results, + RlibFlavor::Normal, + &path, + )? + .build(&out_filename); + } + CrateType::Staticlib => { + link_staticlib( + sess, + archive_builder_builder, + codegen_results, + &out_filename, + &path, + )?; + } + _ => { + link_natively( + sess, + archive_builder_builder, + crate_type, + &out_filename, + codegen_results, + path.as_ref(), + )?; + } + } + if sess.opts.json_artifact_notifications { + sess.parse_sess.span_diagnostic.emit_artifact_notification(&out_filename, "link"); + } + + if sess.prof.enabled() { + if let Some(artifact_name) = out_filename.file_name() { + // Record size for self-profiling + let file_size = std::fs::metadata(&out_filename).map(|m| m.len()).unwrap_or(0); + + sess.prof.artifact_size( + "linked_artifact", + artifact_name.to_string_lossy(), + file_size, + ); + } + } + } + } + + // Remove the temporary object file and metadata if we aren't saving temps. + sess.time("link_binary_remove_temps", || { + // If the user requests that temporaries are saved, don't delete any. + if sess.opts.cg.save_temps { + return; + } + + let maybe_remove_temps_from_module = + |preserve_objects: bool, preserve_dwarf_objects: bool, module: &CompiledModule| { + if !preserve_objects { + if let Some(ref obj) = module.object { + ensure_removed(sess.diagnostic(), obj); + } + } + + if !preserve_dwarf_objects { + if let Some(ref dwo_obj) = module.dwarf_object { + ensure_removed(sess.diagnostic(), dwo_obj); + } + } + }; + + let remove_temps_from_module = + |module: &CompiledModule| maybe_remove_temps_from_module(false, false, module); + + // Otherwise, always remove the metadata and allocator module temporaries. + if let Some(ref metadata_module) = codegen_results.metadata_module { + remove_temps_from_module(metadata_module); + } + + if let Some(ref allocator_module) = codegen_results.allocator_module { + remove_temps_from_module(allocator_module); + } + + // If no requested outputs require linking, then the object temporaries should + // be kept. + if !sess.opts.output_types.should_link() { + return; + } + + // Potentially keep objects for their debuginfo. + let (preserve_objects, preserve_dwarf_objects) = preserve_objects_for_their_debuginfo(sess); + debug!(?preserve_objects, ?preserve_dwarf_objects); + + for module in &codegen_results.modules { + maybe_remove_temps_from_module(preserve_objects, preserve_dwarf_objects, module); + } + }); + + Ok(()) +} + +pub fn each_linked_rlib( + info: &CrateInfo, + f: &mut dyn FnMut(CrateNum, &Path), +) -> Result<(), String> { + let crates = info.used_crates.iter(); + let mut fmts = None; + for (ty, list) in info.dependency_formats.iter() { + match ty { + CrateType::Executable + | CrateType::Staticlib + | CrateType::Cdylib + | CrateType::ProcMacro => { + fmts = Some(list); + break; + } + _ => {} + } + } + let Some(fmts) = fmts else { + return Err("could not find formats for rlibs".to_string()); + }; + for &cnum in crates { + match fmts.get(cnum.as_usize() - 1) { + Some(&Linkage::NotLinked | &Linkage::IncludedFromDylib) => continue, + Some(_) => {} + None => return Err("could not find formats for rlibs".to_string()), + } + let name = info.crate_name[&cnum]; + let used_crate_source = &info.used_crate_source[&cnum]; + if let Some((path, _)) = &used_crate_source.rlib { + f(cnum, &path); + } else { + if used_crate_source.rmeta.is_some() { + return Err(format!( + "could not find rlib for: `{}`, found rmeta (metadata) file", + name + )); + } else { + return Err(format!("could not find rlib for: `{}`", name)); + } + } + } + Ok(()) +} + +/// Create an 'rlib'. +/// +/// An rlib in its current incarnation is essentially a renamed .a file. The rlib primarily contains +/// the object file of the crate, but it also contains all of the object files from native +/// libraries. This is done by unzipping native libraries and inserting all of the contents into +/// this archive. +fn link_rlib<'a>( + sess: &'a Session, + archive_builder_builder: &dyn ArchiveBuilderBuilder, + codegen_results: &CodegenResults, + flavor: RlibFlavor, + tmpdir: &MaybeTempDir, +) -> Result + 'a>, ErrorGuaranteed> { + let lib_search_paths = archive_search_paths(sess); + + let mut ab = archive_builder_builder.new_archive_builder(sess); + + let trailing_metadata = match flavor { + RlibFlavor::Normal => { + let (metadata, metadata_position) = + create_rmeta_file(sess, codegen_results.metadata.raw_data()); + let metadata = emit_metadata(sess, &metadata, tmpdir); + match metadata_position { + MetadataPosition::First => { + // Most of the time metadata in rlib files is wrapped in a "dummy" object + // file for the target platform so the rlib can be processed entirely by + // normal linkers for the platform. Sometimes this is not possible however. + // If it is possible however, placing the metadata object first improves + // performance of getting metadata from rlibs. + ab.add_file(&metadata); + None + } + MetadataPosition::Last => Some(metadata), + } + } + + RlibFlavor::StaticlibBase => None, + }; + + for m in &codegen_results.modules { + if let Some(obj) = m.object.as_ref() { + ab.add_file(obj); + } + + if let Some(dwarf_obj) = m.dwarf_object.as_ref() { + ab.add_file(dwarf_obj); + } + } + + match flavor { + RlibFlavor::Normal => {} + RlibFlavor::StaticlibBase => { + let obj = codegen_results.allocator_module.as_ref().and_then(|m| m.object.as_ref()); + if let Some(obj) = obj { + ab.add_file(obj); + } + } + } + + // Note that in this loop we are ignoring the value of `lib.cfg`. That is, + // we may not be configured to actually include a static library if we're + // adding it here. That's because later when we consume this rlib we'll + // decide whether we actually needed the static library or not. + // + // To do this "correctly" we'd need to keep track of which libraries added + // which object files to the archive. We don't do that here, however. The + // #[link(cfg(..))] feature is unstable, though, and only intended to get + // liblibc working. In that sense the check below just indicates that if + // there are any libraries we want to omit object files for at link time we + // just exclude all custom object files. + // + // Eventually if we want to stabilize or flesh out the #[link(cfg(..))] + // feature then we'll need to figure out how to record what objects were + // loaded from the libraries found here and then encode that into the + // metadata of the rlib we're generating somehow. + for lib in codegen_results.crate_info.used_libraries.iter() { + match lib.kind { + NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) } + if flavor == RlibFlavor::Normal => + { + // Don't allow mixing +bundle with +whole_archive since an rlib may contain + // multiple native libs, some of which are +whole-archive and some of which are + // -whole-archive and it isn't clear how we can currently handle such a + // situation correctly. + // See https://github.com/rust-lang/rust/issues/88085#issuecomment-901050897 + sess.err( + "the linking modifiers `+bundle` and `+whole-archive` are not compatible \ + with each other when generating rlibs", + ); + } + NativeLibKind::Static { bundle: None | Some(true), .. } => {} + NativeLibKind::Static { bundle: Some(false), .. } + | NativeLibKind::Dylib { .. } + | NativeLibKind::Framework { .. } + | NativeLibKind::RawDylib + | NativeLibKind::LinkArg + | NativeLibKind::Unspecified => continue, + } + if let Some(name) = lib.name { + let location = + find_library(name.as_str(), lib.verbatim.unwrap_or(false), &lib_search_paths, sess); + ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|e| { + sess.fatal(&format!( + "failed to add native library {}: {}", + location.to_string_lossy(), + e + )); + }); + } + } + + for (raw_dylib_name, raw_dylib_imports) in + collate_raw_dylibs(sess, &codegen_results.crate_info.used_libraries)? + { + let output_path = archive_builder_builder.create_dll_import_lib( + sess, + &raw_dylib_name, + &raw_dylib_imports, + tmpdir.as_ref(), + ); + + ab.add_archive(&output_path, Box::new(|_| false)).unwrap_or_else(|e| { + sess.fatal(&format!("failed to add native library {}: {}", output_path.display(), e)); + }); + } + + if let Some(trailing_metadata) = trailing_metadata { + // Note that it is important that we add all of our non-object "magical + // files" *after* all of the object files in the archive. The reason for + // this is as follows: + // + // * When performing LTO, this archive will be modified to remove + // objects from above. The reason for this is described below. + // + // * When the system linker looks at an archive, it will attempt to + // determine the architecture of the archive in order to see whether its + // linkable. + // + // The algorithm for this detection is: iterate over the files in the + // archive. Skip magical SYMDEF names. Interpret the first file as an + // object file. Read architecture from the object file. + // + // * As one can probably see, if "metadata" and "foo.bc" were placed + // before all of the objects, then the architecture of this archive would + // not be correctly inferred once 'foo.o' is removed. + // + // * Most of the time metadata in rlib files is wrapped in a "dummy" object + // file for the target platform so the rlib can be processed entirely by + // normal linkers for the platform. Sometimes this is not possible however. + // + // Basically, all this means is that this code should not move above the + // code above. + ab.add_file(&trailing_metadata); + } + + return Ok(ab); +} + +/// Extract all symbols defined in raw-dylib libraries, collated by library name. +/// +/// If we have multiple extern blocks that specify symbols defined in the same raw-dylib library, +/// then the CodegenResults value contains one NativeLib instance for each block. However, the +/// linker appears to expect only a single import library for each library used, so we need to +/// collate the symbols together by library name before generating the import libraries. +fn collate_raw_dylibs( + sess: &Session, + used_libraries: &[NativeLib], +) -> Result)>, ErrorGuaranteed> { + // Use index maps to preserve original order of imports and libraries. + let mut dylib_table = FxIndexMap::>::default(); + + for lib in used_libraries { + if lib.kind == NativeLibKind::RawDylib { + let ext = if matches!(lib.verbatim, Some(true)) { "" } else { ".dll" }; + let name = format!("{}{}", lib.name.expect("unnamed raw-dylib library"), ext); + let imports = dylib_table.entry(name.clone()).or_default(); + for import in &lib.dll_imports { + if let Some(old_import) = imports.insert(import.name, import) { + // FIXME: when we add support for ordinals, figure out if we need to do anything + // if we have two DllImport values with the same name but different ordinals. + if import.calling_convention != old_import.calling_convention { + sess.span_err( + import.span, + &format!( + "multiple declarations of external function `{}` from \ + library `{}` have different calling conventions", + import.name, name, + ), + ); + } + } + } + } + } + sess.compile_status()?; + Ok(dylib_table + .into_iter() + .map(|(name, imports)| { + (name, imports.into_iter().map(|(_, import)| import.clone()).collect()) + }) + .collect()) +} + +/// Create a static archive. +/// +/// This is essentially the same thing as an rlib, but it also involves adding all of the upstream +/// crates' objects into the archive. This will slurp in all of the native libraries of upstream +/// dependencies as well. +/// +/// Additionally, there's no way for us to link dynamic libraries, so we warn about all dynamic +/// library dependencies that they're not linked in. +/// +/// There's no need to include metadata in a static archive, so ensure to not link in the metadata +/// object file (and also don't prepare the archive with a metadata file). +fn link_staticlib<'a>( + sess: &'a Session, + archive_builder_builder: &dyn ArchiveBuilderBuilder, + codegen_results: &CodegenResults, + out_filename: &Path, + tempdir: &MaybeTempDir, +) -> Result<(), ErrorGuaranteed> { + info!("preparing staticlib to {:?}", out_filename); + let mut ab = link_rlib( + sess, + archive_builder_builder, + codegen_results, + RlibFlavor::StaticlibBase, + tempdir, + )?; + let mut all_native_libs = vec![]; + + let res = each_linked_rlib(&codegen_results.crate_info, &mut |cnum, path| { + let name = codegen_results.crate_info.crate_name[&cnum]; + let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; + + // Here when we include the rlib into our staticlib we need to make a + // decision whether to include the extra object files along the way. + // These extra object files come from statically included native + // libraries, but they may be cfg'd away with #[link(cfg(..))]. + // + // This unstable feature, though, only needs liblibc to work. The only + // use case there is where musl is statically included in liblibc.rlib, + // so if we don't want the included version we just need to skip it. As + // a result the logic here is that if *any* linked library is cfg'd away + // we just skip all object files. + // + // Clearly this is not sufficient for a general purpose feature, and + // we'd want to read from the library's metadata to determine which + // object files come from where and selectively skip them. + let skip_object_files = native_libs.iter().any(|lib| { + matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. }) + && !relevant_lib(sess, lib) + }); + + let lto = are_upstream_rust_objects_already_included(sess) + && !ignored_for_lto(sess, &codegen_results.crate_info, cnum); + + // Ignoring obj file starting with the crate name + // as simple comparison is not enough - there + // might be also an extra name suffix + let obj_start = name.as_str().to_owned(); + + ab.add_archive( + path, + Box::new(move |fname: &str| { + // Ignore metadata files, no matter the name. + if fname == METADATA_FILENAME { + return true; + } + + // Don't include Rust objects if LTO is enabled + if lto && looks_like_rust_object_file(fname) { + return true; + } + + // Otherwise if this is *not* a rust object and we're skipping + // objects then skip this file + if skip_object_files && (!fname.starts_with(&obj_start) || !fname.ends_with(".o")) { + return true; + } + + // ok, don't skip this + false + }), + ) + .unwrap(); + + all_native_libs.extend(codegen_results.crate_info.native_libraries[&cnum].iter().cloned()); + }); + if let Err(e) = res { + sess.fatal(&e); + } + + ab.build(out_filename); + + if !all_native_libs.is_empty() { + if sess.opts.prints.contains(&PrintRequest::NativeStaticLibs) { + print_native_static_libs(sess, &all_native_libs); + } + } + + Ok(()) +} + +fn escape_stdout_stderr_string(s: &[u8]) -> String { + str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| { + let mut x = "Non-UTF-8 output: ".to_string(); + x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from)); + x + }) +} + +/// Use `thorin` (rust implementation of a dwarf packaging utility) to link DWARF objects into a +/// DWARF package. +fn link_dwarf_object<'a>( + sess: &'a Session, + cg_results: &CodegenResults, + executable_out_filename: &Path, +) { + let dwp_out_filename = executable_out_filename.with_extension("dwp"); + debug!(?dwp_out_filename, ?executable_out_filename); + + #[derive(Default)] + struct ThorinSession { + arena_data: TypedArena>, + arena_mmap: TypedArena, + arena_relocations: TypedArena, + } + + impl ThorinSession { + fn alloc_mmap<'arena>(&'arena self, data: Mmap) -> &'arena Mmap { + (*self.arena_mmap.alloc(data)).borrow() + } + } + + impl thorin::Session for ThorinSession { + fn alloc_data<'arena>(&'arena self, data: Vec) -> &'arena [u8] { + (*self.arena_data.alloc(data)).borrow() + } + + fn alloc_relocation<'arena>(&'arena self, data: Relocations) -> &'arena Relocations { + (*self.arena_relocations.alloc(data)).borrow() + } + + fn read_input<'arena>(&'arena self, path: &Path) -> std::io::Result<&'arena [u8]> { + let file = File::open(&path)?; + let mmap = (unsafe { Mmap::map(file) })?; + Ok(self.alloc_mmap(mmap)) + } + } + + match sess.time("run_thorin", || -> Result<(), thorin::Error> { + let thorin_sess = ThorinSession::default(); + let mut package = thorin::DwarfPackage::new(&thorin_sess); + + // Input objs contain .o/.dwo files from the current crate. + match sess.opts.unstable_opts.split_dwarf_kind { + SplitDwarfKind::Single => { + for input_obj in cg_results.modules.iter().filter_map(|m| m.object.as_ref()) { + package.add_input_object(input_obj)?; + } + } + SplitDwarfKind::Split => { + for input_obj in cg_results.modules.iter().filter_map(|m| m.dwarf_object.as_ref()) { + package.add_input_object(input_obj)?; + } + } + } + + // Input rlibs contain .o/.dwo files from dependencies. + let input_rlibs = cg_results + .crate_info + .used_crate_source + .values() + .filter_map(|csource| csource.rlib.as_ref()) + .map(|(path, _)| path); + for input_rlib in input_rlibs { + debug!(?input_rlib); + package.add_input_object(input_rlib)?; + } + + // Failing to read the referenced objects is expected for dependencies where the path in the + // executable will have been cleaned by Cargo, but the referenced objects will be contained + // within rlibs provided as inputs. + // + // If paths have been remapped, then .o/.dwo files from the current crate also won't be + // found, but are provided explicitly above. + // + // Adding an executable is primarily done to make `thorin` check that all the referenced + // dwarf objects are found in the end. + package.add_executable( + &executable_out_filename, + thorin::MissingReferencedObjectBehaviour::Skip, + )?; + + let output = package.finish()?.write()?; + let mut output_stream = BufWriter::new( + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(dwp_out_filename)?, + ); + output_stream.write_all(&output)?; + output_stream.flush()?; + + Ok(()) + }) { + Ok(()) => {} + Err(e) => { + sess.struct_err("linking dwarf objects with thorin failed") + .note(&format!("{:?}", e)) + .emit(); + sess.abort_if_errors(); + } + } +} + +/// Create a dynamic library or executable. +/// +/// This will invoke the system linker/cc to create the resulting file. This links to all upstream +/// files as well. +fn link_natively<'a>( + sess: &'a Session, + archive_builder_builder: &dyn ArchiveBuilderBuilder, + crate_type: CrateType, + out_filename: &Path, + codegen_results: &CodegenResults, + tmpdir: &Path, +) -> Result<(), ErrorGuaranteed> { + info!("preparing {:?} to {:?}", crate_type, out_filename); + let (linker_path, flavor) = linker_and_flavor(sess); + let mut cmd = linker_with_args( + &linker_path, + flavor, + sess, + archive_builder_builder, + crate_type, + tmpdir, + out_filename, + codegen_results, + )?; + + linker::disable_localization(&mut cmd); + + for &(ref k, ref v) in sess.target.link_env.as_ref() { + cmd.env(k.as_ref(), v.as_ref()); + } + for k in sess.target.link_env_remove.as_ref() { + cmd.env_remove(k.as_ref()); + } + + if sess.opts.prints.contains(&PrintRequest::LinkArgs) { + println!("{:?}", &cmd); + } + + // May have not found libraries in the right formats. + sess.abort_if_errors(); + + // Invoke the system linker + info!("{:?}", &cmd); + let retry_on_segfault = env::var("RUSTC_RETRY_LINKER_ON_SEGFAULT").is_ok(); + let unknown_arg_regex = + Regex::new(r"(unknown|unrecognized) (command line )?(option|argument)").unwrap(); + let mut prog; + let mut i = 0; + loop { + i += 1; + prog = sess.time("run_linker", || exec_linker(sess, &cmd, out_filename, tmpdir)); + let Ok(ref output) = prog else { + break; + }; + if output.status.success() { + break; + } + let mut out = output.stderr.clone(); + out.extend(&output.stdout); + let out = String::from_utf8_lossy(&out); + + // Check to see if the link failed with an error message that indicates it + // doesn't recognize the -no-pie option. If so, re-perform the link step + // without it. This is safe because if the linker doesn't support -no-pie + // then it should not default to linking executables as pie. Different + // versions of gcc seem to use different quotes in the error message so + // don't check for them. + if sess.target.linker_is_gnu + && flavor != LinkerFlavor::Ld + && unknown_arg_regex.is_match(&out) + && out.contains("-no-pie") + && cmd.get_args().iter().any(|e| e.to_string_lossy() == "-no-pie") + { + info!("linker output: {:?}", out); + warn!("Linker does not support -no-pie command line option. Retrying without."); + for arg in cmd.take_args() { + if arg.to_string_lossy() != "-no-pie" { + cmd.arg(arg); + } + } + info!("{:?}", &cmd); + continue; + } + + // Detect '-static-pie' used with an older version of gcc or clang not supporting it. + // Fallback from '-static-pie' to '-static' in that case. + if sess.target.linker_is_gnu + && flavor != LinkerFlavor::Ld + && unknown_arg_regex.is_match(&out) + && (out.contains("-static-pie") || out.contains("--no-dynamic-linker")) + && cmd.get_args().iter().any(|e| e.to_string_lossy() == "-static-pie") + { + info!("linker output: {:?}", out); + warn!( + "Linker does not support -static-pie command line option. Retrying with -static instead." + ); + // Mirror `add_(pre,post)_link_objects` to replace CRT objects. + let self_contained = crt_objects_fallback(sess, crate_type); + let opts = &sess.target; + let pre_objects = if self_contained { + &opts.pre_link_objects_fallback + } else { + &opts.pre_link_objects + }; + let post_objects = if self_contained { + &opts.post_link_objects_fallback + } else { + &opts.post_link_objects + }; + let get_objects = |objects: &CrtObjects, kind| { + objects + .get(&kind) + .iter() + .copied() + .flatten() + .map(|obj| get_object_file_path(sess, obj, self_contained).into_os_string()) + .collect::>() + }; + let pre_objects_static_pie = get_objects(pre_objects, LinkOutputKind::StaticPicExe); + let post_objects_static_pie = get_objects(post_objects, LinkOutputKind::StaticPicExe); + let mut pre_objects_static = get_objects(pre_objects, LinkOutputKind::StaticNoPicExe); + let mut post_objects_static = get_objects(post_objects, LinkOutputKind::StaticNoPicExe); + // Assume that we know insertion positions for the replacement arguments from replaced + // arguments, which is true for all supported targets. + assert!(pre_objects_static.is_empty() || !pre_objects_static_pie.is_empty()); + assert!(post_objects_static.is_empty() || !post_objects_static_pie.is_empty()); + for arg in cmd.take_args() { + if arg.to_string_lossy() == "-static-pie" { + // Replace the output kind. + cmd.arg("-static"); + } else if pre_objects_static_pie.contains(&arg) { + // Replace the pre-link objects (replace the first and remove the rest). + cmd.args(mem::take(&mut pre_objects_static)); + } else if post_objects_static_pie.contains(&arg) { + // Replace the post-link objects (replace the first and remove the rest). + cmd.args(mem::take(&mut post_objects_static)); + } else { + cmd.arg(arg); + } + } + info!("{:?}", &cmd); + continue; + } + + // Here's a terribly awful hack that really shouldn't be present in any + // compiler. Here an environment variable is supported to automatically + // retry the linker invocation if the linker looks like it segfaulted. + // + // Gee that seems odd, normally segfaults are things we want to know + // about! Unfortunately though in rust-lang/rust#38878 we're + // experiencing the linker segfaulting on Travis quite a bit which is + // causing quite a bit of pain to land PRs when they spuriously fail + // due to a segfault. + // + // The issue #38878 has some more debugging information on it as well, + // but this unfortunately looks like it's just a race condition in + // macOS's linker with some thread pool working in the background. It + // seems that no one currently knows a fix for this so in the meantime + // we're left with this... + if !retry_on_segfault || i > 3 { + break; + } + let msg_segv = "clang: error: unable to execute command: Segmentation fault: 11"; + let msg_bus = "clang: error: unable to execute command: Bus error: 10"; + if out.contains(msg_segv) || out.contains(msg_bus) { + warn!( + ?cmd, %out, + "looks like the linker segfaulted when we tried to call it, \ + automatically retrying again", + ); + continue; + } + + if is_illegal_instruction(&output.status) { + warn!( + ?cmd, %out, status = %output.status, + "looks like the linker hit an illegal instruction when we \ + tried to call it, automatically retrying again.", + ); + continue; + } + + #[cfg(unix)] + fn is_illegal_instruction(status: &ExitStatus) -> bool { + use std::os::unix::prelude::*; + status.signal() == Some(libc::SIGILL) + } + + #[cfg(not(unix))] + fn is_illegal_instruction(_status: &ExitStatus) -> bool { + false + } + } + + match prog { + Ok(prog) => { + if !prog.status.success() { + let mut output = prog.stderr.clone(); + output.extend_from_slice(&prog.stdout); + let escaped_output = escape_stdout_stderr_string(&output); + let mut err = sess.struct_err(&format!( + "linking with `{}` failed: {}", + linker_path.display(), + prog.status + )); + err.note(&format!("{:?}", &cmd)).note(&escaped_output); + if escaped_output.contains("undefined reference to") { + err.help( + "some `extern` functions couldn't be found; some native libraries may \ + need to be installed or have their path specified", + ); + err.note("use the `-l` flag to specify native libraries to link"); + err.note("use the `cargo:rustc-link-lib` directive to specify the native \ + libraries to link with Cargo (see https://doc.rust-lang.org/cargo/reference/build-scripts.html#cargorustc-link-libkindname)"); + } + err.emit(); + + // If MSVC's `link.exe` was expected but the return code + // is not a Microsoft LNK error then suggest a way to fix or + // install the Visual Studio build tools. + if let Some(code) = prog.status.code() { + if sess.target.is_like_msvc + && flavor == LinkerFlavor::Msvc + // Respect the command line override + && sess.opts.cg.linker.is_none() + // Match exactly "link.exe" + && linker_path.to_str() == Some("link.exe") + // All Microsoft `link.exe` linking error codes are + // four digit numbers in the range 1000 to 9999 inclusive + && (code < 1000 || code > 9999) + { + let is_vs_installed = windows_registry::find_vs_version().is_ok(); + let has_linker = windows_registry::find_tool( + &sess.opts.target_triple.triple(), + "link.exe", + ) + .is_some(); + + sess.note_without_error("`link.exe` returned an unexpected error"); + if is_vs_installed && has_linker { + // the linker is broken + sess.note_without_error( + "the Visual Studio build tools may need to be repaired \ + using the Visual Studio installer", + ); + sess.note_without_error( + "or a necessary component may be missing from the \ + \"C++ build tools\" workload", + ); + } else if is_vs_installed { + // the linker is not installed + sess.note_without_error( + "in the Visual Studio installer, ensure the \ + \"C++ build tools\" workload is selected", + ); + } else { + // visual studio is not installed + sess.note_without_error( + "you may need to install Visual Studio build tools with the \ + \"C++ build tools\" workload", + ); + } + } + } + + sess.abort_if_errors(); + } + info!("linker stderr:\n{}", escape_stdout_stderr_string(&prog.stderr)); + info!("linker stdout:\n{}", escape_stdout_stderr_string(&prog.stdout)); + } + Err(e) => { + let linker_not_found = e.kind() == io::ErrorKind::NotFound; + + let mut linker_error = { + if linker_not_found { + sess.struct_err(&format!("linker `{}` not found", linker_path.display())) + } else { + sess.struct_err(&format!( + "could not exec the linker `{}`", + linker_path.display() + )) + } + }; + + linker_error.note(&e.to_string()); + + if !linker_not_found { + linker_error.note(&format!("{:?}", &cmd)); + } + + linker_error.emit(); + + if sess.target.is_like_msvc && linker_not_found { + sess.note_without_error( + "the msvc targets depend on the msvc linker \ + but `link.exe` was not found", + ); + sess.note_without_error( + "please ensure that VS 2013, VS 2015, VS 2017, VS 2019 or VS 2022 \ + was installed with the Visual C++ option", + ); + } + sess.abort_if_errors(); + } + } + + match sess.split_debuginfo() { + // If split debug information is disabled or located in individual files + // there's nothing to do here. + SplitDebuginfo::Off | SplitDebuginfo::Unpacked => {} + + // If packed split-debuginfo is requested, but the final compilation + // doesn't actually have any debug information, then we skip this step. + SplitDebuginfo::Packed if sess.opts.debuginfo == DebugInfo::None => {} + + // On macOS the external `dsymutil` tool is used to create the packed + // debug information. Note that this will read debug information from + // the objects on the filesystem which we'll clean up later. + SplitDebuginfo::Packed if sess.target.is_like_osx => { + let prog = Command::new("dsymutil").arg(out_filename).output(); + match prog { + Ok(prog) => { + if !prog.status.success() { + let mut output = prog.stderr.clone(); + output.extend_from_slice(&prog.stdout); + sess.struct_warn(&format!( + "processing debug info with `dsymutil` failed: {}", + prog.status + )) + .note(&escape_string(&output)) + .emit(); + } + } + Err(e) => sess.fatal(&format!("unable to run `dsymutil`: {}", e)), + } + } + + // On MSVC packed debug information is produced by the linker itself so + // there's no need to do anything else here. + SplitDebuginfo::Packed if sess.target.is_like_windows => {} + + // ... and otherwise we're processing a `*.dwp` packed dwarf file. + // + // We cannot rely on the .o paths in the executable because they may have been + // remapped by --remap-path-prefix and therefore invalid, so we need to provide + // the .o/.dwo paths explicitly. + SplitDebuginfo::Packed => link_dwarf_object(sess, codegen_results, out_filename), + } + + let strip = strip_value(sess); + + if sess.target.is_like_osx { + match (strip, crate_type) { + (Strip::Debuginfo, _) => strip_symbols_in_osx(sess, &out_filename, Some("-S")), + // Per the manpage, `-x` is the maximum safe strip level for dynamic libraries. (#93988) + (Strip::Symbols, CrateType::Dylib | CrateType::Cdylib | CrateType::ProcMacro) => { + strip_symbols_in_osx(sess, &out_filename, Some("-x")) + } + (Strip::Symbols, _) => strip_symbols_in_osx(sess, &out_filename, None), + (Strip::None, _) => {} + } + } + + Ok(()) +} + +// Temporarily support both -Z strip and -C strip +fn strip_value(sess: &Session) -> Strip { + match (sess.opts.unstable_opts.strip, sess.opts.cg.strip) { + (s, Strip::None) => s, + (_, s) => s, + } +} + +fn strip_symbols_in_osx<'a>(sess: &'a Session, out_filename: &Path, option: Option<&str>) { + let mut cmd = Command::new("strip"); + if let Some(option) = option { + cmd.arg(option); + } + let prog = cmd.arg(out_filename).output(); + match prog { + Ok(prog) => { + if !prog.status.success() { + let mut output = prog.stderr.clone(); + output.extend_from_slice(&prog.stdout); + sess.struct_warn(&format!( + "stripping debug info with `strip` failed: {}", + prog.status + )) + .note(&escape_string(&output)) + .emit(); + } + } + Err(e) => sess.fatal(&format!("unable to run `strip`: {}", e)), + } +} + +fn escape_string(s: &[u8]) -> String { + str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| { + let mut x = "Non-UTF-8 output: ".to_string(); + x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from)); + x + }) +} + +fn add_sanitizer_libraries(sess: &Session, crate_type: CrateType, linker: &mut dyn Linker) { + // On macOS the runtimes are distributed as dylibs which should be linked to + // both executables and dynamic shared objects. Everywhere else the runtimes + // are currently distributed as static libraries which should be linked to + // executables only. + let needs_runtime = match crate_type { + CrateType::Executable => true, + CrateType::Dylib | CrateType::Cdylib | CrateType::ProcMacro => sess.target.is_like_osx, + CrateType::Rlib | CrateType::Staticlib => false, + }; + + if !needs_runtime { + return; + } + + let sanitizer = sess.opts.unstable_opts.sanitizer; + if sanitizer.contains(SanitizerSet::ADDRESS) { + link_sanitizer_runtime(sess, linker, "asan"); + } + if sanitizer.contains(SanitizerSet::LEAK) { + link_sanitizer_runtime(sess, linker, "lsan"); + } + if sanitizer.contains(SanitizerSet::MEMORY) { + link_sanitizer_runtime(sess, linker, "msan"); + } + if sanitizer.contains(SanitizerSet::THREAD) { + link_sanitizer_runtime(sess, linker, "tsan"); + } + if sanitizer.contains(SanitizerSet::HWADDRESS) { + link_sanitizer_runtime(sess, linker, "hwasan"); + } +} + +fn link_sanitizer_runtime(sess: &Session, linker: &mut dyn Linker, name: &str) { + fn find_sanitizer_runtime(sess: &Session, filename: &str) -> PathBuf { + let session_tlib = + filesearch::make_target_lib_path(&sess.sysroot, sess.opts.target_triple.triple()); + let path = session_tlib.join(filename); + if path.exists() { + return session_tlib; + } else { + let default_sysroot = filesearch::get_or_default_sysroot(); + let default_tlib = filesearch::make_target_lib_path( + &default_sysroot, + sess.opts.target_triple.triple(), + ); + return default_tlib; + } + } + + let channel = option_env!("CFG_RELEASE_CHANNEL") + .map(|channel| format!("-{}", channel)) + .unwrap_or_default(); + + if sess.target.is_like_osx { + // On Apple platforms, the sanitizer is always built as a dylib, and + // LLVM will link to `@rpath/*.dylib`, so we need to specify an + // rpath to the library as well (the rpath should be absolute, see + // PR #41352 for details). + let filename = format!("rustc{}_rt.{}", channel, name); + let path = find_sanitizer_runtime(&sess, &filename); + let rpath = path.to_str().expect("non-utf8 component in path"); + linker.args(&["-Wl,-rpath", "-Xlinker", rpath]); + linker.link_dylib(&filename, false, true); + } else { + let filename = format!("librustc{}_rt.{}.a", channel, name); + let path = find_sanitizer_runtime(&sess, &filename).join(&filename); + linker.link_whole_rlib(&path); + } +} + +/// Returns a boolean indicating whether the specified crate should be ignored +/// during LTO. +/// +/// Crates ignored during LTO are not lumped together in the "massive object +/// file" that we create and are linked in their normal rlib states. See +/// comments below for what crates do not participate in LTO. +/// +/// It's unusual for a crate to not participate in LTO. Typically only +/// compiler-specific and unstable crates have a reason to not participate in +/// LTO. +pub fn ignored_for_lto(sess: &Session, info: &CrateInfo, cnum: CrateNum) -> bool { + // If our target enables builtin function lowering in LLVM then the + // crates providing these functions don't participate in LTO (e.g. + // no_builtins or compiler builtins crates). + !sess.target.no_builtins + && (info.compiler_builtins == Some(cnum) || info.is_no_builtins.contains(&cnum)) +} + +// This functions tries to determine the appropriate linker (and corresponding LinkerFlavor) to use +pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { + fn infer_from( + sess: &Session, + linker: Option, + flavor: Option, + ) -> Option<(PathBuf, LinkerFlavor)> { + match (linker, flavor) { + (Some(linker), Some(flavor)) => Some((linker, flavor)), + // only the linker flavor is known; use the default linker for the selected flavor + (None, Some(flavor)) => Some(( + PathBuf::from(match flavor { + LinkerFlavor::Em => { + if cfg!(windows) { + "emcc.bat" + } else { + "emcc" + } + } + LinkerFlavor::Gcc => { + if cfg!(any(target_os = "solaris", target_os = "illumos")) { + // On historical Solaris systems, "cc" may have + // been Sun Studio, which is not flag-compatible + // with "gcc". This history casts a long shadow, + // and many modern illumos distributions today + // ship GCC as "gcc" without also making it + // available as "cc". + "gcc" + } else { + "cc" + } + } + LinkerFlavor::Ld => "ld", + LinkerFlavor::Msvc => "link.exe", + LinkerFlavor::Lld(_) => "lld", + LinkerFlavor::PtxLinker => "rust-ptx-linker", + LinkerFlavor::BpfLinker => "bpf-linker", + LinkerFlavor::L4Bender => "l4-bender", + }), + flavor, + )), + (Some(linker), None) => { + let stem = linker.file_stem().and_then(|stem| stem.to_str()).unwrap_or_else(|| { + sess.fatal("couldn't extract file stem from specified linker") + }); + + let flavor = if stem == "emcc" { + LinkerFlavor::Em + } else if stem == "gcc" + || stem.ends_with("-gcc") + || stem == "clang" + || stem.ends_with("-clang") + { + LinkerFlavor::Gcc + } else if stem == "wasm-ld" || stem.ends_with("-wasm-ld") { + LinkerFlavor::Lld(LldFlavor::Wasm) + } else if stem == "ld" || stem == "ld.lld" || stem.ends_with("-ld") { + LinkerFlavor::Ld + } else if stem == "link" || stem == "lld-link" { + LinkerFlavor::Msvc + } else if stem == "lld" || stem == "rust-lld" { + LinkerFlavor::Lld(sess.target.lld_flavor) + } else { + // fall back to the value in the target spec + sess.target.linker_flavor + }; + + Some((linker, flavor)) + } + (None, None) => None, + } + } + + // linker and linker flavor specified via command line have precedence over what the target + // specification specifies + if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) { + return ret; + } + + if let Some(ret) = infer_from( + sess, + sess.target.linker.as_deref().map(PathBuf::from), + Some(sess.target.linker_flavor), + ) { + return ret; + } + + bug!("Not enough information provided to determine how to invoke the linker"); +} + +/// Returns a pair of boolean indicating whether we should preserve the object and +/// dwarf object files on the filesystem for their debug information. This is often +/// useful with split-dwarf like schemes. +fn preserve_objects_for_their_debuginfo(sess: &Session) -> (bool, bool) { + // If the objects don't have debuginfo there's nothing to preserve. + if sess.opts.debuginfo == config::DebugInfo::None { + return (false, false); + } + + // If we're only producing artifacts that are archives, no need to preserve + // the objects as they're losslessly contained inside the archives. + if sess.crate_types().iter().all(|&x| x.is_archive()) { + return (false, false); + } + + match (sess.split_debuginfo(), sess.opts.unstable_opts.split_dwarf_kind) { + // If there is no split debuginfo then do not preserve objects. + (SplitDebuginfo::Off, _) => (false, false), + // If there is packed split debuginfo, then the debuginfo in the objects + // has been packaged and the objects can be deleted. + (SplitDebuginfo::Packed, _) => (false, false), + // If there is unpacked split debuginfo and the current target can not use + // split dwarf, then keep objects. + (SplitDebuginfo::Unpacked, _) if !sess.target_can_use_split_dwarf() => (true, false), + // If there is unpacked split debuginfo and the target can use split dwarf, then + // keep the object containing that debuginfo (whether that is an object file or + // dwarf object file depends on the split dwarf kind). + (SplitDebuginfo::Unpacked, SplitDwarfKind::Single) => (true, false), + (SplitDebuginfo::Unpacked, SplitDwarfKind::Split) => (false, true), + } +} + +fn archive_search_paths(sess: &Session) -> Vec { + sess.target_filesearch(PathKind::Native).search_path_dirs() +} + +#[derive(PartialEq)] +enum RlibFlavor { + Normal, + StaticlibBase, +} + +fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) { + let lib_args: Vec<_> = all_native_libs + .iter() + .filter(|l| relevant_lib(sess, l)) + .filter_map(|lib| { + let name = lib.name?; + match lib.kind { + NativeLibKind::Static { bundle: Some(false), .. } + | NativeLibKind::Dylib { .. } + | NativeLibKind::Unspecified => { + let verbatim = lib.verbatim.unwrap_or(false); + if sess.target.is_like_msvc { + Some(format!("{}{}", name, if verbatim { "" } else { ".lib" })) + } else if sess.target.linker_is_gnu { + Some(format!("-l{}{}", if verbatim { ":" } else { "" }, name)) + } else { + Some(format!("-l{}", name)) + } + } + NativeLibKind::Framework { .. } => { + // ld-only syntax, since there are no frameworks in MSVC + Some(format!("-framework {}", name)) + } + // These are included, no need to print them + NativeLibKind::Static { bundle: None | Some(true), .. } + | NativeLibKind::LinkArg + | NativeLibKind::RawDylib => None, + } + }) + .collect(); + if !lib_args.is_empty() { + sess.note_without_error( + "Link against the following native artifacts when linking \ + against this static library. The order and any duplication \ + can be significant on some platforms.", + ); + // Prefix for greppability + sess.note_without_error(&format!("native-static-libs: {}", &lib_args.join(" "))); + } +} + +fn get_object_file_path(sess: &Session, name: &str, self_contained: bool) -> PathBuf { + let fs = sess.target_filesearch(PathKind::Native); + let file_path = fs.get_lib_path().join(name); + if file_path.exists() { + return file_path; + } + // Special directory with objects used only in self-contained linkage mode + if self_contained { + let file_path = fs.get_self_contained_lib_path().join(name); + if file_path.exists() { + return file_path; + } + } + for search_path in fs.search_paths() { + let file_path = search_path.dir.join(name); + if file_path.exists() { + return file_path; + } + } + PathBuf::from(name) +} + +fn exec_linker( + sess: &Session, + cmd: &Command, + out_filename: &Path, + tmpdir: &Path, +) -> io::Result { + // When attempting to spawn the linker we run a risk of blowing out the + // size limits for spawning a new process with respect to the arguments + // we pass on the command line. + // + // Here we attempt to handle errors from the OS saying "your list of + // arguments is too big" by reinvoking the linker again with an `@`-file + // that contains all the arguments. The theory is that this is then + // accepted on all linkers and the linker will read all its options out of + // there instead of looking at the command line. + if !cmd.very_likely_to_exceed_some_spawn_limit() { + match cmd.command().stdout(Stdio::piped()).stderr(Stdio::piped()).spawn() { + Ok(child) => { + let output = child.wait_with_output(); + flush_linked_file(&output, out_filename)?; + return output; + } + Err(ref e) if command_line_too_big(e) => { + info!("command line to linker was too big: {}", e); + } + Err(e) => return Err(e), + } + } + + info!("falling back to passing arguments to linker via an @-file"); + let mut cmd2 = cmd.clone(); + let mut args = String::new(); + for arg in cmd2.take_args() { + args.push_str( + &Escape { arg: arg.to_str().unwrap(), is_like_msvc: sess.target.is_like_msvc } + .to_string(), + ); + args.push('\n'); + } + let file = tmpdir.join("linker-arguments"); + let bytes = if sess.target.is_like_msvc { + let mut out = Vec::with_capacity((1 + args.len()) * 2); + // start the stream with a UTF-16 BOM + for c in std::iter::once(0xFEFF).chain(args.encode_utf16()) { + // encode in little endian + out.push(c as u8); + out.push((c >> 8) as u8); + } + out + } else { + args.into_bytes() + }; + fs::write(&file, &bytes)?; + cmd2.arg(format!("@{}", file.display())); + info!("invoking linker {:?}", cmd2); + let output = cmd2.output(); + flush_linked_file(&output, out_filename)?; + return output; + + #[cfg(not(windows))] + fn flush_linked_file(_: &io::Result, _: &Path) -> io::Result<()> { + Ok(()) + } + + #[cfg(windows)] + fn flush_linked_file( + command_output: &io::Result, + out_filename: &Path, + ) -> io::Result<()> { + // On Windows, under high I/O load, output buffers are sometimes not flushed, + // even long after process exit, causing nasty, non-reproducible output bugs. + // + // File::sync_all() calls FlushFileBuffers() down the line, which solves the problem. + // + // А full writeup of the original Chrome bug can be found at + // randomascii.wordpress.com/2018/02/25/compiler-bug-linker-bug-windows-kernel-bug/amp + + if let &Ok(ref out) = command_output { + if out.status.success() { + if let Ok(of) = fs::OpenOptions::new().write(true).open(out_filename) { + of.sync_all()?; + } + } + } + + Ok(()) + } + + #[cfg(unix)] + fn command_line_too_big(err: &io::Error) -> bool { + err.raw_os_error() == Some(::libc::E2BIG) + } + + #[cfg(windows)] + fn command_line_too_big(err: &io::Error) -> bool { + const ERROR_FILENAME_EXCED_RANGE: i32 = 206; + err.raw_os_error() == Some(ERROR_FILENAME_EXCED_RANGE) + } + + #[cfg(not(any(unix, windows)))] + fn command_line_too_big(_: &io::Error) -> bool { + false + } + + struct Escape<'a> { + arg: &'a str, + is_like_msvc: bool, + } + + impl<'a> fmt::Display for Escape<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.is_like_msvc { + // This is "documented" at + // https://docs.microsoft.com/en-us/cpp/build/reference/at-specify-a-linker-response-file + // + // Unfortunately there's not a great specification of the + // syntax I could find online (at least) but some local + // testing showed that this seemed sufficient-ish to catch + // at least a few edge cases. + write!(f, "\"")?; + for c in self.arg.chars() { + match c { + '"' => write!(f, "\\{}", c)?, + c => write!(f, "{}", c)?, + } + } + write!(f, "\"")?; + } else { + // This is documented at https://linux.die.net/man/1/ld, namely: + // + // > Options in file are separated by whitespace. A whitespace + // > character may be included in an option by surrounding the + // > entire option in either single or double quotes. Any + // > character (including a backslash) may be included by + // > prefixing the character to be included with a backslash. + // + // We put an argument on each line, so all we need to do is + // ensure the line is interpreted as one whole argument. + for c in self.arg.chars() { + match c { + '\\' | ' ' => write!(f, "\\{}", c)?, + c => write!(f, "{}", c)?, + } + } + } + Ok(()) + } + } +} + +fn link_output_kind(sess: &Session, crate_type: CrateType) -> LinkOutputKind { + let kind = match (crate_type, sess.crt_static(Some(crate_type)), sess.relocation_model()) { + (CrateType::Executable, _, _) if sess.is_wasi_reactor() => LinkOutputKind::WasiReactorExe, + (CrateType::Executable, false, RelocModel::Pic | RelocModel::Pie) => { + LinkOutputKind::DynamicPicExe + } + (CrateType::Executable, false, _) => LinkOutputKind::DynamicNoPicExe, + (CrateType::Executable, true, RelocModel::Pic | RelocModel::Pie) => { + LinkOutputKind::StaticPicExe + } + (CrateType::Executable, true, _) => LinkOutputKind::StaticNoPicExe, + (_, true, _) => LinkOutputKind::StaticDylib, + (_, false, _) => LinkOutputKind::DynamicDylib, + }; + + // Adjust the output kind to target capabilities. + let opts = &sess.target; + let pic_exe_supported = opts.position_independent_executables; + let static_pic_exe_supported = opts.static_position_independent_executables; + let static_dylib_supported = opts.crt_static_allows_dylibs; + match kind { + LinkOutputKind::DynamicPicExe if !pic_exe_supported => LinkOutputKind::DynamicNoPicExe, + LinkOutputKind::StaticPicExe if !static_pic_exe_supported => LinkOutputKind::StaticNoPicExe, + LinkOutputKind::StaticDylib if !static_dylib_supported => LinkOutputKind::DynamicDylib, + _ => kind, + } +} + +// Returns true if linker is located within sysroot +fn detect_self_contained_mingw(sess: &Session) -> bool { + let (linker, _) = linker_and_flavor(&sess); + // Assume `-C linker=rust-lld` as self-contained mode + if linker == Path::new("rust-lld") { + return true; + } + let linker_with_extension = if cfg!(windows) && linker.extension().is_none() { + linker.with_extension("exe") + } else { + linker + }; + for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { + let full_path = dir.join(&linker_with_extension); + // If linker comes from sysroot assume self-contained mode + if full_path.is_file() && !full_path.starts_with(&sess.sysroot) { + return false; + } + } + true +} + +/// Whether we link to our own CRT objects instead of relying on gcc to pull them. +/// We only provide such support for a very limited number of targets. +fn crt_objects_fallback(sess: &Session, crate_type: CrateType) -> bool { + if let Some(self_contained) = sess.opts.cg.link_self_contained { + return self_contained; + } + + match sess.target.crt_objects_fallback { + // FIXME: Find a better heuristic for "native musl toolchain is available", + // based on host and linker path, for example. + // (https://github.com/rust-lang/rust/pull/71769#issuecomment-626330237). + Some(CrtObjectsFallback::Musl) => sess.crt_static(Some(crate_type)), + Some(CrtObjectsFallback::Mingw) => { + sess.host == sess.target + && sess.target.vendor != "uwp" + && detect_self_contained_mingw(&sess) + } + // FIXME: Figure out cases in which WASM needs to link with a native toolchain. + Some(CrtObjectsFallback::Wasm) => true, + None => false, + } +} + +/// Add pre-link object files defined by the target spec. +fn add_pre_link_objects( + cmd: &mut dyn Linker, + sess: &Session, + link_output_kind: LinkOutputKind, + self_contained: bool, +) { + let opts = &sess.target; + let objects = + if self_contained { &opts.pre_link_objects_fallback } else { &opts.pre_link_objects }; + for obj in objects.get(&link_output_kind).iter().copied().flatten() { + cmd.add_object(&get_object_file_path(sess, obj, self_contained)); + } +} + +/// Add post-link object files defined by the target spec. +fn add_post_link_objects( + cmd: &mut dyn Linker, + sess: &Session, + link_output_kind: LinkOutputKind, + self_contained: bool, +) { + let opts = &sess.target; + let objects = + if self_contained { &opts.post_link_objects_fallback } else { &opts.post_link_objects }; + for obj in objects.get(&link_output_kind).iter().copied().flatten() { + cmd.add_object(&get_object_file_path(sess, obj, self_contained)); + } +} + +/// Add arbitrary "pre-link" args defined by the target spec or from command line. +/// FIXME: Determine where exactly these args need to be inserted. +fn add_pre_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { + if let Some(args) = sess.target.pre_link_args.get(&flavor) { + cmd.args(args.iter().map(Deref::deref)); + } + cmd.args(&sess.opts.unstable_opts.pre_link_args); +} + +/// Add a link script embedded in the target, if applicable. +fn add_link_script(cmd: &mut dyn Linker, sess: &Session, tmpdir: &Path, crate_type: CrateType) { + match (crate_type, &sess.target.link_script) { + (CrateType::Cdylib | CrateType::Executable, Some(script)) => { + if !sess.target.linker_is_gnu { + sess.fatal("can only use link script when linking with GNU-like linker"); + } + + let file_name = ["rustc", &sess.target.llvm_target, "linkfile.ld"].join("-"); + + let path = tmpdir.join(file_name); + if let Err(e) = fs::write(&path, script.as_ref()) { + sess.fatal(&format!("failed to write link script to {}: {}", path.display(), e)); + } + + cmd.arg("--script"); + cmd.arg(path); + } + _ => {} + } +} + +/// Add arbitrary "user defined" args defined from command line. +/// FIXME: Determine where exactly these args need to be inserted. +fn add_user_defined_link_args(cmd: &mut dyn Linker, sess: &Session) { + cmd.args(&sess.opts.cg.link_args); +} + +/// Add arbitrary "late link" args defined by the target spec. +/// FIXME: Determine where exactly these args need to be inserted. +fn add_late_link_args( + cmd: &mut dyn Linker, + sess: &Session, + flavor: LinkerFlavor, + crate_type: CrateType, + codegen_results: &CodegenResults, +) { + let any_dynamic_crate = crate_type == CrateType::Dylib + || codegen_results.crate_info.dependency_formats.iter().any(|(ty, list)| { + *ty == crate_type && list.iter().any(|&linkage| linkage == Linkage::Dynamic) + }); + if any_dynamic_crate { + if let Some(args) = sess.target.late_link_args_dynamic.get(&flavor) { + cmd.args(args.iter().map(Deref::deref)); + } + } else { + if let Some(args) = sess.target.late_link_args_static.get(&flavor) { + cmd.args(args.iter().map(Deref::deref)); + } + } + if let Some(args) = sess.target.late_link_args.get(&flavor) { + cmd.args(args.iter().map(Deref::deref)); + } +} + +/// Add arbitrary "post-link" args defined by the target spec. +/// FIXME: Determine where exactly these args need to be inserted. +fn add_post_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { + if let Some(args) = sess.target.post_link_args.get(&flavor) { + cmd.args(args.iter().map(Deref::deref)); + } +} + +/// Add a synthetic object file that contains reference to all symbols that we want to expose to +/// the linker. +/// +/// Background: we implement rlibs as static library (archives). Linkers treat archives +/// differently from object files: all object files participate in linking, while archives will +/// only participate in linking if they can satisfy at least one undefined reference (version +/// scripts doesn't count). This causes `#[no_mangle]` or `#[used]` items to be ignored by the +/// linker, and since they never participate in the linking, using `KEEP` in the linker scripts +/// can't keep them either. This causes #47384. +/// +/// To keep them around, we could use `--whole-archive` and equivalents to force rlib to +/// participate in linking like object files, but this proves to be expensive (#93791). Therefore +/// we instead just introduce an undefined reference to them. This could be done by `-u` command +/// line option to the linker or `EXTERN(...)` in linker scripts, however they does not only +/// introduce an undefined reference, but also make them the GC roots, preventing `--gc-sections` +/// from removing them, and this is especially problematic for embedded programming where every +/// byte counts. +/// +/// This method creates a synthetic object file, which contains undefined references to all symbols +/// that are necessary for the linking. They are only present in symbol table but not actually +/// used in any sections, so the linker will therefore pick relevant rlibs for linking, but +/// unused `#[no_mangle]` or `#[used]` can still be discard by GC sections. +fn add_linked_symbol_object( + cmd: &mut dyn Linker, + sess: &Session, + tmpdir: &Path, + symbols: &[(String, SymbolExportKind)], +) { + if symbols.is_empty() { + return; + } + + let Some(mut file) = super::metadata::create_object_file(sess) else { + return; + }; + + // NOTE(nbdd0121): MSVC will hang if the input object file contains no sections, + // so add an empty section. + if file.format() == object::BinaryFormat::Coff { + file.add_section(Vec::new(), ".text".into(), object::SectionKind::Text); + + // We handle the name decoration of COFF targets in `symbol_export.rs`, so disable the + // default mangler in `object` crate. + file.set_mangling(object::write::Mangling::None); + + // Add feature flags to the object file. On MSVC this is optional but LLD will complain if + // not present. + let mut feature = 0; + + if file.architecture() == object::Architecture::I386 { + // Indicate that all SEH handlers are registered in .sxdata section. + // We don't have generate any code, so we don't need .sxdata section but LLD still + // expects us to set this bit (see #96498). + // Reference: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format + feature |= 1; + } + + file.add_symbol(object::write::Symbol { + name: "@feat.00".into(), + value: feature, + size: 0, + kind: object::SymbolKind::Data, + scope: object::SymbolScope::Compilation, + weak: false, + section: object::write::SymbolSection::Absolute, + flags: object::SymbolFlags::None, + }); + } + + for (sym, kind) in symbols.iter() { + file.add_symbol(object::write::Symbol { + name: sym.clone().into(), + value: 0, + size: 0, + kind: match kind { + SymbolExportKind::Text => object::SymbolKind::Text, + SymbolExportKind::Data => object::SymbolKind::Data, + SymbolExportKind::Tls => object::SymbolKind::Tls, + }, + scope: object::SymbolScope::Unknown, + weak: false, + section: object::write::SymbolSection::Undefined, + flags: object::SymbolFlags::None, + }); + } + + let path = tmpdir.join("symbols.o"); + let result = std::fs::write(&path, file.write().unwrap()); + if let Err(e) = result { + sess.fatal(&format!("failed to write {}: {}", path.display(), e)); + } + cmd.add_object(&path); +} + +/// Add object files containing code from the current crate. +fn add_local_crate_regular_objects(cmd: &mut dyn Linker, codegen_results: &CodegenResults) { + for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) { + cmd.add_object(obj); + } +} + +/// Add object files for allocator code linked once for the whole crate tree. +fn add_local_crate_allocator_objects(cmd: &mut dyn Linker, codegen_results: &CodegenResults) { + if let Some(obj) = codegen_results.allocator_module.as_ref().and_then(|m| m.object.as_ref()) { + cmd.add_object(obj); + } +} + +/// Add object files containing metadata for the current crate. +fn add_local_crate_metadata_objects( + cmd: &mut dyn Linker, + crate_type: CrateType, + codegen_results: &CodegenResults, +) { + // When linking a dynamic library, we put the metadata into a section of the + // executable. This metadata is in a separate object file from the main + // object file, so we link that in here. + if crate_type == CrateType::Dylib || crate_type == CrateType::ProcMacro { + if let Some(obj) = codegen_results.metadata_module.as_ref().and_then(|m| m.object.as_ref()) + { + cmd.add_object(obj); + } + } +} + +/// Add sysroot and other globally set directories to the directory search list. +fn add_library_search_dirs(cmd: &mut dyn Linker, sess: &Session, self_contained: bool) { + // The default library location, we need this to find the runtime. + // The location of crates will be determined as needed. + let lib_path = sess.target_filesearch(PathKind::All).get_lib_path(); + cmd.include_path(&fix_windows_verbatim_for_gcc(&lib_path)); + + // Special directory with libraries used only in self-contained linkage mode + if self_contained { + let lib_path = sess.target_filesearch(PathKind::All).get_self_contained_lib_path(); + cmd.include_path(&fix_windows_verbatim_for_gcc(&lib_path)); + } +} + +/// Add options making relocation sections in the produced ELF files read-only +/// and suppressing lazy binding. +fn add_relro_args(cmd: &mut dyn Linker, sess: &Session) { + match sess.opts.unstable_opts.relro_level.unwrap_or(sess.target.relro_level) { + RelroLevel::Full => cmd.full_relro(), + RelroLevel::Partial => cmd.partial_relro(), + RelroLevel::Off => cmd.no_relro(), + RelroLevel::None => {} + } +} + +/// Add library search paths used at runtime by dynamic linkers. +fn add_rpath_args( + cmd: &mut dyn Linker, + sess: &Session, + codegen_results: &CodegenResults, + out_filename: &Path, +) { + // FIXME (#2397): At some point we want to rpath our guesses as to + // where extern libraries might live, based on the + // add_lib_search_paths + if sess.opts.cg.rpath { + let libs = codegen_results + .crate_info + .used_crates + .iter() + .filter_map(|cnum| { + codegen_results.crate_info.used_crate_source[cnum] + .dylib + .as_ref() + .map(|(path, _)| &**path) + }) + .collect::>(); + let mut rpath_config = RPathConfig { + libs: &*libs, + out_filename: out_filename.to_path_buf(), + has_rpath: sess.target.has_rpath, + is_like_osx: sess.target.is_like_osx, + linker_is_gnu: sess.target.linker_is_gnu, + }; + cmd.args(&rpath::get_rpath_flags(&mut rpath_config)); + } +} + +/// Produce the linker command line containing linker path and arguments. +/// +/// When comments in the function say "order-(in)dependent" they mean order-dependence between +/// options and libraries/object files. For example `--whole-archive` (order-dependent) applies +/// to specific libraries passed after it, and `-o` (output file, order-independent) applies +/// to the linking process as a whole. +/// Order-independent options may still override each other in order-dependent fashion, +/// e.g `--foo=yes --foo=no` may be equivalent to `--foo=no`. +fn linker_with_args<'a>( + path: &Path, + flavor: LinkerFlavor, + sess: &'a Session, + archive_builder_builder: &dyn ArchiveBuilderBuilder, + crate_type: CrateType, + tmpdir: &Path, + out_filename: &Path, + codegen_results: &CodegenResults, +) -> Result { + let crt_objects_fallback = crt_objects_fallback(sess, crate_type); + let cmd = &mut *super::linker::get_linker( + sess, + path, + flavor, + crt_objects_fallback, + &codegen_results.crate_info.target_cpu, + ); + let link_output_kind = link_output_kind(sess, crate_type); + + // ------------ Early order-dependent options ------------ + + // If we're building something like a dynamic library then some platforms + // need to make sure that all symbols are exported correctly from the + // dynamic library. + // Must be passed before any libraries to prevent the symbols to export from being thrown away, + // at least on some platforms (e.g. windows-gnu). + cmd.export_symbols( + tmpdir, + crate_type, + &codegen_results.crate_info.exported_symbols[&crate_type], + ); + + // Can be used for adding custom CRT objects or overriding order-dependent options above. + // FIXME: In practice built-in target specs use this for arbitrary order-independent options, + // introduce a target spec option for order-independent linker options and migrate built-in + // specs to it. + add_pre_link_args(cmd, sess, flavor); + + // ------------ Object code and libraries, order-dependent ------------ + + // Pre-link CRT objects. + add_pre_link_objects(cmd, sess, link_output_kind, crt_objects_fallback); + + add_linked_symbol_object( + cmd, + sess, + tmpdir, + &codegen_results.crate_info.linked_symbols[&crate_type], + ); + + // Sanitizer libraries. + add_sanitizer_libraries(sess, crate_type, cmd); + + // Object code from the current crate. + // Take careful note of the ordering of the arguments we pass to the linker + // here. Linkers will assume that things on the left depend on things to the + // right. Things on the right cannot depend on things on the left. This is + // all formally implemented in terms of resolving symbols (libs on the right + // resolve unknown symbols of libs on the left, but not vice versa). + // + // For this reason, we have organized the arguments we pass to the linker as + // such: + // + // 1. The local object that LLVM just generated + // 2. Local native libraries + // 3. Upstream rust libraries + // 4. Upstream native libraries + // + // The rationale behind this ordering is that those items lower down in the + // list can't depend on items higher up in the list. For example nothing can + // depend on what we just generated (e.g., that'd be a circular dependency). + // Upstream rust libraries are not supposed to depend on our local native + // libraries as that would violate the structure of the DAG, in that + // scenario they are required to link to them as well in a shared fashion. + // (The current implementation still doesn't prevent it though, see the FIXME below.) + // + // Note that upstream rust libraries may contain native dependencies as + // well, but they also can't depend on what we just started to add to the + // link line. And finally upstream native libraries can't depend on anything + // in this DAG so far because they can only depend on other native libraries + // and such dependencies are also required to be specified. + add_local_crate_regular_objects(cmd, codegen_results); + add_local_crate_metadata_objects(cmd, crate_type, codegen_results); + add_local_crate_allocator_objects(cmd, codegen_results); + + // Avoid linking to dynamic libraries unless they satisfy some undefined symbols + // at the point at which they are specified on the command line. + // Must be passed before any (dynamic) libraries to have effect on them. + // On Solaris-like systems, `-z ignore` acts as both `--as-needed` and `--gc-sections` + // so it will ignore unreferenced ELF sections from relocatable objects. + // For that reason, we put this flag after metadata objects as they would otherwise be removed. + // FIXME: Support more fine-grained dead code removal on Solaris/illumos + // and move this option back to the top. + cmd.add_as_needed(); + + // FIXME: Move this below to other native libraries + // (or alternatively link all native libraries after their respective crates). + // This change is somewhat breaking in practice due to local static libraries being linked + // as whole-archive (#85144), so removing whole-archive may be a pre-requisite. + if sess.opts.unstable_opts.link_native_libraries { + add_local_native_libraries(cmd, sess, codegen_results); + } + + // Upstream rust libraries and their non-bundled static libraries + add_upstream_rust_crates( + cmd, + sess, + archive_builder_builder, + codegen_results, + crate_type, + tmpdir, + ); + + // Upstream dynamic native libraries linked with `#[link]` attributes at and `-l` + // command line options. + // If -Zlink-native-libraries=false is set, then the assumption is that an + // external build system already has the native dependencies defined, and it + // will provide them to the linker itself. + if sess.opts.unstable_opts.link_native_libraries { + add_upstream_native_libraries(cmd, sess, codegen_results); + } + + // Link with the import library generated for any raw-dylib functions. + for (raw_dylib_name, raw_dylib_imports) in + collate_raw_dylibs(sess, &codegen_results.crate_info.used_libraries)? + { + cmd.add_object(&archive_builder_builder.create_dll_import_lib( + sess, + &raw_dylib_name, + &raw_dylib_imports, + tmpdir, + )); + } + + // Library linking above uses some global state for things like `-Bstatic`/`-Bdynamic` to make + // command line shorter, reset it to default here before adding more libraries. + cmd.reset_per_library_state(); + + // FIXME: Built-in target specs occasionally use this for linking system libraries, + // eliminate all such uses by migrating them to `#[link]` attributes in `lib(std,c,unwind)` + // and remove the option. + add_late_link_args(cmd, sess, flavor, crate_type, codegen_results); + + // ------------ Arbitrary order-independent options ------------ + + // Add order-independent options determined by rustc from its compiler options, + // target properties and source code. + add_order_independent_options( + cmd, + sess, + link_output_kind, + crt_objects_fallback, + flavor, + crate_type, + codegen_results, + out_filename, + tmpdir, + ); + + // Can be used for arbitrary order-independent options. + // In practice may also be occasionally used for linking native libraries. + // Passed after compiler-generated options to support manual overriding when necessary. + add_user_defined_link_args(cmd, sess); + + // ------------ Object code and libraries, order-dependent ------------ + + // Post-link CRT objects. + add_post_link_objects(cmd, sess, link_output_kind, crt_objects_fallback); + + // ------------ Late order-dependent options ------------ + + // Doesn't really make sense. + // FIXME: In practice built-in target specs use this for arbitrary order-independent options, + // introduce a target spec option for order-independent linker options, migrate built-in specs + // to it and remove the option. + add_post_link_args(cmd, sess, flavor); + + Ok(cmd.take_cmd()) +} + +fn add_order_independent_options( + cmd: &mut dyn Linker, + sess: &Session, + link_output_kind: LinkOutputKind, + crt_objects_fallback: bool, + flavor: LinkerFlavor, + crate_type: CrateType, + codegen_results: &CodegenResults, + out_filename: &Path, + tmpdir: &Path, +) { + add_gcc_ld_path(cmd, sess, flavor); + + add_apple_sdk(cmd, sess, flavor); + + add_link_script(cmd, sess, tmpdir, crate_type); + + if sess.target.os == "fuchsia" && crate_type == CrateType::Executable { + let prefix = if sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::ADDRESS) { + "asan/" + } else { + "" + }; + cmd.arg(format!("--dynamic-linker={}ld.so.1", prefix)); + } + + if sess.target.eh_frame_header { + cmd.add_eh_frame_header(); + } + + // Make the binary compatible with data execution prevention schemes. + cmd.add_no_exec(); + + if crt_objects_fallback { + cmd.no_crt_objects(); + } + + if sess.target.os == "emscripten" { + cmd.arg("-s"); + cmd.arg(if sess.panic_strategy() == PanicStrategy::Abort { + "DISABLE_EXCEPTION_CATCHING=1" + } else { + "DISABLE_EXCEPTION_CATCHING=0" + }); + } + + if flavor == LinkerFlavor::PtxLinker { + // Provide the linker with fallback to internal `target-cpu`. + cmd.arg("--fallback-arch"); + cmd.arg(&codegen_results.crate_info.target_cpu); + } else if flavor == LinkerFlavor::BpfLinker { + cmd.arg("--cpu"); + cmd.arg(&codegen_results.crate_info.target_cpu); + cmd.arg("--cpu-features"); + cmd.arg(match &sess.opts.cg.target_feature { + feat if !feat.is_empty() => feat.as_ref(), + _ => sess.target.options.features.as_ref(), + }); + } + + cmd.linker_plugin_lto(); + + add_library_search_dirs(cmd, sess, crt_objects_fallback); + + cmd.output_filename(out_filename); + + if crate_type == CrateType::Executable && sess.target.is_like_windows { + if let Some(ref s) = codegen_results.crate_info.windows_subsystem { + cmd.subsystem(s); + } + } + + // Try to strip as much out of the generated object by removing unused + // sections if possible. See more comments in linker.rs + if !sess.link_dead_code() { + // If PGO is enabled sometimes gc_sections will remove the profile data section + // as it appears to be unused. This can then cause the PGO profile file to lose + // some functions. If we are generating a profile we shouldn't strip those metadata + // sections to ensure we have all the data for PGO. + let keep_metadata = + crate_type == CrateType::Dylib || sess.opts.cg.profile_generate.enabled(); + if crate_type != CrateType::Executable || !sess.opts.unstable_opts.export_executable_symbols + { + cmd.gc_sections(keep_metadata); + } else { + cmd.no_gc_sections(); + } + } + + cmd.set_output_kind(link_output_kind, out_filename); + + add_relro_args(cmd, sess); + + // Pass optimization flags down to the linker. + cmd.optimize(); + + // Gather the set of NatVis files, if any, and write them out to a temp directory. + let natvis_visualizers = collect_natvis_visualizers( + tmpdir, + sess, + &codegen_results.crate_info.local_crate_name, + &codegen_results.crate_info.natvis_debugger_visualizers, + ); + + // Pass debuginfo, NatVis debugger visualizers and strip flags down to the linker. + cmd.debuginfo(strip_value(sess), &natvis_visualizers); + + // We want to prevent the compiler from accidentally leaking in any system libraries, + // so by default we tell linkers not to link to any default libraries. + if !sess.opts.cg.default_linker_libraries && sess.target.no_default_libraries { + cmd.no_default_libraries(); + } + + if sess.opts.cg.profile_generate.enabled() || sess.instrument_coverage() { + cmd.pgo_gen(); + } + + if sess.opts.cg.control_flow_guard != CFGuard::Disabled { + cmd.control_flow_guard(); + } + + add_rpath_args(cmd, sess, codegen_results, out_filename); +} + +// Write the NatVis debugger visualizer files for each crate to the temp directory and gather the file paths. +fn collect_natvis_visualizers( + tmpdir: &Path, + sess: &Session, + crate_name: &Symbol, + natvis_debugger_visualizers: &BTreeSet, +) -> Vec { + let mut visualizer_paths = Vec::with_capacity(natvis_debugger_visualizers.len()); + + for (index, visualizer) in natvis_debugger_visualizers.iter().enumerate() { + let visualizer_out_file = tmpdir.join(format!("{}-{}.natvis", crate_name.as_str(), index)); + + match fs::write(&visualizer_out_file, &visualizer.src) { + Ok(()) => { + visualizer_paths.push(visualizer_out_file); + } + Err(error) => { + sess.warn( + format!( + "Unable to write debugger visualizer file `{}`: {} ", + visualizer_out_file.display(), + error + ) + .as_str(), + ); + } + }; + } + visualizer_paths +} + +/// # Native library linking +/// +/// User-supplied library search paths (-L on the command line). These are the same paths used to +/// find Rust crates, so some of them may have been added already by the previous crate linking +/// code. This only allows them to be found at compile time so it is still entirely up to outside +/// forces to make sure that library can be found at runtime. +/// +/// Also note that the native libraries linked here are only the ones located in the current crate. +/// Upstream crates with native library dependencies may have their native library pulled in above. +fn add_local_native_libraries( + cmd: &mut dyn Linker, + sess: &Session, + codegen_results: &CodegenResults, +) { + let filesearch = sess.target_filesearch(PathKind::All); + for search_path in filesearch.search_paths() { + match search_path.kind { + PathKind::Framework => { + cmd.framework_path(&search_path.dir); + } + _ => { + cmd.include_path(&fix_windows_verbatim_for_gcc(&search_path.dir)); + } + } + } + + let relevant_libs = + codegen_results.crate_info.used_libraries.iter().filter(|l| relevant_lib(sess, l)); + + let search_path = OnceCell::new(); + let mut last = (None, NativeLibKind::Unspecified, None); + for lib in relevant_libs { + let Some(name) = lib.name else { + continue; + }; + let name = name.as_str(); + + // Skip if this library is the same as the last. + last = if (lib.name, lib.kind, lib.verbatim) == last { + continue; + } else { + (lib.name, lib.kind, lib.verbatim) + }; + + let verbatim = lib.verbatim.unwrap_or(false); + match lib.kind { + NativeLibKind::Dylib { as_needed } => { + cmd.link_dylib(name, verbatim, as_needed.unwrap_or(true)) + } + NativeLibKind::Unspecified => cmd.link_dylib(name, verbatim, true), + NativeLibKind::Framework { as_needed } => { + cmd.link_framework(name, as_needed.unwrap_or(true)) + } + NativeLibKind::Static { whole_archive, bundle, .. } => { + if whole_archive == Some(true) + // Backward compatibility case: this can be a rlib (so `+whole-archive` cannot + // be added explicitly if necessary, see the error in `fn link_rlib`) compiled + // as an executable due to `--test`. Use whole-archive implicitly, like before + // the introduction of native lib modifiers. + || (whole_archive == None && bundle != Some(false) && sess.opts.test) + { + cmd.link_whole_staticlib( + name, + verbatim, + &search_path.get_or_init(|| archive_search_paths(sess)), + ); + } else { + cmd.link_staticlib(name, verbatim) + } + } + NativeLibKind::RawDylib => { + // Ignore RawDylib here, they are handled separately in linker_with_args(). + } + NativeLibKind::LinkArg => { + cmd.arg(name); + } + } + } +} + +/// # Linking Rust crates and their non-bundled static libraries +/// +/// Rust crates are not considered at all when creating an rlib output. All dependencies will be +/// linked when producing the final output (instead of the intermediate rlib version). +fn add_upstream_rust_crates<'a>( + cmd: &mut dyn Linker, + sess: &'a Session, + archive_builder_builder: &dyn ArchiveBuilderBuilder, + codegen_results: &CodegenResults, + crate_type: CrateType, + tmpdir: &Path, +) { + // All of the heavy lifting has previously been accomplished by the + // dependency_format module of the compiler. This is just crawling the + // output of that module, adding crates as necessary. + // + // Linking to a rlib involves just passing it to the linker (the linker + // will slurp up the object files inside), and linking to a dynamic library + // involves just passing the right -l flag. + + let (_, data) = codegen_results + .crate_info + .dependency_formats + .iter() + .find(|(ty, _)| *ty == crate_type) + .expect("failed to find crate type in dependency format list"); + + // Invoke get_used_crates to ensure that we get a topological sorting of + // crates. + let deps = &codegen_results.crate_info.used_crates; + + // There's a few internal crates in the standard library (aka libcore and + // libstd) which actually have a circular dependence upon one another. This + // currently arises through "weak lang items" where libcore requires things + // like `rust_begin_unwind` but libstd ends up defining it. To get this + // circular dependence to work correctly in all situations we'll need to be + // sure to correctly apply the `--start-group` and `--end-group` options to + // GNU linkers, otherwise if we don't use any other symbol from the standard + // library it'll get discarded and the whole application won't link. + // + // In this loop we're calculating the `group_end`, after which crate to + // pass `--end-group` and `group_start`, before which crate to pass + // `--start-group`. We currently do this by passing `--end-group` after + // the first crate (when iterating backwards) that requires a lang item + // defined somewhere else. Once that's set then when we've defined all the + // necessary lang items we'll pass `--start-group`. + // + // Note that this isn't amazing logic for now but it should do the trick + // for the current implementation of the standard library. + let mut group_end = None; + let mut group_start = None; + // Crates available for linking thus far. + let mut available = FxHashSet::default(); + // Crates required to satisfy dependencies discovered so far. + let mut required = FxHashSet::default(); + + let info = &codegen_results.crate_info; + for &cnum in deps.iter().rev() { + if let Some(missing) = info.missing_lang_items.get(&cnum) { + let missing_crates = missing.iter().map(|i| info.lang_item_to_crate.get(i).copied()); + required.extend(missing_crates); + } + + required.insert(Some(cnum)); + available.insert(Some(cnum)); + + if required.len() > available.len() && group_end.is_none() { + group_end = Some(cnum); + } + if required.len() == available.len() && group_end.is_some() { + group_start = Some(cnum); + break; + } + } + + // If we didn't end up filling in all lang items from upstream crates then + // we'll be filling it in with our crate. This probably means we're the + // standard library itself, so skip this for now. + if group_end.is_some() && group_start.is_none() { + group_end = None; + } + + let mut compiler_builtins = None; + let search_path = OnceCell::new(); + + for &cnum in deps.iter() { + if group_start == Some(cnum) { + cmd.group_start(); + } + + // We may not pass all crates through to the linker. Some crates may + // appear statically in an existing dylib, meaning we'll pick up all the + // symbols from the dylib. + let src = &codegen_results.crate_info.used_crate_source[&cnum]; + match data[cnum.as_usize() - 1] { + _ if codegen_results.crate_info.profiler_runtime == Some(cnum) => { + add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum); + } + // compiler-builtins are always placed last to ensure that they're + // linked correctly. + _ if codegen_results.crate_info.compiler_builtins == Some(cnum) => { + assert!(compiler_builtins.is_none()); + compiler_builtins = Some(cnum); + } + Linkage::NotLinked | Linkage::IncludedFromDylib => {} + Linkage::Static => { + add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum); + + // Link static native libs with "-bundle" modifier only if the crate they originate from + // is being linked statically to the current crate. If it's linked dynamically + // or is an rlib already included via some other dylib crate, the symbols from + // native libs will have already been included in that dylib. + // + // If -Zlink-native-libraries=false is set, then the assumption is that an + // external build system already has the native dependencies defined, and it + // will provide them to the linker itself. + if sess.opts.unstable_opts.link_native_libraries { + let mut last = (None, NativeLibKind::Unspecified, None); + for lib in &codegen_results.crate_info.native_libraries[&cnum] { + let Some(name) = lib.name else { + continue; + }; + let name = name.as_str(); + if !relevant_lib(sess, lib) { + continue; + } + + // Skip if this library is the same as the last. + last = if (lib.name, lib.kind, lib.verbatim) == last { + continue; + } else { + (lib.name, lib.kind, lib.verbatim) + }; + + match lib.kind { + NativeLibKind::Static { + bundle: Some(false), + whole_archive: Some(true), + } => { + cmd.link_whole_staticlib( + name, + lib.verbatim.unwrap_or(false), + search_path.get_or_init(|| archive_search_paths(sess)), + ); + } + NativeLibKind::Static { + bundle: Some(false), + whole_archive: Some(false) | None, + } => { + cmd.link_staticlib(name, lib.verbatim.unwrap_or(false)); + } + NativeLibKind::LinkArg => { + cmd.arg(name); + } + NativeLibKind::Dylib { .. } + | NativeLibKind::Framework { .. } + | NativeLibKind::Unspecified + | NativeLibKind::RawDylib => {} + NativeLibKind::Static { + bundle: Some(true) | None, + whole_archive: _, + } => {} + } + } + } + } + Linkage::Dynamic => add_dynamic_crate(cmd, sess, &src.dylib.as_ref().unwrap().0), + } + + if group_end == Some(cnum) { + cmd.group_end(); + } + } + + // compiler-builtins are always placed last to ensure that they're + // linked correctly. + // We must always link the `compiler_builtins` crate statically. Even if it + // was already "included" in a dylib (e.g., `libstd` when `-C prefer-dynamic` + // is used) + if let Some(cnum) = compiler_builtins { + add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum); + } + + // Converts a library file-stem into a cc -l argument + fn unlib<'a>(target: &Target, stem: &'a str) -> &'a str { + if stem.starts_with("lib") && !target.is_like_windows { &stem[3..] } else { stem } + } + + // Adds the static "rlib" versions of all crates to the command line. + // There's a bit of magic which happens here specifically related to LTO, + // namely that we remove upstream object files. + // + // When performing LTO, almost(*) all of the bytecode from the upstream + // libraries has already been included in our object file output. As a + // result we need to remove the object files in the upstream libraries so + // the linker doesn't try to include them twice (or whine about duplicate + // symbols). We must continue to include the rest of the rlib, however, as + // it may contain static native libraries which must be linked in. + // + // (*) Crates marked with `#![no_builtins]` don't participate in LTO and + // their bytecode wasn't included. The object files in those libraries must + // still be passed to the linker. + // + // Note, however, that if we're not doing LTO we can just pass the rlib + // blindly to the linker (fast) because it's fine if it's not actually + // included as we're at the end of the dependency chain. + fn add_static_crate<'a>( + cmd: &mut dyn Linker, + sess: &'a Session, + archive_builder_builder: &dyn ArchiveBuilderBuilder, + codegen_results: &CodegenResults, + tmpdir: &Path, + cnum: CrateNum, + ) { + let src = &codegen_results.crate_info.used_crate_source[&cnum]; + let cratepath = &src.rlib.as_ref().unwrap().0; + + let mut link_upstream = |path: &Path| { + cmd.link_rlib(&fix_windows_verbatim_for_gcc(path)); + }; + + // See the comment above in `link_staticlib` and `link_rlib` for why if + // there's a static library that's not relevant we skip all object + // files. + let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; + let skip_native = native_libs.iter().any(|lib| { + matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. }) + && !relevant_lib(sess, lib) + }); + + if (!are_upstream_rust_objects_already_included(sess) + || ignored_for_lto(sess, &codegen_results.crate_info, cnum)) + && !skip_native + { + link_upstream(cratepath); + return; + } + + let dst = tmpdir.join(cratepath.file_name().unwrap()); + let name = cratepath.file_name().unwrap().to_str().unwrap(); + let name = &name[3..name.len() - 5]; // chop off lib/.rlib + + sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| { + let canonical_name = name.replace('-', "_"); + let upstream_rust_objects_already_included = + are_upstream_rust_objects_already_included(sess); + let is_builtins = sess.target.no_builtins + || !codegen_results.crate_info.is_no_builtins.contains(&cnum); + + let mut archive = archive_builder_builder.new_archive_builder(sess); + if let Err(e) = archive.add_archive( + cratepath, + Box::new(move |f| { + if f == METADATA_FILENAME { + return true; + } + + let canonical = f.replace('-', "_"); + + let is_rust_object = + canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f); + + // If we've been requested to skip all native object files + // (those not generated by the rust compiler) then we can skip + // this file. See above for why we may want to do this. + let skip_because_cfg_say_so = skip_native && !is_rust_object; + + // If we're performing LTO and this is a rust-generated object + // file, then we don't need the object file as it's part of the + // LTO module. Note that `#![no_builtins]` is excluded from LTO, + // though, so we let that object file slide. + let skip_because_lto = + upstream_rust_objects_already_included && is_rust_object && is_builtins; + + if skip_because_cfg_say_so || skip_because_lto { + return true; + } + + false + }), + ) { + sess.fatal(&format!("failed to build archive from rlib: {}", e)); + } + if archive.build(&dst) { + link_upstream(&dst); + } + }); + } + + // Same thing as above, but for dynamic crates instead of static crates. + fn add_dynamic_crate(cmd: &mut dyn Linker, sess: &Session, cratepath: &Path) { + // Just need to tell the linker about where the library lives and + // what its name is + let parent = cratepath.parent(); + if let Some(dir) = parent { + cmd.include_path(&fix_windows_verbatim_for_gcc(dir)); + } + let filestem = cratepath.file_stem().unwrap().to_str().unwrap(); + cmd.link_rust_dylib( + &unlib(&sess.target, filestem), + parent.unwrap_or_else(|| Path::new("")), + ); + } +} + +/// Link in all of our upstream crates' native dependencies. Remember that all of these upstream +/// native dependencies are all non-static dependencies. We've got two cases then: +/// +/// 1. The upstream crate is an rlib. In this case we *must* link in the native dependency because +/// the rlib is just an archive. +/// +/// 2. The upstream crate is a dylib. In order to use the dylib, we have to have the dependency +/// present on the system somewhere. Thus, we don't gain a whole lot from not linking in the +/// dynamic dependency to this crate as well. +/// +/// The use case for this is a little subtle. In theory the native dependencies of a crate are +/// purely an implementation detail of the crate itself, but the problem arises with generic and +/// inlined functions. If a generic function calls a native function, then the generic function +/// must be instantiated in the target crate, meaning that the native symbol must also be resolved +/// in the target crate. +fn add_upstream_native_libraries( + cmd: &mut dyn Linker, + sess: &Session, + codegen_results: &CodegenResults, +) { + let mut last = (None, NativeLibKind::Unspecified, None); + for &cnum in &codegen_results.crate_info.used_crates { + for lib in codegen_results.crate_info.native_libraries[&cnum].iter() { + let Some(name) = lib.name else { + continue; + }; + let name = name.as_str(); + if !relevant_lib(sess, &lib) { + continue; + } + + // Skip if this library is the same as the last. + last = if (lib.name, lib.kind, lib.verbatim) == last { + continue; + } else { + (lib.name, lib.kind, lib.verbatim) + }; + + let verbatim = lib.verbatim.unwrap_or(false); + match lib.kind { + NativeLibKind::Dylib { as_needed } => { + cmd.link_dylib(name, verbatim, as_needed.unwrap_or(true)) + } + NativeLibKind::Unspecified => cmd.link_dylib(name, verbatim, true), + NativeLibKind::Framework { as_needed } => { + cmd.link_framework(name, as_needed.unwrap_or(true)) + } + // ignore static native libraries here as we've + // already included them in add_local_native_libraries and + // add_upstream_rust_crates + NativeLibKind::Static { .. } => {} + NativeLibKind::RawDylib | NativeLibKind::LinkArg => {} + } + } + } +} + +fn relevant_lib(sess: &Session, lib: &NativeLib) -> bool { + match lib.cfg { + Some(ref cfg) => rustc_attr::cfg_matches(cfg, &sess.parse_sess, CRATE_NODE_ID, None), + None => true, + } +} + +fn are_upstream_rust_objects_already_included(sess: &Session) -> bool { + match sess.lto() { + config::Lto::Fat => true, + config::Lto::Thin => { + // If we defer LTO to the linker, we haven't run LTO ourselves, so + // any upstream object files have not been copied yet. + !sess.opts.cg.linker_plugin_lto.enabled() + } + config::Lto::No | config::Lto::ThinLocal => false, + } +} + +fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { + let arch = &sess.target.arch; + let os = &sess.target.os; + let llvm_target = &sess.target.llvm_target; + if sess.target.vendor != "apple" + || !matches!(os.as_ref(), "ios" | "tvos" | "watchos") + || (flavor != LinkerFlavor::Gcc && flavor != LinkerFlavor::Lld(LldFlavor::Ld64)) + { + return; + } + let sdk_name = match (arch.as_ref(), os.as_ref()) { + ("aarch64", "tvos") => "appletvos", + ("x86_64", "tvos") => "appletvsimulator", + ("arm", "ios") => "iphoneos", + ("aarch64", "ios") if llvm_target.contains("macabi") => "macosx", + ("aarch64", "ios") if llvm_target.ends_with("-simulator") => "iphonesimulator", + ("aarch64", "ios") => "iphoneos", + ("x86", "ios") => "iphonesimulator", + ("x86_64", "ios") if llvm_target.contains("macabi") => "macosx", + ("x86_64", "ios") => "iphonesimulator", + ("x86_64", "watchos") => "watchsimulator", + ("arm64_32", "watchos") => "watchos", + ("aarch64", "watchos") if llvm_target.ends_with("-simulator") => "watchsimulator", + ("aarch64", "watchos") => "watchos", + ("arm", "watchos") => "watchos", + _ => { + sess.err(&format!("unsupported arch `{}` for os `{}`", arch, os)); + return; + } + }; + let sdk_root = match get_apple_sdk_root(sdk_name) { + Ok(s) => s, + Err(e) => { + sess.err(&e); + return; + } + }; + + match flavor { + LinkerFlavor::Gcc => { + cmd.args(&["-isysroot", &sdk_root, "-Wl,-syslibroot", &sdk_root]); + } + LinkerFlavor::Lld(LldFlavor::Ld64) => { + cmd.args(&["-syslibroot", &sdk_root]); + } + _ => unreachable!(), + } +} + +fn get_apple_sdk_root(sdk_name: &str) -> Result { + // Following what clang does + // (https://github.com/llvm/llvm-project/blob/ + // 296a80102a9b72c3eda80558fb78a3ed8849b341/clang/lib/Driver/ToolChains/Darwin.cpp#L1661-L1678) + // to allow the SDK path to be set. (For clang, xcrun sets + // SDKROOT; for rustc, the user or build system can set it, or we + // can fall back to checking for xcrun on PATH.) + if let Ok(sdkroot) = env::var("SDKROOT") { + let p = Path::new(&sdkroot); + match sdk_name { + // Ignore `SDKROOT` if it's clearly set for the wrong platform. + "appletvos" + if sdkroot.contains("TVSimulator.platform") + || sdkroot.contains("MacOSX.platform") => {} + "appletvsimulator" + if sdkroot.contains("TVOS.platform") || sdkroot.contains("MacOSX.platform") => {} + "iphoneos" + if sdkroot.contains("iPhoneSimulator.platform") + || sdkroot.contains("MacOSX.platform") => {} + "iphonesimulator" + if sdkroot.contains("iPhoneOS.platform") || sdkroot.contains("MacOSX.platform") => { + } + "macosx10.15" + if sdkroot.contains("iPhoneOS.platform") + || sdkroot.contains("iPhoneSimulator.platform") => {} + "watchos" + if sdkroot.contains("WatchSimulator.platform") + || sdkroot.contains("MacOSX.platform") => {} + "watchsimulator" + if sdkroot.contains("WatchOS.platform") || sdkroot.contains("MacOSX.platform") => {} + // Ignore `SDKROOT` if it's not a valid path. + _ if !p.is_absolute() || p == Path::new("/") || !p.exists() => {} + _ => return Ok(sdkroot), + } + } + let res = + Command::new("xcrun").arg("--show-sdk-path").arg("-sdk").arg(sdk_name).output().and_then( + |output| { + if output.status.success() { + Ok(String::from_utf8(output.stdout).unwrap()) + } else { + let error = String::from_utf8(output.stderr); + let error = format!("process exit with error: {}", error.unwrap()); + Err(io::Error::new(io::ErrorKind::Other, &error[..])) + } + }, + ); + + match res { + Ok(output) => Ok(output.trim().to_string()), + Err(e) => Err(format!("failed to get {} SDK path: {}", sdk_name, e)), + } +} + +fn add_gcc_ld_path(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { + if let Some(ld_impl) = sess.opts.unstable_opts.gcc_ld { + if let LinkerFlavor::Gcc = flavor { + match ld_impl { + LdImpl::Lld => { + let tools_path = sess.get_tools_search_paths(false); + let gcc_ld_dir = tools_path + .into_iter() + .map(|p| p.join("gcc-ld")) + .find(|p| { + p.join(if sess.host.is_like_windows { "ld.exe" } else { "ld" }).exists() + }) + .unwrap_or_else(|| sess.fatal("rust-lld (as ld) not found")); + cmd.arg({ + let mut arg = OsString::from("-B"); + arg.push(gcc_ld_dir); + arg + }); + cmd.arg(format!("-Wl,-rustc-lld-flavor={}", sess.target.lld_flavor.as_str())); + } + } + } else { + sess.fatal("option `-Z gcc-ld` is used even though linker flavor is not gcc"); + } + } +} diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs new file mode 100644 index 00000000000..ce51b2e9531 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -0,0 +1,1788 @@ +use super::archive; +use super::command::Command; +use super::symbol_export; +use rustc_span::symbol::sym; + +use std::ffi::{OsStr, OsString}; +use std::fs::{self, File}; +use std::io::prelude::*; +use std::io::{self, BufWriter}; +use std::path::{Path, PathBuf}; +use std::{env, mem, str}; + +use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; +use rustc_middle::middle::dependency_format::Linkage; +use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind}; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::{self, CrateType, DebugInfo, LinkerPluginLto, Lto, OptLevel, Strip}; +use rustc_session::Session; +use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor}; + +use cc::windows_registry; + +/// Disables non-English messages from localized linkers. +/// Such messages may cause issues with text encoding on Windows (#35785) +/// and prevent inspection of linker output in case of errors, which we occasionally do. +/// This should be acceptable because other messages from rustc are in English anyway, +/// and may also be desirable to improve searchability of the linker diagnostics. +pub fn disable_localization(linker: &mut Command) { + // No harm in setting both env vars simultaneously. + // Unix-style linkers. + linker.env("LC_ALL", "C"); + // MSVC's `link.exe`. + linker.env("VSLANG", "1033"); +} + +// The third parameter is for env vars, used on windows to set up the +// path for MSVC to find its DLLs, and gcc to find its bundled +// toolchain +pub fn get_linker<'a>( + sess: &'a Session, + linker: &Path, + flavor: LinkerFlavor, + self_contained: bool, + target_cpu: &'a str, +) -> Box { + let msvc_tool = windows_registry::find_tool(&sess.opts.target_triple.triple(), "link.exe"); + + // If our linker looks like a batch script on Windows then to execute this + // we'll need to spawn `cmd` explicitly. This is primarily done to handle + // emscripten where the linker is `emcc.bat` and needs to be spawned as + // `cmd /c emcc.bat ...`. + // + // This worked historically but is needed manually since #42436 (regression + // was tagged as #42791) and some more info can be found on #44443 for + // emscripten itself. + let mut cmd = match linker.to_str() { + Some(linker) if cfg!(windows) && linker.ends_with(".bat") => Command::bat_script(linker), + _ => match flavor { + LinkerFlavor::Lld(f) => Command::lld(linker, f), + LinkerFlavor::Msvc if sess.opts.cg.linker.is_none() && sess.target.linker.is_none() => { + Command::new(msvc_tool.as_ref().map_or(linker, |t| t.path())) + } + _ => Command::new(linker), + }, + }; + + // UWP apps have API restrictions enforced during Store submissions. + // To comply with the Windows App Certification Kit, + // MSVC needs to link with the Store versions of the runtime libraries (vcruntime, msvcrt, etc). + let t = &sess.target; + if (flavor == LinkerFlavor::Msvc || flavor == LinkerFlavor::Lld(LldFlavor::Link)) + && t.vendor == "uwp" + { + if let Some(ref tool) = msvc_tool { + let original_path = tool.path(); + if let Some(ref root_lib_path) = original_path.ancestors().nth(4) { + let arch = match t.arch.as_ref() { + "x86_64" => Some("x64"), + "x86" => Some("x86"), + "aarch64" => Some("arm64"), + "arm" => Some("arm"), + _ => None, + }; + if let Some(ref a) = arch { + // FIXME: Move this to `fn linker_with_args`. + let mut arg = OsString::from("/LIBPATH:"); + arg.push(format!("{}\\lib\\{}\\store", root_lib_path.display(), a)); + cmd.arg(&arg); + } else { + warn!("arch is not supported"); + } + } else { + warn!("MSVC root path lib location not found"); + } + } else { + warn!("link.exe not found"); + } + } + + // The compiler's sysroot often has some bundled tools, so add it to the + // PATH for the child. + let mut new_path = sess.get_tools_search_paths(self_contained); + let mut msvc_changed_path = false; + if sess.target.is_like_msvc { + if let Some(ref tool) = msvc_tool { + cmd.args(tool.args()); + for &(ref k, ref v) in tool.env() { + if k == "PATH" { + new_path.extend(env::split_paths(v)); + msvc_changed_path = true; + } else { + cmd.env(k, v); + } + } + } + } + + if !msvc_changed_path { + if let Some(path) = env::var_os("PATH") { + new_path.extend(env::split_paths(&path)); + } + } + cmd.env("PATH", env::join_paths(new_path).unwrap()); + + // FIXME: Move `/LIBPATH` addition for uwp targets from the linker construction + // to the linker args construction. + assert!(cmd.get_args().is_empty() || sess.target.vendor == "uwp"); + match flavor { + LinkerFlavor::Lld(LldFlavor::Link) | LinkerFlavor::Msvc => { + Box::new(MsvcLinker { cmd, sess }) as Box + } + LinkerFlavor::Em => Box::new(EmLinker { cmd, sess }) as Box, + LinkerFlavor::Gcc => { + Box::new(GccLinker { cmd, sess, target_cpu, hinted_static: false, is_ld: false }) + as Box + } + + LinkerFlavor::Lld(LldFlavor::Ld) + | LinkerFlavor::Lld(LldFlavor::Ld64) + | LinkerFlavor::Ld => { + Box::new(GccLinker { cmd, sess, target_cpu, hinted_static: false, is_ld: true }) + as Box + } + + LinkerFlavor::Lld(LldFlavor::Wasm) => Box::new(WasmLd::new(cmd, sess)) as Box, + + LinkerFlavor::PtxLinker => Box::new(PtxLinker { cmd, sess }) as Box, + + LinkerFlavor::BpfLinker => Box::new(BpfLinker { cmd, sess }) as Box, + + LinkerFlavor::L4Bender => Box::new(L4Bender::new(cmd, sess)) as Box, + } +} + +/// Linker abstraction used by `back::link` to build up the command to invoke a +/// linker. +/// +/// This trait is the total list of requirements needed by `back::link` and +/// represents the meaning of each option being passed down. This trait is then +/// used to dispatch on whether a GNU-like linker (generally `ld.exe`) or an +/// MSVC linker (e.g., `link.exe`) is being used. +pub trait Linker { + fn cmd(&mut self) -> &mut Command; + fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path); + fn link_dylib(&mut self, lib: &str, verbatim: bool, as_needed: bool); + fn link_rust_dylib(&mut self, lib: &str, path: &Path); + fn link_framework(&mut self, framework: &str, as_needed: bool); + fn link_staticlib(&mut self, lib: &str, verbatim: bool); + fn link_rlib(&mut self, lib: &Path); + fn link_whole_rlib(&mut self, lib: &Path); + fn link_whole_staticlib(&mut self, lib: &str, verbatim: bool, search_path: &[PathBuf]); + fn include_path(&mut self, path: &Path); + fn framework_path(&mut self, path: &Path); + fn output_filename(&mut self, path: &Path); + fn add_object(&mut self, path: &Path); + fn gc_sections(&mut self, keep_metadata: bool); + fn no_gc_sections(&mut self); + fn full_relro(&mut self); + fn partial_relro(&mut self); + fn no_relro(&mut self); + fn optimize(&mut self); + fn pgo_gen(&mut self); + fn control_flow_guard(&mut self); + fn debuginfo(&mut self, strip: Strip, natvis_debugger_visualizers: &[PathBuf]); + fn no_crt_objects(&mut self); + fn no_default_libraries(&mut self); + fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]); + fn subsystem(&mut self, subsystem: &str); + fn group_start(&mut self); + fn group_end(&mut self); + fn linker_plugin_lto(&mut self); + fn add_eh_frame_header(&mut self) {} + fn add_no_exec(&mut self) {} + fn add_as_needed(&mut self) {} + fn reset_per_library_state(&mut self) {} +} + +impl dyn Linker + '_ { + pub fn arg(&mut self, arg: impl AsRef) { + self.cmd().arg(arg); + } + + pub fn args(&mut self, args: impl IntoIterator>) { + self.cmd().args(args); + } + + pub fn take_cmd(&mut self) -> Command { + mem::replace(self.cmd(), Command::new("")) + } +} + +pub struct GccLinker<'a> { + cmd: Command, + sess: &'a Session, + target_cpu: &'a str, + hinted_static: bool, // Keeps track of the current hinting mode. + // Link as ld + is_ld: bool, +} + +impl<'a> GccLinker<'a> { + /// Passes an argument directly to the linker. + /// + /// When the linker is not ld-like such as when using a compiler as a linker, the argument is + /// prepended by `-Wl,`. + fn linker_arg(&mut self, arg: impl AsRef) -> &mut Self { + self.linker_args(&[arg]); + self + } + + /// Passes a series of arguments directly to the linker. + /// + /// When the linker is ld-like, the arguments are simply appended to the command. When the + /// linker is not ld-like such as when using a compiler as a linker, the arguments are joined by + /// commas to form an argument that is then prepended with `-Wl`. In this situation, only a + /// single argument is appended to the command to ensure that the order of the arguments is + /// preserved by the compiler. + fn linker_args(&mut self, args: &[impl AsRef]) -> &mut Self { + if self.is_ld { + args.into_iter().for_each(|a| { + self.cmd.arg(a); + }); + } else { + if !args.is_empty() { + let mut s = OsString::from("-Wl"); + for a in args { + s.push(","); + s.push(a); + } + self.cmd.arg(s); + } + } + self + } + + fn takes_hints(&self) -> bool { + // Really this function only returns true if the underlying linker + // configured for a compiler is binutils `ld.bfd` and `ld.gold`. We + // don't really have a foolproof way to detect that, so rule out some + // platforms where currently this is guaranteed to *not* be the case: + // + // * On OSX they have their own linker, not binutils' + // * For WebAssembly the only functional linker is LLD, which doesn't + // support hint flags + !self.sess.target.is_like_osx && !self.sess.target.is_like_wasm + } + + // Some platforms take hints about whether a library is static or dynamic. + // For those that support this, we ensure we pass the option if the library + // was flagged "static" (most defaults are dynamic) to ensure that if + // libfoo.a and libfoo.so both exist that the right one is chosen. + fn hint_static(&mut self) { + if !self.takes_hints() { + return; + } + if !self.hinted_static { + self.linker_arg("-Bstatic"); + self.hinted_static = true; + } + } + + fn hint_dynamic(&mut self) { + if !self.takes_hints() { + return; + } + if self.hinted_static { + self.linker_arg("-Bdynamic"); + self.hinted_static = false; + } + } + + fn push_linker_plugin_lto_args(&mut self, plugin_path: Option<&OsStr>) { + if let Some(plugin_path) = plugin_path { + let mut arg = OsString::from("-plugin="); + arg.push(plugin_path); + self.linker_arg(&arg); + } + + let opt_level = match self.sess.opts.optimize { + config::OptLevel::No => "O0", + config::OptLevel::Less => "O1", + config::OptLevel::Default | config::OptLevel::Size | config::OptLevel::SizeMin => "O2", + config::OptLevel::Aggressive => "O3", + }; + + if let Some(path) = &self.sess.opts.unstable_opts.profile_sample_use { + self.linker_arg(&format!("-plugin-opt=sample-profile={}", path.display())); + }; + self.linker_args(&[ + &format!("-plugin-opt={}", opt_level), + &format!("-plugin-opt=mcpu={}", self.target_cpu), + ]); + } + + fn build_dylib(&mut self, out_filename: &Path) { + // On mac we need to tell the linker to let this library be rpathed + if self.sess.target.is_like_osx { + if !self.is_ld { + self.cmd.arg("-dynamiclib"); + } + + self.linker_arg("-dylib"); + + // Note that the `osx_rpath_install_name` option here is a hack + // purely to support rustbuild right now, we should get a more + // principled solution at some point to force the compiler to pass + // the right `-Wl,-install_name` with an `@rpath` in it. + if self.sess.opts.cg.rpath || self.sess.opts.unstable_opts.osx_rpath_install_name { + let mut rpath = OsString::from("@rpath/"); + rpath.push(out_filename.file_name().unwrap()); + self.linker_args(&[OsString::from("-install_name"), rpath]); + } + } else { + self.cmd.arg("-shared"); + if self.sess.target.is_like_windows { + // The output filename already contains `dll_suffix` so + // the resulting import library will have a name in the + // form of libfoo.dll.a + let implib_name = + out_filename.file_name().and_then(|file| file.to_str()).map(|file| { + format!( + "{}{}{}", + self.sess.target.staticlib_prefix, + file, + self.sess.target.staticlib_suffix + ) + }); + if let Some(implib_name) = implib_name { + let implib = out_filename.parent().map(|dir| dir.join(&implib_name)); + if let Some(implib) = implib { + self.linker_arg(&format!("--out-implib={}", (*implib).to_str().unwrap())); + } + } + } + } + } +} + +impl<'a> Linker for GccLinker<'a> { + fn cmd(&mut self) -> &mut Command { + &mut self.cmd + } + + fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path) { + match output_kind { + LinkOutputKind::DynamicNoPicExe => { + if !self.is_ld && self.sess.target.linker_is_gnu { + self.cmd.arg("-no-pie"); + } + } + LinkOutputKind::DynamicPicExe => { + // noop on windows w/ gcc & ld, error w/ lld + if !self.sess.target.is_like_windows { + // `-pie` works for both gcc wrapper and ld. + self.cmd.arg("-pie"); + } + } + LinkOutputKind::StaticNoPicExe => { + // `-static` works for both gcc wrapper and ld. + self.cmd.arg("-static"); + if !self.is_ld && self.sess.target.linker_is_gnu { + self.cmd.arg("-no-pie"); + } + } + LinkOutputKind::StaticPicExe => { + if !self.is_ld { + // Note that combination `-static -pie` doesn't work as expected + // for the gcc wrapper, `-static` in that case suppresses `-pie`. + self.cmd.arg("-static-pie"); + } else { + // `--no-dynamic-linker` and `-z text` are not strictly necessary for producing + // a static pie, but currently passed because gcc and clang pass them. + // The former suppresses the `INTERP` ELF header specifying dynamic linker, + // which is otherwise implicitly injected by ld (but not lld). + // The latter doesn't change anything, only ensures that everything is pic. + self.cmd.args(&["-static", "-pie", "--no-dynamic-linker", "-z", "text"]); + } + } + LinkOutputKind::DynamicDylib => self.build_dylib(out_filename), + LinkOutputKind::StaticDylib => { + self.cmd.arg("-static"); + self.build_dylib(out_filename); + } + LinkOutputKind::WasiReactorExe => { + self.linker_args(&["--entry", "_initialize"]); + } + } + // VxWorks compiler driver introduced `--static-crt` flag specifically for rustc, + // it switches linking for libc and similar system libraries to static without using + // any `#[link]` attributes in the `libc` crate, see #72782 for details. + // FIXME: Switch to using `#[link]` attributes in the `libc` crate + // similarly to other targets. + if self.sess.target.os == "vxworks" + && matches!( + output_kind, + LinkOutputKind::StaticNoPicExe + | LinkOutputKind::StaticPicExe + | LinkOutputKind::StaticDylib + ) + { + self.cmd.arg("--static-crt"); + } + } + + fn link_dylib(&mut self, lib: &str, verbatim: bool, as_needed: bool) { + if self.sess.target.os == "illumos" && lib == "c" { + // libc will be added via late_link_args on illumos so that it will + // appear last in the library search order. + // FIXME: This should be replaced by a more complete and generic + // mechanism for controlling the order of library arguments passed + // to the linker. + return; + } + if !as_needed { + if self.sess.target.is_like_osx { + // FIXME(81490): ld64 doesn't support these flags but macOS 11 + // has -needed-l{} / -needed_library {} + // but we have no way to detect that here. + self.sess.warn("`as-needed` modifier not implemented yet for ld64"); + } else if self.sess.target.linker_is_gnu && !self.sess.target.is_like_windows { + self.linker_arg("--no-as-needed"); + } else { + self.sess.warn("`as-needed` modifier not supported for current linker"); + } + } + self.hint_dynamic(); + self.cmd.arg(format!("-l{}{}", if verbatim { ":" } else { "" }, lib)); + if !as_needed { + if self.sess.target.is_like_osx { + // See above FIXME comment + } else if self.sess.target.linker_is_gnu && !self.sess.target.is_like_windows { + self.linker_arg("--as-needed"); + } + } + } + fn link_staticlib(&mut self, lib: &str, verbatim: bool) { + self.hint_static(); + self.cmd.arg(format!("-l{}{}", if verbatim { ":" } else { "" }, lib)); + } + fn link_rlib(&mut self, lib: &Path) { + self.hint_static(); + self.cmd.arg(lib); + } + fn include_path(&mut self, path: &Path) { + self.cmd.arg("-L").arg(path); + } + fn framework_path(&mut self, path: &Path) { + self.cmd.arg("-F").arg(path); + } + fn output_filename(&mut self, path: &Path) { + self.cmd.arg("-o").arg(path); + } + fn add_object(&mut self, path: &Path) { + self.cmd.arg(path); + } + fn full_relro(&mut self) { + self.linker_args(&["-zrelro", "-znow"]); + } + fn partial_relro(&mut self) { + self.linker_arg("-zrelro"); + } + fn no_relro(&mut self) { + self.linker_arg("-znorelro"); + } + + fn link_rust_dylib(&mut self, lib: &str, _path: &Path) { + self.hint_dynamic(); + self.cmd.arg(format!("-l{}", lib)); + } + + fn link_framework(&mut self, framework: &str, as_needed: bool) { + self.hint_dynamic(); + if !as_needed { + // FIXME(81490): ld64 as of macOS 11 supports the -needed_framework + // flag but we have no way to detect that here. + // self.cmd.arg("-needed_framework").arg(framework); + self.sess.warn("`as-needed` modifier not implemented yet for ld64"); + } + self.cmd.arg("-framework").arg(framework); + } + + // Here we explicitly ask that the entire archive is included into the + // result artifact. For more details see #15460, but the gist is that + // the linker will strip away any unused objects in the archive if we + // don't otherwise explicitly reference them. This can occur for + // libraries which are just providing bindings, libraries with generic + // functions, etc. + fn link_whole_staticlib(&mut self, lib: &str, verbatim: bool, search_path: &[PathBuf]) { + self.hint_static(); + let target = &self.sess.target; + if !target.is_like_osx { + self.linker_arg("--whole-archive").cmd.arg(format!( + "-l{}{}", + if verbatim { ":" } else { "" }, + lib + )); + self.linker_arg("--no-whole-archive"); + } else { + // -force_load is the macOS equivalent of --whole-archive, but it + // involves passing the full path to the library to link. + self.linker_arg("-force_load"); + let lib = archive::find_library(lib, verbatim, search_path, &self.sess); + self.linker_arg(&lib); + } + } + + fn link_whole_rlib(&mut self, lib: &Path) { + self.hint_static(); + if self.sess.target.is_like_osx { + self.linker_arg("-force_load"); + self.linker_arg(&lib); + } else { + self.linker_arg("--whole-archive").cmd.arg(lib); + self.linker_arg("--no-whole-archive"); + } + } + + fn gc_sections(&mut self, keep_metadata: bool) { + // The dead_strip option to the linker specifies that functions and data + // unreachable by the entry point will be removed. This is quite useful + // with Rust's compilation model of compiling libraries at a time into + // one object file. For example, this brings hello world from 1.7MB to + // 458K. + // + // Note that this is done for both executables and dynamic libraries. We + // won't get much benefit from dylibs because LLVM will have already + // stripped away as much as it could. This has not been seen to impact + // link times negatively. + // + // -dead_strip can't be part of the pre_link_args because it's also used + // for partial linking when using multiple codegen units (-r). So we + // insert it here. + if self.sess.target.is_like_osx { + self.linker_arg("-dead_strip"); + + // If we're building a dylib, we don't use --gc-sections because LLVM + // has already done the best it can do, and we also don't want to + // eliminate the metadata. If we're building an executable, however, + // --gc-sections drops the size of hello world from 1.8MB to 597K, a 67% + // reduction. + } else if (self.sess.target.linker_is_gnu || self.sess.target.is_like_wasm) + && !keep_metadata + { + self.linker_arg("--gc-sections"); + } + } + + fn no_gc_sections(&mut self) { + if self.sess.target.linker_is_gnu || self.sess.target.is_like_wasm { + self.linker_arg("--no-gc-sections"); + } + } + + fn optimize(&mut self) { + if !self.sess.target.linker_is_gnu && !self.sess.target.is_like_wasm { + return; + } + + // GNU-style linkers support optimization with -O. GNU ld doesn't + // need a numeric argument, but other linkers do. + if self.sess.opts.optimize == config::OptLevel::Default + || self.sess.opts.optimize == config::OptLevel::Aggressive + { + self.linker_arg("-O1"); + } + } + + fn pgo_gen(&mut self) { + if !self.sess.target.linker_is_gnu { + return; + } + + // If we're doing PGO generation stuff and on a GNU-like linker, use the + // "-u" flag to properly pull in the profiler runtime bits. + // + // This is because LLVM otherwise won't add the needed initialization + // for us on Linux (though the extra flag should be harmless if it + // does). + // + // See https://reviews.llvm.org/D14033 and https://reviews.llvm.org/D14030. + // + // Though it may be worth to try to revert those changes upstream, since + // the overhead of the initialization should be minor. + self.cmd.arg("-u"); + self.cmd.arg("__llvm_profile_runtime"); + } + + fn control_flow_guard(&mut self) {} + + fn debuginfo(&mut self, strip: Strip, _: &[PathBuf]) { + // MacOS linker doesn't support stripping symbols directly anymore. + if self.sess.target.is_like_osx { + return; + } + + match strip { + Strip::None => {} + Strip::Debuginfo => { + self.linker_arg("--strip-debug"); + } + Strip::Symbols => { + self.linker_arg("--strip-all"); + } + } + } + + fn no_crt_objects(&mut self) { + if !self.is_ld { + self.cmd.arg("-nostartfiles"); + } + } + + fn no_default_libraries(&mut self) { + if !self.is_ld { + self.cmd.arg("-nodefaultlibs"); + } + } + + fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]) { + // Symbol visibility in object files typically takes care of this. + if crate_type == CrateType::Executable { + let should_export_executable_symbols = + self.sess.opts.unstable_opts.export_executable_symbols; + if self.sess.target.override_export_symbols.is_none() + && !should_export_executable_symbols + { + return; + } + } + + // We manually create a list of exported symbols to ensure we don't expose any more. + // The object files have far more public symbols than we actually want to export, + // so we hide them all here. + + if !self.sess.target.limit_rdylib_exports { + return; + } + + // FIXME(#99978) hide #[no_mangle] symbols for proc-macros + + let is_windows = self.sess.target.is_like_windows; + let path = tmpdir.join(if is_windows { "list.def" } else { "list" }); + + debug!("EXPORTED SYMBOLS:"); + + if self.sess.target.is_like_osx { + // Write a plain, newline-separated list of symbols + let res: io::Result<()> = try { + let mut f = BufWriter::new(File::create(&path)?); + for sym in symbols { + debug!(" _{}", sym); + writeln!(f, "_{}", sym)?; + } + }; + if let Err(e) = res { + self.sess.fatal(&format!("failed to write lib.def file: {}", e)); + } + } else if is_windows { + let res: io::Result<()> = try { + let mut f = BufWriter::new(File::create(&path)?); + + // .def file similar to MSVC one but without LIBRARY section + // because LD doesn't like when it's empty + writeln!(f, "EXPORTS")?; + for symbol in symbols { + debug!(" _{}", symbol); + writeln!(f, " {}", symbol)?; + } + }; + if let Err(e) = res { + self.sess.fatal(&format!("failed to write list.def file: {}", e)); + } + } else { + // Write an LD version script + let res: io::Result<()> = try { + let mut f = BufWriter::new(File::create(&path)?); + writeln!(f, "{{")?; + if !symbols.is_empty() { + writeln!(f, " global:")?; + for sym in symbols { + debug!(" {};", sym); + writeln!(f, " {};", sym)?; + } + } + writeln!(f, "\n local:\n *;\n}};")?; + }; + if let Err(e) = res { + self.sess.fatal(&format!("failed to write version script: {}", e)); + } + } + + if self.sess.target.is_like_osx { + self.linker_args(&[OsString::from("-exported_symbols_list"), path.into()]); + } else if self.sess.target.is_like_solaris { + self.linker_args(&[OsString::from("-M"), path.into()]); + } else { + if is_windows { + self.linker_arg(path); + } else { + let mut arg = OsString::from("--version-script="); + arg.push(path); + self.linker_arg(arg); + } + } + } + + fn subsystem(&mut self, subsystem: &str) { + self.linker_arg("--subsystem"); + self.linker_arg(&subsystem); + } + + fn reset_per_library_state(&mut self) { + self.hint_dynamic(); // Reset to default before returning the composed command line. + } + + fn group_start(&mut self) { + if self.takes_hints() { + self.linker_arg("--start-group"); + } + } + + fn group_end(&mut self) { + if self.takes_hints() { + self.linker_arg("--end-group"); + } + } + + fn linker_plugin_lto(&mut self) { + match self.sess.opts.cg.linker_plugin_lto { + LinkerPluginLto::Disabled => { + // Nothing to do + } + LinkerPluginLto::LinkerPluginAuto => { + self.push_linker_plugin_lto_args(None); + } + LinkerPluginLto::LinkerPlugin(ref path) => { + self.push_linker_plugin_lto_args(Some(path.as_os_str())); + } + } + } + + // Add the `GNU_EH_FRAME` program header which is required to locate unwinding information. + // Some versions of `gcc` add it implicitly, some (e.g. `musl-gcc`) don't, + // so we just always add it. + fn add_eh_frame_header(&mut self) { + self.linker_arg("--eh-frame-hdr"); + } + + fn add_no_exec(&mut self) { + if self.sess.target.is_like_windows { + self.linker_arg("--nxcompat"); + } else if self.sess.target.linker_is_gnu { + self.linker_arg("-znoexecstack"); + } + } + + fn add_as_needed(&mut self) { + if self.sess.target.linker_is_gnu && !self.sess.target.is_like_windows { + self.linker_arg("--as-needed"); + } else if self.sess.target.is_like_solaris { + // -z ignore is the Solaris equivalent to the GNU ld --as-needed option + self.linker_args(&["-z", "ignore"]); + } + } +} + +pub struct MsvcLinker<'a> { + cmd: Command, + sess: &'a Session, +} + +impl<'a> Linker for MsvcLinker<'a> { + fn cmd(&mut self) -> &mut Command { + &mut self.cmd + } + + fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path) { + match output_kind { + LinkOutputKind::DynamicNoPicExe + | LinkOutputKind::DynamicPicExe + | LinkOutputKind::StaticNoPicExe + | LinkOutputKind::StaticPicExe => {} + LinkOutputKind::DynamicDylib | LinkOutputKind::StaticDylib => { + self.cmd.arg("/DLL"); + let mut arg: OsString = "/IMPLIB:".into(); + arg.push(out_filename.with_extension("dll.lib")); + self.cmd.arg(arg); + } + LinkOutputKind::WasiReactorExe => { + panic!("can't link as reactor on non-wasi target"); + } + } + } + + fn link_rlib(&mut self, lib: &Path) { + self.cmd.arg(lib); + } + fn add_object(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn gc_sections(&mut self, _keep_metadata: bool) { + // MSVC's ICF (Identical COMDAT Folding) link optimization is + // slow for Rust and thus we disable it by default when not in + // optimization build. + if self.sess.opts.optimize != config::OptLevel::No { + self.cmd.arg("/OPT:REF,ICF"); + } else { + // It is necessary to specify NOICF here, because /OPT:REF + // implies ICF by default. + self.cmd.arg("/OPT:REF,NOICF"); + } + } + + fn no_gc_sections(&mut self) { + self.cmd.arg("/OPT:NOREF,NOICF"); + } + + fn link_dylib(&mut self, lib: &str, verbatim: bool, _as_needed: bool) { + self.cmd.arg(format!("{}{}", lib, if verbatim { "" } else { ".lib" })); + } + + fn link_rust_dylib(&mut self, lib: &str, path: &Path) { + // When producing a dll, the MSVC linker may not actually emit a + // `foo.lib` file if the dll doesn't actually export any symbols, so we + // check to see if the file is there and just omit linking to it if it's + // not present. + let name = format!("{}.dll.lib", lib); + if path.join(&name).exists() { + self.cmd.arg(name); + } + } + + fn link_staticlib(&mut self, lib: &str, verbatim: bool) { + self.cmd.arg(format!("{}{}", lib, if verbatim { "" } else { ".lib" })); + } + + fn full_relro(&mut self) { + // noop + } + + fn partial_relro(&mut self) { + // noop + } + + fn no_relro(&mut self) { + // noop + } + + fn no_crt_objects(&mut self) { + // noop + } + + fn no_default_libraries(&mut self) { + self.cmd.arg("/NODEFAULTLIB"); + } + + fn include_path(&mut self, path: &Path) { + let mut arg = OsString::from("/LIBPATH:"); + arg.push(path); + self.cmd.arg(&arg); + } + + fn output_filename(&mut self, path: &Path) { + let mut arg = OsString::from("/OUT:"); + arg.push(path); + self.cmd.arg(&arg); + } + + fn framework_path(&mut self, _path: &Path) { + bug!("frameworks are not supported on windows") + } + fn link_framework(&mut self, _framework: &str, _as_needed: bool) { + bug!("frameworks are not supported on windows") + } + + fn link_whole_staticlib(&mut self, lib: &str, verbatim: bool, _search_path: &[PathBuf]) { + self.cmd.arg(format!("/WHOLEARCHIVE:{}{}", lib, if verbatim { "" } else { ".lib" })); + } + fn link_whole_rlib(&mut self, path: &Path) { + let mut arg = OsString::from("/WHOLEARCHIVE:"); + arg.push(path); + self.cmd.arg(arg); + } + fn optimize(&mut self) { + // Needs more investigation of `/OPT` arguments + } + + fn pgo_gen(&mut self) { + // Nothing needed here. + } + + fn control_flow_guard(&mut self) { + self.cmd.arg("/guard:cf"); + } + + fn debuginfo(&mut self, strip: Strip, natvis_debugger_visualizers: &[PathBuf]) { + match strip { + Strip::None => { + // This will cause the Microsoft linker to generate a PDB file + // from the CodeView line tables in the object files. + self.cmd.arg("/DEBUG"); + + // This will cause the Microsoft linker to embed .natvis info into the PDB file + let natvis_dir_path = self.sess.sysroot.join("lib\\rustlib\\etc"); + if let Ok(natvis_dir) = fs::read_dir(&natvis_dir_path) { + for entry in natvis_dir { + match entry { + Ok(entry) => { + let path = entry.path(); + if path.extension() == Some("natvis".as_ref()) { + let mut arg = OsString::from("/NATVIS:"); + arg.push(path); + self.cmd.arg(arg); + } + } + Err(err) => { + self.sess + .warn(&format!("error enumerating natvis directory: {}", err)); + } + } + } + } + + // This will cause the Microsoft linker to embed .natvis info for all crates into the PDB file + for path in natvis_debugger_visualizers { + let mut arg = OsString::from("/NATVIS:"); + arg.push(path); + self.cmd.arg(arg); + } + } + Strip::Debuginfo | Strip::Symbols => { + self.cmd.arg("/DEBUG:NONE"); + } + } + } + + // Currently the compiler doesn't use `dllexport` (an LLVM attribute) to + // export symbols from a dynamic library. When building a dynamic library, + // however, we're going to want some symbols exported, so this function + // generates a DEF file which lists all the symbols. + // + // The linker will read this `*.def` file and export all the symbols from + // the dynamic library. Note that this is not as simple as just exporting + // all the symbols in the current crate (as specified by `codegen.reachable`) + // but rather we also need to possibly export the symbols of upstream + // crates. Upstream rlibs may be linked statically to this dynamic library, + // in which case they may continue to transitively be used and hence need + // their symbols exported. + fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]) { + // Symbol visibility takes care of this typically + if crate_type == CrateType::Executable { + let should_export_executable_symbols = + self.sess.opts.unstable_opts.export_executable_symbols; + if !should_export_executable_symbols { + return; + } + } + + let path = tmpdir.join("lib.def"); + let res: io::Result<()> = try { + let mut f = BufWriter::new(File::create(&path)?); + + // Start off with the standard module name header and then go + // straight to exports. + writeln!(f, "LIBRARY")?; + writeln!(f, "EXPORTS")?; + for symbol in symbols { + debug!(" _{}", symbol); + writeln!(f, " {}", symbol)?; + } + }; + if let Err(e) = res { + self.sess.fatal(&format!("failed to write lib.def file: {}", e)); + } + let mut arg = OsString::from("/DEF:"); + arg.push(path); + self.cmd.arg(&arg); + } + + fn subsystem(&mut self, subsystem: &str) { + // Note that previous passes of the compiler validated this subsystem, + // so we just blindly pass it to the linker. + self.cmd.arg(&format!("/SUBSYSTEM:{}", subsystem)); + + // Windows has two subsystems we're interested in right now, the console + // and windows subsystems. These both implicitly have different entry + // points (starting symbols). The console entry point starts with + // `mainCRTStartup` and the windows entry point starts with + // `WinMainCRTStartup`. These entry points, defined in system libraries, + // will then later probe for either `main` or `WinMain`, respectively to + // start the application. + // + // In Rust we just always generate a `main` function so we want control + // to always start there, so we force the entry point on the windows + // subsystem to be `mainCRTStartup` to get everything booted up + // correctly. + // + // For more information see RFC #1665 + if subsystem == "windows" { + self.cmd.arg("/ENTRY:mainCRTStartup"); + } + } + + // MSVC doesn't need group indicators + fn group_start(&mut self) {} + fn group_end(&mut self) {} + + fn linker_plugin_lto(&mut self) { + // Do nothing + } + + fn add_no_exec(&mut self) { + self.cmd.arg("/NXCOMPAT"); + } +} + +pub struct EmLinker<'a> { + cmd: Command, + sess: &'a Session, +} + +impl<'a> Linker for EmLinker<'a> { + fn cmd(&mut self) -> &mut Command { + &mut self.cmd + } + + fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {} + + fn include_path(&mut self, path: &Path) { + self.cmd.arg("-L").arg(path); + } + + fn link_staticlib(&mut self, lib: &str, _verbatim: bool) { + self.cmd.arg("-l").arg(lib); + } + + fn output_filename(&mut self, path: &Path) { + self.cmd.arg("-o").arg(path); + } + + fn add_object(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn link_dylib(&mut self, lib: &str, verbatim: bool, _as_needed: bool) { + // Emscripten always links statically + self.link_staticlib(lib, verbatim); + } + + fn link_whole_staticlib(&mut self, lib: &str, verbatim: bool, _search_path: &[PathBuf]) { + // not supported? + self.link_staticlib(lib, verbatim); + } + + fn link_whole_rlib(&mut self, lib: &Path) { + // not supported? + self.link_rlib(lib); + } + + fn link_rust_dylib(&mut self, lib: &str, _path: &Path) { + self.link_dylib(lib, false, true); + } + + fn link_rlib(&mut self, lib: &Path) { + self.add_object(lib); + } + + fn full_relro(&mut self) { + // noop + } + + fn partial_relro(&mut self) { + // noop + } + + fn no_relro(&mut self) { + // noop + } + + fn framework_path(&mut self, _path: &Path) { + bug!("frameworks are not supported on Emscripten") + } + + fn link_framework(&mut self, _framework: &str, _as_needed: bool) { + bug!("frameworks are not supported on Emscripten") + } + + fn gc_sections(&mut self, _keep_metadata: bool) { + // noop + } + + fn no_gc_sections(&mut self) { + // noop + } + + fn optimize(&mut self) { + // Emscripten performs own optimizations + self.cmd.arg(match self.sess.opts.optimize { + OptLevel::No => "-O0", + OptLevel::Less => "-O1", + OptLevel::Default => "-O2", + OptLevel::Aggressive => "-O3", + OptLevel::Size => "-Os", + OptLevel::SizeMin => "-Oz", + }); + } + + fn pgo_gen(&mut self) { + // noop, but maybe we need something like the gnu linker? + } + + fn control_flow_guard(&mut self) {} + + fn debuginfo(&mut self, _strip: Strip, _: &[PathBuf]) { + // Preserve names or generate source maps depending on debug info + self.cmd.arg(match self.sess.opts.debuginfo { + DebugInfo::None => "-g0", + DebugInfo::Limited => "--profiling-funcs", + DebugInfo::Full => "-g", + }); + } + + fn no_crt_objects(&mut self) {} + + fn no_default_libraries(&mut self) { + self.cmd.arg("-nodefaultlibs"); + } + + fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) { + debug!("EXPORTED SYMBOLS:"); + + self.cmd.arg("-s"); + + let mut arg = OsString::from("EXPORTED_FUNCTIONS="); + let encoded = serde_json::to_string( + &symbols.iter().map(|sym| "_".to_owned() + sym).collect::>(), + ) + .unwrap(); + debug!("{}", encoded); + + arg.push(encoded); + + self.cmd.arg(arg); + } + + fn subsystem(&mut self, _subsystem: &str) { + // noop + } + + // Appears not necessary on Emscripten + fn group_start(&mut self) {} + fn group_end(&mut self) {} + + fn linker_plugin_lto(&mut self) { + // Do nothing + } +} + +pub struct WasmLd<'a> { + cmd: Command, + sess: &'a Session, +} + +impl<'a> WasmLd<'a> { + fn new(mut cmd: Command, sess: &'a Session) -> WasmLd<'a> { + // If the atomics feature is enabled for wasm then we need a whole bunch + // of flags: + // + // * `--shared-memory` - the link won't even succeed without this, flags + // the one linear memory as `shared` + // + // * `--max-memory=1G` - when specifying a shared memory this must also + // be specified. We conservatively choose 1GB but users should be able + // to override this with `-C link-arg`. + // + // * `--import-memory` - it doesn't make much sense for memory to be + // exported in a threaded module because typically you're + // sharing memory and instantiating the module multiple times. As a + // result if it were exported then we'd just have no sharing. + // + // * `--export=__wasm_init_memory` - when using `--passive-segments` the + // linker will synthesize this function, and so we need to make sure + // that our usage of `--export` below won't accidentally cause this + // function to get deleted. + // + // * `--export=*tls*` - when `#[thread_local]` symbols are used these + // symbols are how the TLS segments are initialized and configured. + if sess.target_features.contains(&sym::atomics) { + cmd.arg("--shared-memory"); + cmd.arg("--max-memory=1073741824"); + cmd.arg("--import-memory"); + cmd.arg("--export=__wasm_init_memory"); + cmd.arg("--export=__wasm_init_tls"); + cmd.arg("--export=__tls_size"); + cmd.arg("--export=__tls_align"); + cmd.arg("--export=__tls_base"); + } + WasmLd { cmd, sess } + } +} + +impl<'a> Linker for WasmLd<'a> { + fn cmd(&mut self) -> &mut Command { + &mut self.cmd + } + + fn set_output_kind(&mut self, output_kind: LinkOutputKind, _out_filename: &Path) { + match output_kind { + LinkOutputKind::DynamicNoPicExe + | LinkOutputKind::DynamicPicExe + | LinkOutputKind::StaticNoPicExe + | LinkOutputKind::StaticPicExe => {} + LinkOutputKind::DynamicDylib | LinkOutputKind::StaticDylib => { + self.cmd.arg("--no-entry"); + } + LinkOutputKind::WasiReactorExe => { + self.cmd.arg("--entry"); + self.cmd.arg("_initialize"); + } + } + } + + fn link_dylib(&mut self, lib: &str, _verbatim: bool, _as_needed: bool) { + self.cmd.arg("-l").arg(lib); + } + + fn link_staticlib(&mut self, lib: &str, _verbatim: bool) { + self.cmd.arg("-l").arg(lib); + } + + fn link_rlib(&mut self, lib: &Path) { + self.cmd.arg(lib); + } + + fn include_path(&mut self, path: &Path) { + self.cmd.arg("-L").arg(path); + } + + fn framework_path(&mut self, _path: &Path) { + panic!("frameworks not supported") + } + + fn output_filename(&mut self, path: &Path) { + self.cmd.arg("-o").arg(path); + } + + fn add_object(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn full_relro(&mut self) {} + + fn partial_relro(&mut self) {} + + fn no_relro(&mut self) {} + + fn link_rust_dylib(&mut self, lib: &str, _path: &Path) { + self.cmd.arg("-l").arg(lib); + } + + fn link_framework(&mut self, _framework: &str, _as_needed: bool) { + panic!("frameworks not supported") + } + + fn link_whole_staticlib(&mut self, lib: &str, _verbatim: bool, _search_path: &[PathBuf]) { + self.cmd.arg("-l").arg(lib); + } + + fn link_whole_rlib(&mut self, lib: &Path) { + self.cmd.arg(lib); + } + + fn gc_sections(&mut self, _keep_metadata: bool) { + self.cmd.arg("--gc-sections"); + } + + fn no_gc_sections(&mut self) { + self.cmd.arg("--no-gc-sections"); + } + + fn optimize(&mut self) { + self.cmd.arg(match self.sess.opts.optimize { + OptLevel::No => "-O0", + OptLevel::Less => "-O1", + OptLevel::Default => "-O2", + OptLevel::Aggressive => "-O3", + // Currently LLD doesn't support `Os` and `Oz`, so pass through `O2` + // instead. + OptLevel::Size => "-O2", + OptLevel::SizeMin => "-O2", + }); + } + + fn pgo_gen(&mut self) {} + + fn debuginfo(&mut self, strip: Strip, _: &[PathBuf]) { + match strip { + Strip::None => {} + Strip::Debuginfo => { + self.cmd.arg("--strip-debug"); + } + Strip::Symbols => { + self.cmd.arg("--strip-all"); + } + } + } + + fn control_flow_guard(&mut self) {} + + fn no_crt_objects(&mut self) {} + + fn no_default_libraries(&mut self) {} + + fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) { + for sym in symbols { + self.cmd.arg("--export").arg(&sym); + } + + // LLD will hide these otherwise-internal symbols since it only exports + // symbols explicitly passed via the `--export` flags above and hides all + // others. Various bits and pieces of tooling use this, so be sure these + // symbols make their way out of the linker as well. + self.cmd.arg("--export=__heap_base"); + self.cmd.arg("--export=__data_end"); + } + + fn subsystem(&mut self, _subsystem: &str) {} + + // Not needed for now with LLD + fn group_start(&mut self) {} + fn group_end(&mut self) {} + + fn linker_plugin_lto(&mut self) { + // Do nothing for now + } +} + +/// Linker shepherd script for L4Re (Fiasco) +pub struct L4Bender<'a> { + cmd: Command, + sess: &'a Session, + hinted_static: bool, +} + +impl<'a> Linker for L4Bender<'a> { + fn link_dylib(&mut self, _lib: &str, _verbatim: bool, _as_needed: bool) { + bug!("dylibs are not supported on L4Re"); + } + fn link_staticlib(&mut self, lib: &str, _verbatim: bool) { + self.hint_static(); + self.cmd.arg(format!("-PC{}", lib)); + } + fn link_rlib(&mut self, lib: &Path) { + self.hint_static(); + self.cmd.arg(lib); + } + fn include_path(&mut self, path: &Path) { + self.cmd.arg("-L").arg(path); + } + fn framework_path(&mut self, _: &Path) { + bug!("frameworks are not supported on L4Re"); + } + fn output_filename(&mut self, path: &Path) { + self.cmd.arg("-o").arg(path); + } + + fn add_object(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn full_relro(&mut self) { + self.cmd.arg("-zrelro"); + self.cmd.arg("-znow"); + } + + fn partial_relro(&mut self) { + self.cmd.arg("-zrelro"); + } + + fn no_relro(&mut self) { + self.cmd.arg("-znorelro"); + } + + fn cmd(&mut self) -> &mut Command { + &mut self.cmd + } + + fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {} + + fn link_rust_dylib(&mut self, _: &str, _: &Path) { + panic!("Rust dylibs not supported"); + } + + fn link_framework(&mut self, _framework: &str, _as_needed: bool) { + bug!("frameworks not supported on L4Re"); + } + + fn link_whole_staticlib(&mut self, lib: &str, _verbatim: bool, _search_path: &[PathBuf]) { + self.hint_static(); + self.cmd.arg("--whole-archive").arg(format!("-l{}", lib)); + self.cmd.arg("--no-whole-archive"); + } + + fn link_whole_rlib(&mut self, lib: &Path) { + self.hint_static(); + self.cmd.arg("--whole-archive").arg(lib).arg("--no-whole-archive"); + } + + fn gc_sections(&mut self, keep_metadata: bool) { + if !keep_metadata { + self.cmd.arg("--gc-sections"); + } + } + + fn no_gc_sections(&mut self) { + self.cmd.arg("--no-gc-sections"); + } + + fn optimize(&mut self) { + // GNU-style linkers support optimization with -O. GNU ld doesn't + // need a numeric argument, but other linkers do. + if self.sess.opts.optimize == config::OptLevel::Default + || self.sess.opts.optimize == config::OptLevel::Aggressive + { + self.cmd.arg("-O1"); + } + } + + fn pgo_gen(&mut self) {} + + fn debuginfo(&mut self, strip: Strip, _: &[PathBuf]) { + match strip { + Strip::None => {} + Strip::Debuginfo => { + self.cmd().arg("--strip-debug"); + } + Strip::Symbols => { + self.cmd().arg("--strip-all"); + } + } + } + + fn no_default_libraries(&mut self) { + self.cmd.arg("-nostdlib"); + } + + fn export_symbols(&mut self, _: &Path, _: CrateType, _: &[String]) { + // ToDo, not implemented, copy from GCC + self.sess.warn("exporting symbols not implemented yet for L4Bender"); + return; + } + + fn subsystem(&mut self, subsystem: &str) { + self.cmd.arg(&format!("--subsystem {}", subsystem)); + } + + fn reset_per_library_state(&mut self) { + self.hint_static(); // Reset to default before returning the composed command line. + } + + fn group_start(&mut self) { + self.cmd.arg("--start-group"); + } + + fn group_end(&mut self) { + self.cmd.arg("--end-group"); + } + + fn linker_plugin_lto(&mut self) {} + + fn control_flow_guard(&mut self) {} + + fn no_crt_objects(&mut self) {} +} + +impl<'a> L4Bender<'a> { + pub fn new(cmd: Command, sess: &'a Session) -> L4Bender<'a> { + L4Bender { cmd: cmd, sess: sess, hinted_static: false } + } + + fn hint_static(&mut self) { + if !self.hinted_static { + self.cmd.arg("-static"); + self.hinted_static = true; + } + } +} + +fn for_each_exported_symbols_include_dep<'tcx>( + tcx: TyCtxt<'tcx>, + crate_type: CrateType, + mut callback: impl FnMut(ExportedSymbol<'tcx>, SymbolExportInfo, CrateNum), +) { + for &(symbol, info) in tcx.exported_symbols(LOCAL_CRATE).iter() { + callback(symbol, info, LOCAL_CRATE); + } + + let formats = tcx.dependency_formats(()); + let deps = formats.iter().find_map(|(t, list)| (*t == crate_type).then_some(list)).unwrap(); + + for (index, dep_format) in deps.iter().enumerate() { + let cnum = CrateNum::new(index + 1); + // For each dependency that we are linking to statically ... + if *dep_format == Linkage::Static { + for &(symbol, info) in tcx.exported_symbols(cnum).iter() { + callback(symbol, info, cnum); + } + } + } +} + +pub(crate) fn exported_symbols(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec { + if let Some(ref exports) = tcx.sess.target.override_export_symbols { + return exports.iter().map(ToString::to_string).collect(); + } + + let mut symbols = Vec::new(); + + let export_threshold = symbol_export::crates_export_threshold(&[crate_type]); + for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| { + if info.level.is_below_threshold(export_threshold) { + symbols.push(symbol_export::symbol_name_for_instance_in_crate(tcx, symbol, cnum)); + } + }); + + symbols +} + +pub(crate) fn linked_symbols( + tcx: TyCtxt<'_>, + crate_type: CrateType, +) -> Vec<(String, SymbolExportKind)> { + match crate_type { + CrateType::Executable | CrateType::Cdylib | CrateType::Dylib => (), + CrateType::Staticlib | CrateType::ProcMacro | CrateType::Rlib => { + return Vec::new(); + } + } + + let mut symbols = Vec::new(); + + let export_threshold = symbol_export::crates_export_threshold(&[crate_type]); + for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| { + if info.level.is_below_threshold(export_threshold) || info.used { + symbols.push(( + symbol_export::linking_symbol_name_for_instance_in_crate(tcx, symbol, cnum), + info.kind, + )); + } + }); + + symbols +} + +/// Much simplified and explicit CLI for the NVPTX linker. The linker operates +/// with bitcode and uses LLVM backend to generate a PTX assembly. +pub struct PtxLinker<'a> { + cmd: Command, + sess: &'a Session, +} + +impl<'a> Linker for PtxLinker<'a> { + fn cmd(&mut self) -> &mut Command { + &mut self.cmd + } + + fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {} + + fn link_rlib(&mut self, path: &Path) { + self.cmd.arg("--rlib").arg(path); + } + + fn link_whole_rlib(&mut self, path: &Path) { + self.cmd.arg("--rlib").arg(path); + } + + fn include_path(&mut self, path: &Path) { + self.cmd.arg("-L").arg(path); + } + + fn debuginfo(&mut self, _strip: Strip, _: &[PathBuf]) { + self.cmd.arg("--debug"); + } + + fn add_object(&mut self, path: &Path) { + self.cmd.arg("--bitcode").arg(path); + } + + fn optimize(&mut self) { + match self.sess.lto() { + Lto::Thin | Lto::Fat | Lto::ThinLocal => { + self.cmd.arg("-Olto"); + } + + Lto::No => {} + }; + } + + fn output_filename(&mut self, path: &Path) { + self.cmd.arg("-o").arg(path); + } + + fn link_dylib(&mut self, _lib: &str, _verbatim: bool, _as_needed: bool) { + panic!("external dylibs not supported") + } + + fn link_rust_dylib(&mut self, _lib: &str, _path: &Path) { + panic!("external dylibs not supported") + } + + fn link_staticlib(&mut self, _lib: &str, _verbatim: bool) { + panic!("staticlibs not supported") + } + + fn link_whole_staticlib(&mut self, _lib: &str, _verbatim: bool, _search_path: &[PathBuf]) { + panic!("staticlibs not supported") + } + + fn framework_path(&mut self, _path: &Path) { + panic!("frameworks not supported") + } + + fn link_framework(&mut self, _framework: &str, _as_needed: bool) { + panic!("frameworks not supported") + } + + fn full_relro(&mut self) {} + + fn partial_relro(&mut self) {} + + fn no_relro(&mut self) {} + + fn gc_sections(&mut self, _keep_metadata: bool) {} + + fn no_gc_sections(&mut self) {} + + fn pgo_gen(&mut self) {} + + fn no_crt_objects(&mut self) {} + + fn no_default_libraries(&mut self) {} + + fn control_flow_guard(&mut self) {} + + fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, _symbols: &[String]) {} + + fn subsystem(&mut self, _subsystem: &str) {} + + fn group_start(&mut self) {} + + fn group_end(&mut self) {} + + fn linker_plugin_lto(&mut self) {} +} + +pub struct BpfLinker<'a> { + cmd: Command, + sess: &'a Session, +} + +impl<'a> Linker for BpfLinker<'a> { + fn cmd(&mut self) -> &mut Command { + &mut self.cmd + } + + fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {} + + fn link_rlib(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn link_whole_rlib(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn include_path(&mut self, path: &Path) { + self.cmd.arg("-L").arg(path); + } + + fn debuginfo(&mut self, _strip: Strip, _: &[PathBuf]) { + self.cmd.arg("--debug"); + } + + fn add_object(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn optimize(&mut self) { + self.cmd.arg(match self.sess.opts.optimize { + OptLevel::No => "-O0", + OptLevel::Less => "-O1", + OptLevel::Default => "-O2", + OptLevel::Aggressive => "-O3", + OptLevel::Size => "-Os", + OptLevel::SizeMin => "-Oz", + }); + } + + fn output_filename(&mut self, path: &Path) { + self.cmd.arg("-o").arg(path); + } + + fn link_dylib(&mut self, _lib: &str, _verbatim: bool, _as_needed: bool) { + panic!("external dylibs not supported") + } + + fn link_rust_dylib(&mut self, _lib: &str, _path: &Path) { + panic!("external dylibs not supported") + } + + fn link_staticlib(&mut self, _lib: &str, _verbatim: bool) { + panic!("staticlibs not supported") + } + + fn link_whole_staticlib(&mut self, _lib: &str, _verbatim: bool, _search_path: &[PathBuf]) { + panic!("staticlibs not supported") + } + + fn framework_path(&mut self, _path: &Path) { + panic!("frameworks not supported") + } + + fn link_framework(&mut self, _framework: &str, _as_needed: bool) { + panic!("frameworks not supported") + } + + fn full_relro(&mut self) {} + + fn partial_relro(&mut self) {} + + fn no_relro(&mut self) {} + + fn gc_sections(&mut self, _keep_metadata: bool) {} + + fn no_gc_sections(&mut self) {} + + fn pgo_gen(&mut self) {} + + fn no_crt_objects(&mut self) {} + + fn no_default_libraries(&mut self) {} + + fn control_flow_guard(&mut self) {} + + fn export_symbols(&mut self, tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) { + let path = tmpdir.join("symbols"); + let res: io::Result<()> = try { + let mut f = BufWriter::new(File::create(&path)?); + for sym in symbols { + writeln!(f, "{}", sym)?; + } + }; + if let Err(e) = res { + self.sess.fatal(&format!("failed to write symbols file: {}", e)); + } else { + self.cmd.arg("--export-symbols").arg(&path); + } + } + + fn subsystem(&mut self, _subsystem: &str) {} + + fn group_start(&mut self) {} + + fn group_end(&mut self) {} + + fn linker_plugin_lto(&mut self) {} +} diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs new file mode 100644 index 00000000000..cb6244050df --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/lto.rs @@ -0,0 +1,104 @@ +use super::write::CodegenContext; +use crate::traits::*; +use crate::ModuleCodegen; + +use rustc_data_structures::memmap::Mmap; +use rustc_errors::FatalError; + +use std::ffi::CString; +use std::sync::Arc; + +pub struct ThinModule { + pub shared: Arc>, + pub idx: usize, +} + +impl ThinModule { + pub fn name(&self) -> &str { + self.shared.module_names[self.idx].to_str().unwrap() + } + + pub fn cost(&self) -> u64 { + // Yes, that's correct, we're using the size of the bytecode as an + // indicator for how costly this codegen unit is. + self.data().len() as u64 + } + + pub fn data(&self) -> &[u8] { + let a = self.shared.thin_buffers.get(self.idx).map(|b| b.data()); + a.unwrap_or_else(|| { + let len = self.shared.thin_buffers.len(); + self.shared.serialized_modules[self.idx - len].data() + }) + } +} + +pub struct ThinShared { + pub data: B::ThinData, + pub thin_buffers: Vec, + pub serialized_modules: Vec>, + pub module_names: Vec, +} + +pub enum LtoModuleCodegen { + Fat { + module: ModuleCodegen, + _serialized_bitcode: Vec>, + }, + + Thin(ThinModule), +} + +impl LtoModuleCodegen { + pub fn name(&self) -> &str { + match *self { + LtoModuleCodegen::Fat { .. } => "everything", + LtoModuleCodegen::Thin(ref m) => m.name(), + } + } + + /// Optimize this module within the given codegen context. + /// + /// This function is unsafe as it'll return a `ModuleCodegen` still + /// points to LLVM data structures owned by this `LtoModuleCodegen`. + /// It's intended that the module returned is immediately code generated and + /// dropped, and then this LTO module is dropped. + pub unsafe fn optimize( + self, + cgcx: &CodegenContext, + ) -> Result, FatalError> { + match self { + LtoModuleCodegen::Fat { mut module, .. } => { + B::optimize_fat(cgcx, &mut module)?; + Ok(module) + } + LtoModuleCodegen::Thin(thin) => B::optimize_thin(cgcx, thin), + } + } + + /// A "gauge" of how costly it is to optimize this module, used to sort + /// biggest modules first. + pub fn cost(&self) -> u64 { + match *self { + // Only one module with fat LTO, so the cost doesn't matter. + LtoModuleCodegen::Fat { .. } => 0, + LtoModuleCodegen::Thin(ref m) => m.cost(), + } + } +} + +pub enum SerializedModule { + Local(M), + FromRlib(Vec), + FromUncompressedFile(Mmap), +} + +impl SerializedModule { + pub fn data(&self) -> &[u8] { + match *self { + SerializedModule::Local(ref m) => m.data(), + SerializedModule::FromRlib(ref m) => m, + SerializedModule::FromUncompressedFile(ref m) => m, + } + } +} diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs new file mode 100644 index 00000000000..0302c28815a --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -0,0 +1,314 @@ +//! Reading of the rustc metadata for rlibs and dylibs + +use std::fs::File; +use std::io::Write; +use std::path::Path; + +use object::write::{self, StandardSegment, Symbol, SymbolSection}; +use object::{ + elf, pe, Architecture, BinaryFormat, Endianness, FileFlags, Object, ObjectSection, + SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope, +}; + +use snap::write::FrameEncoder; + +use rustc_data_structures::memmap::Mmap; +use rustc_data_structures::owning_ref::OwningRef; +use rustc_data_structures::rustc_erase_owner; +use rustc_data_structures::sync::MetadataRef; +use rustc_metadata::fs::METADATA_FILENAME; +use rustc_metadata::EncodedMetadata; +use rustc_session::cstore::MetadataLoader; +use rustc_session::Session; +use rustc_target::abi::Endian; +use rustc_target::spec::{RelocModel, Target}; + +/// The default metadata loader. This is used by cg_llvm and cg_clif. +/// +/// # Metadata location +/// +///
+///
rlib
+///
The metadata can be found in the `lib.rmeta` file inside of the ar archive.
+///
dylib
+///
The metadata can be found in the `.rustc` section of the shared library.
+///
+pub struct DefaultMetadataLoader; + +fn load_metadata_with( + path: &Path, + f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>, +) -> Result { + let file = + File::open(path).map_err(|e| format!("failed to open file '{}': {}", path.display(), e))?; + let data = unsafe { Mmap::map(file) } + .map_err(|e| format!("failed to mmap file '{}': {}", path.display(), e))?; + let metadata = OwningRef::new(data).try_map(f)?; + return Ok(rustc_erase_owner!(metadata.map_owner_box())); +} + +impl MetadataLoader for DefaultMetadataLoader { + fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result { + load_metadata_with(path, |data| { + let archive = object::read::archive::ArchiveFile::parse(&*data) + .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; + + for entry_result in archive.members() { + let entry = entry_result + .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; + if entry.name() == METADATA_FILENAME.as_bytes() { + let data = entry + .data(data) + .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; + return search_for_metadata(path, data, ".rmeta"); + } + } + + Err(format!("metadata not found in rlib '{}'", path.display())) + }) + } + + fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result { + load_metadata_with(path, |data| search_for_metadata(path, data, ".rustc")) + } +} + +fn search_for_metadata<'a>( + path: &Path, + bytes: &'a [u8], + section: &str, +) -> Result<&'a [u8], String> { + let Ok(file) = object::File::parse(bytes) else { + // The parse above could fail for odd reasons like corruption, but for + // now we just interpret it as this target doesn't support metadata + // emission in object files so the entire byte slice itself is probably + // a metadata file. Ideally though if necessary we could at least check + // the prefix of bytes to see if it's an actual metadata object and if + // not forward the error along here. + return Ok(bytes); + }; + file.section_by_name(section) + .ok_or_else(|| format!("no `{}` section in '{}'", section, path.display()))? + .data() + .map_err(|e| format!("failed to read {} section in '{}': {}", section, path.display(), e)) +} + +pub(crate) fn create_object_file(sess: &Session) -> Option> { + let endianness = match sess.target.options.endian { + Endian::Little => Endianness::Little, + Endian::Big => Endianness::Big, + }; + let architecture = match &sess.target.arch[..] { + "arm" => Architecture::Arm, + "aarch64" => Architecture::Aarch64, + "x86" => Architecture::I386, + "s390x" => Architecture::S390x, + "mips" => Architecture::Mips, + "mips64" => Architecture::Mips64, + "x86_64" => { + if sess.target.pointer_width == 32 { + Architecture::X86_64_X32 + } else { + Architecture::X86_64 + } + } + "powerpc" => Architecture::PowerPc, + "powerpc64" => Architecture::PowerPc64, + "riscv32" => Architecture::Riscv32, + "riscv64" => Architecture::Riscv64, + "sparc64" => Architecture::Sparc64, + // Unsupported architecture. + _ => return None, + }; + let binary_format = if sess.target.is_like_osx { + BinaryFormat::MachO + } else if sess.target.is_like_windows { + BinaryFormat::Coff + } else { + BinaryFormat::Elf + }; + + let mut file = write::Object::new(binary_format, architecture, endianness); + let e_flags = match architecture { + Architecture::Mips => { + let arch = match sess.target.options.cpu.as_ref() { + "mips1" => elf::EF_MIPS_ARCH_1, + "mips2" => elf::EF_MIPS_ARCH_2, + "mips3" => elf::EF_MIPS_ARCH_3, + "mips4" => elf::EF_MIPS_ARCH_4, + "mips5" => elf::EF_MIPS_ARCH_5, + s if s.contains("r6") => elf::EF_MIPS_ARCH_32R6, + _ => elf::EF_MIPS_ARCH_32R2, + }; + // The only ABI LLVM supports for 32-bit MIPS CPUs is o32. + let mut e_flags = elf::EF_MIPS_CPIC | elf::EF_MIPS_ABI_O32 | arch; + if sess.target.options.relocation_model != RelocModel::Static { + e_flags |= elf::EF_MIPS_PIC; + } + if sess.target.options.cpu.contains("r6") { + e_flags |= elf::EF_MIPS_NAN2008; + } + e_flags + } + Architecture::Mips64 => { + // copied from `mips64el-linux-gnuabi64-gcc foo.c -c` + let e_flags = elf::EF_MIPS_CPIC + | elf::EF_MIPS_PIC + | if sess.target.options.cpu.contains("r6") { + elf::EF_MIPS_ARCH_64R6 | elf::EF_MIPS_NAN2008 + } else { + elf::EF_MIPS_ARCH_64R2 + }; + e_flags + } + Architecture::Riscv64 if sess.target.options.features.contains("+d") => { + // copied from `riscv64-linux-gnu-gcc foo.c -c`, note though + // that the `+d` target feature represents whether the double + // float abi is enabled. + let e_flags = elf::EF_RISCV_RVC | elf::EF_RISCV_FLOAT_ABI_DOUBLE; + e_flags + } + _ => 0, + }; + // adapted from LLVM's `MCELFObjectTargetWriter::getOSABI` + let os_abi = match sess.target.options.os.as_ref() { + "hermit" => elf::ELFOSABI_STANDALONE, + "freebsd" => elf::ELFOSABI_FREEBSD, + "solaris" => elf::ELFOSABI_SOLARIS, + _ => elf::ELFOSABI_NONE, + }; + let abi_version = 0; + file.flags = FileFlags::Elf { os_abi, abi_version, e_flags }; + Some(file) +} + +pub enum MetadataPosition { + First, + Last, +} + +// For rlibs we "pack" rustc metadata into a dummy object file. When rustc +// creates a dylib crate type it will pass `--whole-archive` (or the +// platform equivalent) to include all object files from an rlib into the +// final dylib itself. This causes linkers to iterate and try to include all +// files located in an archive, so if metadata is stored in an archive then +// it needs to be of a form that the linker will be able to process. +// +// Note, though, that we don't actually want this metadata to show up in any +// final output of the compiler. Instead this is purely for rustc's own +// metadata tracking purposes. +// +// With the above in mind, each "flavor" of object format gets special +// handling here depending on the target: +// +// * MachO - macos-like targets will insert the metadata into a section that +// is sort of fake dwarf debug info. Inspecting the source of the macos +// linker this causes these sections to be skipped automatically because +// it's not in an allowlist of otherwise well known dwarf section names to +// go into the final artifact. +// +// * WebAssembly - we actually don't have any container format for this +// target. WebAssembly doesn't support the `dylib` crate type anyway so +// there's no need for us to support this at this time. Consequently the +// metadata bytes are simply stored as-is into an rlib. +// +// * COFF - Windows-like targets create an object with a section that has +// the `IMAGE_SCN_LNK_REMOVE` flag set which ensures that if the linker +// ever sees the section it doesn't process it and it's removed. +// +// * ELF - All other targets are similar to Windows in that there's a +// `SHF_EXCLUDE` flag we can set on sections in an object file to get +// automatically removed from the final output. +pub fn create_rmeta_file(sess: &Session, metadata: &[u8]) -> (Vec, MetadataPosition) { + let Some(mut file) = create_object_file(sess) else { + // This is used to handle all "other" targets. This includes targets + // in two categories: + // + // * Some targets don't have support in the `object` crate just yet + // to write an object file. These targets are likely to get filled + // out over time. + // + // * Targets like WebAssembly don't support dylibs, so the purpose + // of putting metadata in object files, to support linking rlibs + // into dylibs, is moot. + // + // In both of these cases it means that linking into dylibs will + // not be supported by rustc. This doesn't matter for targets like + // WebAssembly and for targets not supported by the `object` crate + // yet it means that work will need to be done in the `object` crate + // to add a case above. + return (metadata.to_vec(), MetadataPosition::Last); + }; + let section = file.add_section( + file.segment_name(StandardSegment::Debug).to_vec(), + b".rmeta".to_vec(), + SectionKind::Debug, + ); + match file.format() { + BinaryFormat::Coff => { + file.section_mut(section).flags = + SectionFlags::Coff { characteristics: pe::IMAGE_SCN_LNK_REMOVE }; + } + BinaryFormat::Elf => { + file.section_mut(section).flags = + SectionFlags::Elf { sh_flags: elf::SHF_EXCLUDE as u64 }; + } + _ => {} + }; + file.append_section_data(section, metadata, 1); + (file.write().unwrap(), MetadataPosition::First) +} + +// Historical note: +// +// When using link.exe it was seen that the section name `.note.rustc` +// was getting shortened to `.note.ru`, and according to the PE and COFF +// specification: +// +// > Executable images do not use a string table and do not support +// > section names longer than 8 characters +// +// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format +// +// As a result, we choose a slightly shorter name! As to why +// `.note.rustc` works on MinGW, see +// https://github.com/llvm/llvm-project/blob/llvmorg-12.0.0/lld/COFF/Writer.cpp#L1190-L1197 +pub fn create_compressed_metadata_file( + sess: &Session, + metadata: &EncodedMetadata, + symbol_name: &str, +) -> Vec { + let mut compressed = rustc_metadata::METADATA_HEADER.to_vec(); + FrameEncoder::new(&mut compressed).write_all(metadata.raw_data()).unwrap(); + let Some(mut file) = create_object_file(sess) else { + return compressed.to_vec(); + }; + let section = file.add_section( + file.segment_name(StandardSegment::Data).to_vec(), + b".rustc".to_vec(), + SectionKind::ReadOnlyData, + ); + match file.format() { + BinaryFormat::Elf => { + // Explicitly set no flags to avoid SHF_ALLOC default for data section. + file.section_mut(section).flags = SectionFlags::Elf { sh_flags: 0 }; + } + _ => {} + }; + let offset = file.append_section_data(section, &compressed, 1); + + // For MachO and probably PE this is necessary to prevent the linker from throwing away the + // .rustc section. For ELF this isn't necessary, but it also doesn't harm. + file.add_symbol(Symbol { + name: symbol_name.as_bytes().to_vec(), + value: offset, + size: compressed.len() as u64, + kind: SymbolKind::Data, + scope: SymbolScope::Dynamic, + weak: false, + section: SymbolSection::Section(section), + flags: SymbolFlags::None, + }); + + file.write().unwrap() +} diff --git a/compiler/rustc_codegen_ssa/src/back/mod.rs b/compiler/rustc_codegen_ssa/src/back/mod.rs new file mode 100644 index 00000000000..d11ed54eb20 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/mod.rs @@ -0,0 +1,9 @@ +pub mod archive; +pub mod command; +pub mod link; +pub mod linker; +pub mod lto; +pub mod metadata; +pub mod rpath; +pub mod symbol_export; +pub mod write; diff --git a/compiler/rustc_codegen_ssa/src/back/rpath.rs b/compiler/rustc_codegen_ssa/src/back/rpath.rs new file mode 100644 index 00000000000..0b5656c9ad1 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/rpath.rs @@ -0,0 +1,114 @@ +use pathdiff::diff_paths; +use rustc_data_structures::fx::FxHashSet; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; + +pub struct RPathConfig<'a> { + pub libs: &'a [&'a Path], + pub out_filename: PathBuf, + pub is_like_osx: bool, + pub has_rpath: bool, + pub linker_is_gnu: bool, +} + +pub fn get_rpath_flags(config: &mut RPathConfig<'_>) -> Vec { + // No rpath on windows + if !config.has_rpath { + return Vec::new(); + } + + debug!("preparing the RPATH!"); + + let rpaths = get_rpaths(config); + let mut flags = rpaths_to_flags(&rpaths); + + if config.linker_is_gnu { + // Use DT_RUNPATH instead of DT_RPATH if available + flags.push("-Wl,--enable-new-dtags".to_owned()); + + // Set DF_ORIGIN for substitute $ORIGIN + flags.push("-Wl,-z,origin".to_owned()); + } + + flags +} + +fn rpaths_to_flags(rpaths: &[String]) -> Vec { + let mut ret = Vec::with_capacity(rpaths.len()); // the minimum needed capacity + + for rpath in rpaths { + if rpath.contains(',') { + ret.push("-Wl,-rpath".into()); + ret.push("-Xlinker".into()); + ret.push(rpath.clone()); + } else { + ret.push(format!("-Wl,-rpath,{}", &(*rpath))); + } + } + + ret +} + +fn get_rpaths(config: &mut RPathConfig<'_>) -> Vec { + debug!("output: {:?}", config.out_filename.display()); + debug!("libs:"); + for libpath in config.libs { + debug!(" {:?}", libpath.display()); + } + + // Use relative paths to the libraries. Binaries can be moved + // as long as they maintain the relative relationship to the + // crates they depend on. + let rpaths = get_rpaths_relative_to_output(config); + + debug!("rpaths:"); + for rpath in &rpaths { + debug!(" {}", rpath); + } + + // Remove duplicates + minimize_rpaths(&rpaths) +} + +fn get_rpaths_relative_to_output(config: &mut RPathConfig<'_>) -> Vec { + config.libs.iter().map(|a| get_rpath_relative_to_output(config, a)).collect() +} + +fn get_rpath_relative_to_output(config: &mut RPathConfig<'_>, lib: &Path) -> String { + // Mac doesn't appear to support $ORIGIN + let prefix = if config.is_like_osx { "@loader_path" } else { "$ORIGIN" }; + + let cwd = env::current_dir().unwrap(); + let mut lib = fs::canonicalize(&cwd.join(lib)).unwrap_or_else(|_| cwd.join(lib)); + lib.pop(); // strip filename + let mut output = cwd.join(&config.out_filename); + output.pop(); // strip filename + let output = fs::canonicalize(&output).unwrap_or(output); + let relative = path_relative_from(&lib, &output) + .unwrap_or_else(|| panic!("couldn't create relative path from {:?} to {:?}", output, lib)); + // FIXME (#9639): This needs to handle non-utf8 paths + format!("{}/{}", prefix, relative.to_str().expect("non-utf8 component in path")) +} + +// This routine is adapted from the *old* Path's `path_relative_from` +// function, which works differently from the new `relative_from` function. +// In particular, this handles the case on unix where both paths are +// absolute but with only the root as the common directory. +fn path_relative_from(path: &Path, base: &Path) -> Option { + diff_paths(path, base) +} + +fn minimize_rpaths(rpaths: &[String]) -> Vec { + let mut set = FxHashSet::default(); + let mut minimized = Vec::new(); + for rpath in rpaths { + if set.insert(rpath) { + minimized.push(rpath.clone()); + } + } + minimized +} + +#[cfg(all(unix, test))] +mod tests; diff --git a/compiler/rustc_codegen_ssa/src/back/rpath/tests.rs b/compiler/rustc_codegen_ssa/src/back/rpath/tests.rs new file mode 100644 index 00000000000..604f19144a6 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/rpath/tests.rs @@ -0,0 +1,72 @@ +use super::RPathConfig; +use super::{get_rpath_relative_to_output, minimize_rpaths, rpaths_to_flags}; +use std::path::{Path, PathBuf}; + +#[test] +fn test_rpaths_to_flags() { + let flags = rpaths_to_flags(&["path1".to_string(), "path2".to_string()]); + assert_eq!(flags, ["-Wl,-rpath,path1", "-Wl,-rpath,path2"]); +} + +#[test] +fn test_minimize1() { + let res = minimize_rpaths(&["rpath1".to_string(), "rpath2".to_string(), "rpath1".to_string()]); + assert!(res == ["rpath1", "rpath2",]); +} + +#[test] +fn test_minimize2() { + let res = minimize_rpaths(&[ + "1a".to_string(), + "2".to_string(), + "2".to_string(), + "1a".to_string(), + "4a".to_string(), + "1a".to_string(), + "2".to_string(), + "3".to_string(), + "4a".to_string(), + "3".to_string(), + ]); + assert!(res == ["1a", "2", "4a", "3",]); +} + +#[test] +fn test_rpath_relative() { + if cfg!(target_os = "macos") { + let config = &mut RPathConfig { + libs: &[], + has_rpath: true, + is_like_osx: true, + linker_is_gnu: false, + out_filename: PathBuf::from("bin/rustc"), + }; + let res = get_rpath_relative_to_output(config, Path::new("lib/libstd.so")); + assert_eq!(res, "@loader_path/../lib"); + } else { + let config = &mut RPathConfig { + libs: &[], + out_filename: PathBuf::from("bin/rustc"), + has_rpath: true, + is_like_osx: false, + linker_is_gnu: true, + }; + let res = get_rpath_relative_to_output(config, Path::new("lib/libstd.so")); + assert_eq!(res, "$ORIGIN/../lib"); + } +} + +#[test] +fn test_xlinker() { + let args = rpaths_to_flags(&["a/normal/path".to_string(), "a,comma,path".to_string()]); + + assert_eq!( + args, + vec![ + "-Wl,-rpath,a/normal/path".to_string(), + "-Wl,-rpath".to_string(), + "-Xlinker".to_string(), + "a,comma,path".to_string() + ] + ); +} diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs new file mode 100644 index 00000000000..e6b6055759d --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -0,0 +1,590 @@ +use std::collections::hash_map::Entry::*; + +use rustc_ast::expand::allocator::ALLOCATOR_METHODS; +use rustc_data_structures::fx::FxHashMap; +use rustc_hir as hir; +use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LOCAL_CRATE}; +use rustc_hir::Node; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::middle::exported_symbols::{ + metadata_symbol_name, ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel, +}; +use rustc_middle::ty::query::{ExternProviders, Providers}; +use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; +use rustc_middle::ty::Instance; +use rustc_middle::ty::{self, SymbolName, TyCtxt}; +use rustc_session::config::CrateType; +use rustc_target::spec::SanitizerSet; + +pub fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel { + crates_export_threshold(&tcx.sess.crate_types()) +} + +fn crate_export_threshold(crate_type: CrateType) -> SymbolExportLevel { + match crate_type { + CrateType::Executable | CrateType::Staticlib | CrateType::ProcMacro | CrateType::Cdylib => { + SymbolExportLevel::C + } + CrateType::Rlib | CrateType::Dylib => SymbolExportLevel::Rust, + } +} + +pub fn crates_export_threshold(crate_types: &[CrateType]) -> SymbolExportLevel { + if crate_types + .iter() + .any(|&crate_type| crate_export_threshold(crate_type) == SymbolExportLevel::Rust) + { + SymbolExportLevel::Rust + } else { + SymbolExportLevel::C + } +} + +fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap { + assert_eq!(cnum, LOCAL_CRATE); + + if !tcx.sess.opts.output_types.should_codegen() { + return Default::default(); + } + + // Check to see if this crate is a "special runtime crate". These + // crates, implementation details of the standard library, typically + // have a bunch of `pub extern` and `#[no_mangle]` functions as the + // ABI between them. We don't want their symbols to have a `C` + // export level, however, as they're just implementation details. + // Down below we'll hardwire all of the symbols to the `Rust` export + // level instead. + let special_runtime_crate = + tcx.is_panic_runtime(LOCAL_CRATE) || tcx.is_compiler_builtins(LOCAL_CRATE); + + let mut reachable_non_generics: DefIdMap<_> = tcx + .reachable_set(()) + .iter() + .filter_map(|&def_id| { + // We want to ignore some FFI functions that are not exposed from + // this crate. Reachable FFI functions can be lumped into two + // categories: + // + // 1. Those that are included statically via a static library + // 2. Those included otherwise (e.g., dynamically or via a framework) + // + // Although our LLVM module is not literally emitting code for the + // statically included symbols, it's an export of our library which + // needs to be passed on to the linker and encoded in the metadata. + // + // As a result, if this id is an FFI item (foreign item) then we only + // let it through if it's included statically. + match tcx.hir().get_by_def_id(def_id) { + Node::ForeignItem(..) => { + tcx.is_statically_included_foreign_item(def_id).then_some(def_id) + } + + // Only consider nodes that actually have exported symbols. + Node::Item(&hir::Item { + kind: hir::ItemKind::Static(..) | hir::ItemKind::Fn(..), + .. + }) + | Node::ImplItem(&hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. }) => { + let generics = tcx.generics_of(def_id); + if !generics.requires_monomorphization(tcx) + // Functions marked with #[inline] are codegened with "internal" + // linkage and are not exported unless marked with an extern + // indicator + && (!Instance::mono(tcx, def_id.to_def_id()).def.generates_cgu_internal_copy(tcx) + || tcx.codegen_fn_attrs(def_id.to_def_id()).contains_extern_indicator()) + { + Some(def_id) + } else { + None + } + } + + _ => None, + } + }) + .map(|def_id| { + let (export_level, used) = if special_runtime_crate { + let name = tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())).name; + // We won't link right if these symbols are stripped during LTO. + let used = match name { + "rust_eh_personality" + | "rust_eh_register_frames" + | "rust_eh_unregister_frames" => true, + _ => false, + }; + (SymbolExportLevel::Rust, used) + } else { + (symbol_export_level(tcx, def_id.to_def_id()), false) + }; + let codegen_attrs = tcx.codegen_fn_attrs(def_id.to_def_id()); + debug!( + "EXPORTED SYMBOL (local): {} ({:?})", + tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())), + export_level + ); + (def_id.to_def_id(), SymbolExportInfo { + level: export_level, + kind: if tcx.is_static(def_id.to_def_id()) { + if codegen_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { + SymbolExportKind::Tls + } else { + SymbolExportKind::Data + } + } else { + SymbolExportKind::Text + }, + used: codegen_attrs.flags.contains(CodegenFnAttrFlags::USED) + || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) || used, + }) + }) + .collect(); + + if let Some(id) = tcx.proc_macro_decls_static(()) { + reachable_non_generics.insert( + id.to_def_id(), + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Data, + used: false, + }, + ); + } + + reachable_non_generics +} + +fn is_reachable_non_generic_provider_local(tcx: TyCtxt<'_>, def_id: DefId) -> bool { + let export_threshold = threshold(tcx); + + if let Some(&info) = tcx.reachable_non_generics(def_id.krate).get(&def_id) { + info.level.is_below_threshold(export_threshold) + } else { + false + } +} + +fn is_reachable_non_generic_provider_extern(tcx: TyCtxt<'_>, def_id: DefId) -> bool { + tcx.reachable_non_generics(def_id.krate).contains_key(&def_id) +} + +fn exported_symbols_provider_local<'tcx>( + tcx: TyCtxt<'tcx>, + cnum: CrateNum, +) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] { + assert_eq!(cnum, LOCAL_CRATE); + + if !tcx.sess.opts.output_types.should_codegen() { + return &[]; + } + + let mut symbols: Vec<_> = tcx + .reachable_non_generics(LOCAL_CRATE) + .iter() + .map(|(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)) + .collect(); + + if tcx.entry_fn(()).is_some() { + let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, "main")); + + symbols.push(( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Text, + used: false, + }, + )); + } + + if tcx.allocator_kind(()).is_some() { + for method in ALLOCATOR_METHODS { + let symbol_name = format!("__rust_{}", method.name); + let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); + + symbols.push(( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); + } + } + + if tcx.sess.instrument_coverage() || tcx.sess.opts.cg.profile_generate.enabled() { + // These are weak symbols that point to the profile version and the + // profile name, which need to be treated as exported so LTO doesn't nix + // them. + const PROFILER_WEAK_SYMBOLS: [&str; 2] = + ["__llvm_profile_raw_version", "__llvm_profile_filename"]; + + symbols.extend(PROFILER_WEAK_SYMBOLS.iter().map(|sym| { + let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym)); + ( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Data, + used: false, + }, + ) + })); + } + + if tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) { + let mut msan_weak_symbols = Vec::new(); + + // Similar to profiling, preserve weak msan symbol during LTO. + if tcx.sess.opts.unstable_opts.sanitizer_recover.contains(SanitizerSet::MEMORY) { + msan_weak_symbols.push("__msan_keep_going"); + } + + if tcx.sess.opts.unstable_opts.sanitizer_memory_track_origins != 0 { + msan_weak_symbols.push("__msan_track_origins"); + } + + symbols.extend(msan_weak_symbols.into_iter().map(|sym| { + let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym)); + ( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Data, + used: false, + }, + ) + })); + } + + if tcx.sess.crate_types().contains(&CrateType::Dylib) + || tcx.sess.crate_types().contains(&CrateType::ProcMacro) + { + let symbol_name = metadata_symbol_name(tcx); + let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); + + symbols.push(( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Data, + used: true, + }, + )); + } + + if tcx.sess.opts.share_generics() && tcx.local_crate_exports_generics() { + use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility}; + use rustc_middle::ty::InstanceDef; + + // Normally, we require that shared monomorphizations are not hidden, + // because if we want to re-use a monomorphization from a Rust dylib, it + // needs to be exported. + // However, on platforms that don't allow for Rust dylibs, having + // external linkage is enough for monomorphization to be linked to. + let need_visibility = tcx.sess.target.dynamic_linking && !tcx.sess.target.only_cdylib; + + let (_, cgus) = tcx.collect_and_partition_mono_items(()); + + for (mono_item, &(linkage, visibility)) in cgus.iter().flat_map(|cgu| cgu.items().iter()) { + if linkage != Linkage::External { + // We can only re-use things with external linkage, otherwise + // we'll get a linker error + continue; + } + + if need_visibility && visibility == Visibility::Hidden { + // If we potentially share things from Rust dylibs, they must + // not be hidden + continue; + } + + match *mono_item { + MonoItem::Fn(Instance { def: InstanceDef::Item(def), substs }) => { + if substs.non_erasable_generics().next().is_some() { + let symbol = ExportedSymbol::Generic(def.did, substs); + symbols.push(( + symbol, + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); + } + } + MonoItem::Fn(Instance { def: InstanceDef::DropGlue(_, Some(ty)), substs }) => { + // A little sanity-check + debug_assert_eq!( + substs.non_erasable_generics().next(), + Some(GenericArgKind::Type(ty)) + ); + symbols.push(( + ExportedSymbol::DropGlue(ty), + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); + } + _ => { + // Any other symbols don't qualify for sharing + } + } + } + } + + // Sort so we get a stable incr. comp. hash. + symbols.sort_by_cached_key(|s| s.0.symbol_name_for_local_instance(tcx)); + + tcx.arena.alloc_from_iter(symbols) +} + +fn upstream_monomorphizations_provider( + tcx: TyCtxt<'_>, + (): (), +) -> DefIdMap, CrateNum>> { + let cnums = tcx.crates(()); + + let mut instances: DefIdMap> = Default::default(); + + let drop_in_place_fn_def_id = tcx.lang_items().drop_in_place_fn(); + + for &cnum in cnums.iter() { + for (exported_symbol, _) in tcx.exported_symbols(cnum).iter() { + let (def_id, substs) = match *exported_symbol { + ExportedSymbol::Generic(def_id, substs) => (def_id, substs), + ExportedSymbol::DropGlue(ty) => { + if let Some(drop_in_place_fn_def_id) = drop_in_place_fn_def_id { + (drop_in_place_fn_def_id, tcx.intern_substs(&[ty.into()])) + } else { + // `drop_in_place` in place does not exist, don't try + // to use it. + continue; + } + } + ExportedSymbol::NonGeneric(..) | ExportedSymbol::NoDefId(..) => { + // These are no monomorphizations + continue; + } + }; + + let substs_map = instances.entry(def_id).or_default(); + + match substs_map.entry(substs) { + Occupied(mut e) => { + // If there are multiple monomorphizations available, + // we select one deterministically. + let other_cnum = *e.get(); + if tcx.stable_crate_id(other_cnum) > tcx.stable_crate_id(cnum) { + e.insert(cnum); + } + } + Vacant(e) => { + e.insert(cnum); + } + } + } + } + + instances +} + +fn upstream_monomorphizations_for_provider( + tcx: TyCtxt<'_>, + def_id: DefId, +) -> Option<&FxHashMap, CrateNum>> { + debug_assert!(!def_id.is_local()); + tcx.upstream_monomorphizations(()).get(&def_id) +} + +fn upstream_drop_glue_for_provider<'tcx>( + tcx: TyCtxt<'tcx>, + substs: SubstsRef<'tcx>, +) -> Option { + if let Some(def_id) = tcx.lang_items().drop_in_place_fn() { + tcx.upstream_monomorphizations_for(def_id).and_then(|monos| monos.get(&substs).cloned()) + } else { + None + } +} + +fn is_unreachable_local_definition_provider(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { + !tcx.reachable_set(()).contains(&def_id) +} + +pub fn provide(providers: &mut Providers) { + providers.reachable_non_generics = reachable_non_generics_provider; + providers.is_reachable_non_generic = is_reachable_non_generic_provider_local; + providers.exported_symbols = exported_symbols_provider_local; + providers.upstream_monomorphizations = upstream_monomorphizations_provider; + providers.is_unreachable_local_definition = is_unreachable_local_definition_provider; + providers.upstream_drop_glue_for = upstream_drop_glue_for_provider; + providers.wasm_import_module_map = wasm_import_module_map; +} + +pub fn provide_extern(providers: &mut ExternProviders) { + providers.is_reachable_non_generic = is_reachable_non_generic_provider_extern; + providers.upstream_monomorphizations_for = upstream_monomorphizations_for_provider; +} + +fn symbol_export_level(tcx: TyCtxt<'_>, sym_def_id: DefId) -> SymbolExportLevel { + // We export anything that's not mangled at the "C" layer as it probably has + // to do with ABI concerns. We do not, however, apply such treatment to + // special symbols in the standard library for various plumbing between + // core/std/allocators/etc. For example symbols used to hook up allocation + // are not considered for export + let codegen_fn_attrs = tcx.codegen_fn_attrs(sym_def_id); + let is_extern = codegen_fn_attrs.contains_extern_indicator(); + let std_internal = + codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL); + + if is_extern && !std_internal { + let target = &tcx.sess.target.llvm_target; + // WebAssembly cannot export data symbols, so reduce their export level + if target.contains("emscripten") { + if let Some(Node::Item(&hir::Item { kind: hir::ItemKind::Static(..), .. })) = + tcx.hir().get_if_local(sym_def_id) + { + return SymbolExportLevel::Rust; + } + } + + SymbolExportLevel::C + } else { + SymbolExportLevel::Rust + } +} + +/// This is the symbol name of the given instance instantiated in a specific crate. +pub fn symbol_name_for_instance_in_crate<'tcx>( + tcx: TyCtxt<'tcx>, + symbol: ExportedSymbol<'tcx>, + instantiating_crate: CrateNum, +) -> String { + // If this is something instantiated in the local crate then we might + // already have cached the name as a query result. + if instantiating_crate == LOCAL_CRATE { + return symbol.symbol_name_for_local_instance(tcx).to_string(); + } + + // This is something instantiated in an upstream crate, so we have to use + // the slower (because uncached) version of computing the symbol name. + match symbol { + ExportedSymbol::NonGeneric(def_id) => { + rustc_symbol_mangling::symbol_name_for_instance_in_crate( + tcx, + Instance::mono(tcx, def_id), + instantiating_crate, + ) + } + ExportedSymbol::Generic(def_id, substs) => { + rustc_symbol_mangling::symbol_name_for_instance_in_crate( + tcx, + Instance::new(def_id, substs), + instantiating_crate, + ) + } + ExportedSymbol::DropGlue(ty) => rustc_symbol_mangling::symbol_name_for_instance_in_crate( + tcx, + Instance::resolve_drop_in_place(tcx, ty), + instantiating_crate, + ), + ExportedSymbol::NoDefId(symbol_name) => symbol_name.to_string(), + } +} + +/// This is the symbol name of the given instance as seen by the linker. +/// +/// On 32-bit Windows symbols are decorated according to their calling conventions. +pub fn linking_symbol_name_for_instance_in_crate<'tcx>( + tcx: TyCtxt<'tcx>, + symbol: ExportedSymbol<'tcx>, + instantiating_crate: CrateNum, +) -> String { + use rustc_target::abi::call::Conv; + + let mut undecorated = symbol_name_for_instance_in_crate(tcx, symbol, instantiating_crate); + + let target = &tcx.sess.target; + if !target.is_like_windows { + // Mach-O has a global "_" suffix and `object` crate will handle it. + // ELF does not have any symbol decorations. + return undecorated; + } + + let x86 = match &target.arch[..] { + "x86" => true, + "x86_64" => false, + // Only x86/64 use symbol decorations. + _ => return undecorated, + }; + + let instance = match symbol { + ExportedSymbol::NonGeneric(def_id) | ExportedSymbol::Generic(def_id, _) + if tcx.is_static(def_id) => + { + None + } + ExportedSymbol::NonGeneric(def_id) => Some(Instance::mono(tcx, def_id)), + ExportedSymbol::Generic(def_id, substs) => Some(Instance::new(def_id, substs)), + // DropGlue always use the Rust calling convention and thus follow the target's default + // symbol decoration scheme. + ExportedSymbol::DropGlue(..) => None, + // NoDefId always follow the target's default symbol decoration scheme. + ExportedSymbol::NoDefId(..) => None, + }; + + let (conv, args) = instance + .map(|i| { + tcx.fn_abi_of_instance(ty::ParamEnv::reveal_all().and((i, ty::List::empty()))) + .unwrap_or_else(|_| bug!("fn_abi_of_instance({i:?}) failed")) + }) + .map(|fnabi| (fnabi.conv, &fnabi.args[..])) + .unwrap_or((Conv::Rust, &[])); + + // Decorate symbols with prefices, suffices and total number of bytes of arguments. + // Reference: https://docs.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170 + let (prefix, suffix) = match conv { + Conv::X86Fastcall => ("@", "@"), + Conv::X86Stdcall => ("_", "@"), + Conv::X86VectorCall => ("", "@@"), + _ => { + if x86 { + undecorated.insert(0, '_'); + } + return undecorated; + } + }; + + let args_in_bytes: u64 = args + .iter() + .map(|abi| abi.layout.size.bytes().next_multiple_of(target.pointer_width as u64 / 8)) + .sum(); + format!("{prefix}{undecorated}{suffix}{args_in_bytes}") +} + +fn wasm_import_module_map(tcx: TyCtxt<'_>, cnum: CrateNum) -> FxHashMap { + // Build up a map from DefId to a `NativeLib` structure, where + // `NativeLib` internally contains information about + // `#[link(wasm_import_module = "...")]` for example. + let native_libs = tcx.native_libraries(cnum); + + let def_id_to_native_lib = native_libs + .iter() + .filter_map(|lib| lib.foreign_module.map(|id| (id, lib))) + .collect::>(); + + let mut ret = FxHashMap::default(); + for (def_id, lib) in tcx.foreign_modules(cnum).iter() { + let module = def_id_to_native_lib.get(&def_id).and_then(|s| s.wasm_import_module); + let Some(module) = module else { continue }; + ret.extend(lib.foreign_items.iter().map(|id| { + assert_eq!(id.krate, cnum); + (*id, module.to_string()) + })); + } + + ret +} diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs new file mode 100644 index 00000000000..1b5ad87107a --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -0,0 +1,2015 @@ +use super::link::{self, ensure_removed}; +use super::lto::{self, SerializedModule}; +use super::symbol_export::symbol_name_for_instance_in_crate; + +use crate::{ + CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind, +}; + +use crate::traits::*; +use jobserver::{Acquired, Client}; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::memmap::Mmap; +use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_data_structures::profiling::TimingGuard; +use rustc_data_structures::profiling::VerboseTimingGuard; +use rustc_data_structures::sync::Lrc; +use rustc_errors::emitter::Emitter; +use rustc_errors::{DiagnosticId, FatalError, Handler, Level}; +use rustc_fs_util::link_or_copy; +use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; +use rustc_incremental::{ + copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess, +}; +use rustc_metadata::EncodedMetadata; +use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::middle::exported_symbols::SymbolExportInfo; +use rustc_middle::ty::TyCtxt; +use rustc_session::cgu_reuse_tracker::CguReuseTracker; +use rustc_session::config::{self, CrateType, Lto, OutputFilenames, OutputType}; +use rustc_session::config::{Passes, SwitchWithOptPath}; +use rustc_session::Session; +use rustc_span::source_map::SourceMap; +use rustc_span::symbol::sym; +use rustc_span::{BytePos, FileName, InnerSpan, Pos, Span}; +use rustc_target::spec::{MergeFunctions, SanitizerSet}; + +use std::any::Any; +use std::fs; +use std::io; +use std::marker::PhantomData; +use std::mem; +use std::path::{Path, PathBuf}; +use std::str; +use std::sync::mpsc::{channel, Receiver, Sender}; +use std::sync::Arc; +use std::thread; + +const PRE_LTO_BC_EXT: &str = "pre-lto.bc"; + +/// What kind of object file to emit. +#[derive(Clone, Copy, PartialEq)] +pub enum EmitObj { + // No object file. + None, + + // Just uncompressed llvm bitcode. Provides easy compatibility with + // emscripten's ecc compiler, when used as the linker. + Bitcode, + + // Object code, possibly augmented with a bitcode section. + ObjectCode(BitcodeSection), +} + +/// What kind of llvm bitcode section to embed in an object file. +#[derive(Clone, Copy, PartialEq)] +pub enum BitcodeSection { + // No bitcode section. + None, + + // A full, uncompressed bitcode section. + Full, +} + +/// Module-specific configuration for `optimize_and_codegen`. +pub struct ModuleConfig { + /// Names of additional optimization passes to run. + pub passes: Vec, + /// Some(level) to optimize at a certain level, or None to run + /// absolutely no optimizations (used for the metadata module). + pub opt_level: Option, + + /// Some(level) to optimize binary size, or None to not affect program size. + pub opt_size: Option, + + pub pgo_gen: SwitchWithOptPath, + pub pgo_use: Option, + pub pgo_sample_use: Option, + pub debug_info_for_profiling: bool, + pub instrument_coverage: bool, + pub instrument_gcov: bool, + + pub sanitizer: SanitizerSet, + pub sanitizer_recover: SanitizerSet, + pub sanitizer_memory_track_origins: usize, + + // Flags indicating which outputs to produce. + pub emit_pre_lto_bc: bool, + pub emit_no_opt_bc: bool, + pub emit_bc: bool, + pub emit_ir: bool, + pub emit_asm: bool, + pub emit_obj: EmitObj, + pub emit_thin_lto: bool, + pub bc_cmdline: String, + + // Miscellaneous flags. These are mostly copied from command-line + // options. + pub verify_llvm_ir: bool, + pub no_prepopulate_passes: bool, + pub no_builtins: bool, + pub time_module: bool, + pub vectorize_loop: bool, + pub vectorize_slp: bool, + pub merge_functions: bool, + pub inline_threshold: Option, + pub new_llvm_pass_manager: Option, + pub emit_lifetime_markers: bool, + pub llvm_plugins: Vec, +} + +impl ModuleConfig { + fn new( + kind: ModuleKind, + sess: &Session, + no_builtins: bool, + is_compiler_builtins: bool, + ) -> ModuleConfig { + // If it's a regular module, use `$regular`, otherwise use `$other`. + // `$regular` and `$other` are evaluated lazily. + macro_rules! if_regular { + ($regular: expr, $other: expr) => { + if let ModuleKind::Regular = kind { $regular } else { $other } + }; + } + + let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None); + + let save_temps = sess.opts.cg.save_temps; + + let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe) + || match kind { + ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object), + ModuleKind::Allocator => false, + ModuleKind::Metadata => sess.opts.output_types.contains_key(&OutputType::Metadata), + }; + + let emit_obj = if !should_emit_obj { + EmitObj::None + } else if sess.target.obj_is_bitcode + || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins) + { + // This case is selected if the target uses objects as bitcode, or + // if linker plugin LTO is enabled. In the linker plugin LTO case + // the assumption is that the final link-step will read the bitcode + // and convert it to object code. This may be done by either the + // native linker or rustc itself. + // + // Note, however, that the linker-plugin-lto requested here is + // explicitly ignored for `#![no_builtins]` crates. These crates are + // specifically ignored by rustc's LTO passes and wouldn't work if + // loaded into the linker. These crates define symbols that LLVM + // lowers intrinsics to, and these symbol dependencies aren't known + // until after codegen. As a result any crate marked + // `#![no_builtins]` is assumed to not participate in LTO and + // instead goes on to generate object code. + EmitObj::Bitcode + } else if need_bitcode_in_object(sess) { + EmitObj::ObjectCode(BitcodeSection::Full) + } else { + EmitObj::ObjectCode(BitcodeSection::None) + }; + + ModuleConfig { + passes: if_regular!(sess.opts.cg.passes.clone(), vec![]), + + opt_level: opt_level_and_size, + opt_size: opt_level_and_size, + + pgo_gen: if_regular!( + sess.opts.cg.profile_generate.clone(), + SwitchWithOptPath::Disabled + ), + pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None), + pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None), + debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling, + instrument_coverage: if_regular!(sess.instrument_coverage(), false), + instrument_gcov: if_regular!( + // compiler_builtins overrides the codegen-units settings, + // which is incompatible with -Zprofile which requires that + // only a single codegen unit is used per crate. + sess.opts.unstable_opts.profile && !is_compiler_builtins, + false + ), + + sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()), + sanitizer_recover: if_regular!( + sess.opts.unstable_opts.sanitizer_recover, + SanitizerSet::empty() + ), + sanitizer_memory_track_origins: if_regular!( + sess.opts.unstable_opts.sanitizer_memory_track_origins, + 0 + ), + + emit_pre_lto_bc: if_regular!( + save_temps || need_pre_lto_bitcode_for_incr_comp(sess), + false + ), + emit_no_opt_bc: if_regular!(save_temps, false), + emit_bc: if_regular!( + save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode), + save_temps + ), + emit_ir: if_regular!( + sess.opts.output_types.contains_key(&OutputType::LlvmAssembly), + false + ), + emit_asm: if_regular!( + sess.opts.output_types.contains_key(&OutputType::Assembly), + false + ), + emit_obj, + emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto, + bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(), + + verify_llvm_ir: sess.verify_llvm_ir(), + no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes, + no_builtins: no_builtins || sess.target.no_builtins, + + // Exclude metadata and allocator modules from time_passes output, + // since they throw off the "LLVM passes" measurement. + time_module: if_regular!(true, false), + + // Copy what clang does by turning on loop vectorization at O2 and + // slp vectorization at O3. + vectorize_loop: !sess.opts.cg.no_vectorize_loops + && (sess.opts.optimize == config::OptLevel::Default + || sess.opts.optimize == config::OptLevel::Aggressive), + vectorize_slp: !sess.opts.cg.no_vectorize_slp + && sess.opts.optimize == config::OptLevel::Aggressive, + + // Some targets (namely, NVPTX) interact badly with the + // MergeFunctions pass. This is because MergeFunctions can generate + // new function calls which may interfere with the target calling + // convention; e.g. for the NVPTX target, PTX kernels should not + // call other PTX kernels. MergeFunctions can also be configured to + // generate aliases instead, but aliases are not supported by some + // backends (again, NVPTX). Therefore, allow targets to opt out of + // the MergeFunctions pass, but otherwise keep the pass enabled (at + // O2 and O3) since it can be useful for reducing code size. + merge_functions: match sess + .opts + .unstable_opts + .merge_functions + .unwrap_or(sess.target.merge_functions) + { + MergeFunctions::Disabled => false, + MergeFunctions::Trampolines | MergeFunctions::Aliases => { + sess.opts.optimize == config::OptLevel::Default + || sess.opts.optimize == config::OptLevel::Aggressive + } + }, + + inline_threshold: sess.opts.cg.inline_threshold, + new_llvm_pass_manager: sess.opts.unstable_opts.new_llvm_pass_manager, + emit_lifetime_markers: sess.emit_lifetime_markers(), + llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]), + } + } + + pub fn bitcode_needed(&self) -> bool { + self.emit_bc + || self.emit_obj == EmitObj::Bitcode + || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) + } +} + +/// Configuration passed to the function returned by the `target_machine_factory`. +pub struct TargetMachineFactoryConfig { + /// Split DWARF is enabled in LLVM by checking that `TM.MCOptions.SplitDwarfFile` isn't empty, + /// so the path to the dwarf object has to be provided when we create the target machine. + /// This can be ignored by backends which do not need it for their Split DWARF support. + pub split_dwarf_file: Option, +} + +impl TargetMachineFactoryConfig { + pub fn new( + cgcx: &CodegenContext, + module_name: &str, + ) -> TargetMachineFactoryConfig { + let split_dwarf_file = if cgcx.target_can_use_split_dwarf { + cgcx.output_filenames.split_dwarf_path( + cgcx.split_debuginfo, + cgcx.split_dwarf_kind, + Some(module_name), + ) + } else { + None + }; + TargetMachineFactoryConfig { split_dwarf_file } + } +} + +pub type TargetMachineFactoryFn = Arc< + dyn Fn(TargetMachineFactoryConfig) -> Result<::TargetMachine, String> + + Send + + Sync, +>; + +pub type ExportedSymbols = FxHashMap>>; + +/// Additional resources used by optimize_and_codegen (not module specific) +#[derive(Clone)] +pub struct CodegenContext { + // Resources needed when running LTO + pub backend: B, + pub prof: SelfProfilerRef, + pub lto: Lto, + pub save_temps: bool, + pub fewer_names: bool, + pub time_trace: bool, + pub exported_symbols: Option>, + pub opts: Arc, + pub crate_types: Vec, + pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>, + pub output_filenames: Arc, + pub regular_module_config: Arc, + pub metadata_module_config: Arc, + pub allocator_module_config: Arc, + pub tm_factory: TargetMachineFactoryFn, + pub msvc_imps_needed: bool, + pub is_pe_coff: bool, + pub target_can_use_split_dwarf: bool, + pub target_pointer_width: u32, + pub target_arch: String, + pub debuginfo: config::DebugInfo, + pub split_debuginfo: rustc_target::spec::SplitDebuginfo, + pub split_dwarf_kind: rustc_session::config::SplitDwarfKind, + + // Number of cgus excluding the allocator/metadata modules + pub total_cgus: usize, + // Handler to use for diagnostics produced during codegen. + pub diag_emitter: SharedEmitter, + // LLVM optimizations for which we want to print remarks. + pub remark: Passes, + // Worker thread number + pub worker: usize, + // The incremental compilation session directory, or None if we are not + // compiling incrementally + pub incr_comp_session_dir: Option, + // Used to update CGU re-use information during the thinlto phase. + pub cgu_reuse_tracker: CguReuseTracker, + // Channel back to the main control thread to send messages to + pub coordinator_send: Sender>, +} + +impl CodegenContext { + pub fn create_diag_handler(&self) -> Handler { + Handler::with_emitter(true, None, Box::new(self.diag_emitter.clone())) + } + + pub fn config(&self, kind: ModuleKind) -> &ModuleConfig { + match kind { + ModuleKind::Regular => &self.regular_module_config, + ModuleKind::Metadata => &self.metadata_module_config, + ModuleKind::Allocator => &self.allocator_module_config, + } + } +} + +fn generate_lto_work( + cgcx: &CodegenContext, + needs_fat_lto: Vec>, + needs_thin_lto: Vec<(String, B::ThinBuffer)>, + import_only_modules: Vec<(SerializedModule, WorkProduct)>, +) -> Vec<(WorkItem, u64)> { + let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work"); + + let (lto_modules, copy_jobs) = if !needs_fat_lto.is_empty() { + assert!(needs_thin_lto.is_empty()); + let lto_module = + B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise()); + (vec![lto_module], vec![]) + } else { + assert!(needs_fat_lto.is_empty()); + B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules).unwrap_or_else(|e| e.raise()) + }; + + lto_modules + .into_iter() + .map(|module| { + let cost = module.cost(); + (WorkItem::LTO(module), cost) + }) + .chain(copy_jobs.into_iter().map(|wp| { + ( + WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen { + name: wp.cgu_name.clone(), + source: wp, + }), + 0, + ) + })) + .collect() +} + +pub struct CompiledModules { + pub modules: Vec, + pub allocator_module: Option, +} + +fn need_bitcode_in_object(sess: &Session) -> bool { + let requested_for_rlib = sess.opts.cg.embed_bitcode + && sess.crate_types().contains(&CrateType::Rlib) + && sess.opts.output_types.contains_key(&OutputType::Exe); + let forced_by_target = sess.target.forces_embed_bitcode; + requested_for_rlib || forced_by_target +} + +fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool { + if sess.opts.incremental.is_none() { + return false; + } + + match sess.lto() { + Lto::No => false, + Lto::Fat | Lto::Thin | Lto::ThinLocal => true, + } +} + +pub fn start_async_codegen( + backend: B, + tcx: TyCtxt<'_>, + target_cpu: String, + metadata: EncodedMetadata, + metadata_module: Option, + total_cgus: usize, +) -> OngoingCodegen { + let (coordinator_send, coordinator_receive) = channel(); + let sess = tcx.sess; + + let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID); + let no_builtins = tcx.sess.contains_name(crate_attrs, sym::no_builtins); + let is_compiler_builtins = tcx.sess.contains_name(crate_attrs, sym::compiler_builtins); + + let crate_info = CrateInfo::new(tcx, target_cpu); + + let regular_config = + ModuleConfig::new(ModuleKind::Regular, sess, no_builtins, is_compiler_builtins); + let metadata_config = + ModuleConfig::new(ModuleKind::Metadata, sess, no_builtins, is_compiler_builtins); + let allocator_config = + ModuleConfig::new(ModuleKind::Allocator, sess, no_builtins, is_compiler_builtins); + + let (shared_emitter, shared_emitter_main) = SharedEmitter::new(); + let (codegen_worker_send, codegen_worker_receive) = channel(); + + let coordinator_thread = start_executing_work( + backend.clone(), + tcx, + &crate_info, + shared_emitter, + codegen_worker_send, + coordinator_receive, + total_cgus, + sess.jobserver.clone(), + Arc::new(regular_config), + Arc::new(metadata_config), + Arc::new(allocator_config), + coordinator_send.clone(), + ); + + OngoingCodegen { + backend, + metadata, + metadata_module, + crate_info, + + codegen_worker_receive, + shared_emitter_main, + coordinator: Coordinator { + sender: coordinator_send, + future: Some(coordinator_thread), + phantom: PhantomData, + }, + output_filenames: tcx.output_filenames(()).clone(), + } +} + +fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( + sess: &Session, + compiled_modules: &CompiledModules, +) -> FxHashMap { + let mut work_products = FxHashMap::default(); + + if sess.opts.incremental.is_none() { + return work_products; + } + + let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir"); + + for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) { + let mut files = Vec::new(); + if let Some(object_file_path) = &module.object { + files.push(("o", object_file_path.as_path())); + } + if let Some(dwarf_object_file_path) = &module.dwarf_object { + files.push(("dwo", dwarf_object_file_path.as_path())); + } + + if let Some((id, product)) = + copy_cgu_workproduct_to_incr_comp_cache_dir(sess, &module.name, files.as_slice()) + { + work_products.insert(id, product); + } + } + + work_products +} + +fn produce_final_output_artifacts( + sess: &Session, + compiled_modules: &CompiledModules, + crate_output: &OutputFilenames, +) { + let mut user_wants_bitcode = false; + let mut user_wants_objects = false; + + // Produce final compile outputs. + let copy_gracefully = |from: &Path, to: &Path| { + if let Err(e) = fs::copy(from, to) { + sess.err(&format!("could not copy {:?} to {:?}: {}", from, to, e)); + } + }; + + let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| { + if compiled_modules.modules.len() == 1 { + // 1) Only one codegen unit. In this case it's no difficulty + // to copy `foo.0.x` to `foo.x`. + let module_name = Some(&compiled_modules.modules[0].name[..]); + let path = crate_output.temp_path(output_type, module_name); + copy_gracefully(&path, &crate_output.path(output_type)); + if !sess.opts.cg.save_temps && !keep_numbered { + // The user just wants `foo.x`, not `foo.#module-name#.x`. + ensure_removed(sess.diagnostic(), &path); + } + } else { + let ext = crate_output + .temp_path(output_type, None) + .extension() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + + if crate_output.outputs.contains_key(&output_type) { + // 2) Multiple codegen units, with `--emit foo=some_name`. We have + // no good solution for this case, so warn the user. + sess.warn(&format!( + "ignoring emit path because multiple .{} files \ + were produced", + ext + )); + } else if crate_output.single_output_file.is_some() { + // 3) Multiple codegen units, with `-o some_name`. We have + // no good solution for this case, so warn the user. + sess.warn(&format!( + "ignoring -o because multiple .{} files \ + were produced", + ext + )); + } else { + // 4) Multiple codegen units, but no explicit name. We + // just leave the `foo.0.x` files in place. + // (We don't have to do any work in this case.) + } + } + }; + + // Flag to indicate whether the user explicitly requested bitcode. + // Otherwise, we produced it only as a temporary output, and will need + // to get rid of it. + for output_type in crate_output.outputs.keys() { + match *output_type { + OutputType::Bitcode => { + user_wants_bitcode = true; + // Copy to .bc, but always keep the .0.bc. There is a later + // check to figure out if we should delete .0.bc files, or keep + // them for making an rlib. + copy_if_one_unit(OutputType::Bitcode, true); + } + OutputType::LlvmAssembly => { + copy_if_one_unit(OutputType::LlvmAssembly, false); + } + OutputType::Assembly => { + copy_if_one_unit(OutputType::Assembly, false); + } + OutputType::Object => { + user_wants_objects = true; + copy_if_one_unit(OutputType::Object, true); + } + OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {} + } + } + + // Clean up unwanted temporary files. + + // We create the following files by default: + // - #crate#.#module-name#.bc + // - #crate#.#module-name#.o + // - #crate#.crate.metadata.bc + // - #crate#.crate.metadata.o + // - #crate#.o (linked from crate.##.o) + // - #crate#.bc (copied from crate.##.bc) + // We may create additional files if requested by the user (through + // `-C save-temps` or `--emit=` flags). + + if !sess.opts.cg.save_temps { + // Remove the temporary .#module-name#.o objects. If the user didn't + // explicitly request bitcode (with --emit=bc), and the bitcode is not + // needed for building an rlib, then we must remove .#module-name#.bc as + // well. + + // Specific rules for keeping .#module-name#.bc: + // - If the user requested bitcode (`user_wants_bitcode`), and + // codegen_units > 1, then keep it. + // - If the user requested bitcode but codegen_units == 1, then we + // can toss .#module-name#.bc because we copied it to .bc earlier. + // - If we're not building an rlib and the user didn't request + // bitcode, then delete .#module-name#.bc. + // If you change how this works, also update back::link::link_rlib, + // where .#module-name#.bc files are (maybe) deleted after making an + // rlib. + let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe); + + let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units() > 1; + + let keep_numbered_objects = + needs_crate_object || (user_wants_objects && sess.codegen_units() > 1); + + for module in compiled_modules.modules.iter() { + if let Some(ref path) = module.object { + if !keep_numbered_objects { + ensure_removed(sess.diagnostic(), path); + } + } + + if let Some(ref path) = module.dwarf_object { + if !keep_numbered_objects { + ensure_removed(sess.diagnostic(), path); + } + } + + if let Some(ref path) = module.bytecode { + if !keep_numbered_bitcode { + ensure_removed(sess.diagnostic(), path); + } + } + } + + if !user_wants_bitcode { + if let Some(ref allocator_module) = compiled_modules.allocator_module { + if let Some(ref path) = allocator_module.bytecode { + ensure_removed(sess.diagnostic(), path); + } + } + } + } + + // We leave the following files around by default: + // - #crate#.o + // - #crate#.crate.metadata.o + // - #crate#.bc + // These are used in linking steps and will be cleaned up afterward. +} + +pub enum WorkItem { + /// Optimize a newly codegened, totally unoptimized module. + Optimize(ModuleCodegen), + /// Copy the post-LTO artifacts from the incremental cache to the output + /// directory. + CopyPostLtoArtifacts(CachedModuleCodegen), + /// Performs (Thin)LTO on the given module. + LTO(lto::LtoModuleCodegen), +} + +impl WorkItem { + pub fn module_kind(&self) -> ModuleKind { + match *self { + WorkItem::Optimize(ref m) => m.kind, + WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular, + } + } + + fn start_profiling<'a>(&self, cgcx: &'a CodegenContext) -> TimingGuard<'a> { + match *self { + WorkItem::Optimize(ref m) => { + cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name) + } + WorkItem::CopyPostLtoArtifacts(ref m) => cgcx + .prof + .generic_activity_with_arg("codegen_copy_artifacts_from_incr_cache", &*m.name), + WorkItem::LTO(ref m) => { + cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()) + } + } + } + + /// Generate a short description of this work item suitable for use as a thread name. + fn short_description(&self) -> String { + // `pthread_setname()` on *nix is limited to 15 characters and longer names are ignored. + // Use very short descriptions in this case to maximize the space available for the module name. + // Windows does not have that limitation so use slightly more descriptive names there. + match self { + WorkItem::Optimize(m) => { + #[cfg(windows)] + return format!("optimize module {}", m.name); + #[cfg(not(windows))] + return format!("opt {}", m.name); + } + WorkItem::CopyPostLtoArtifacts(m) => { + #[cfg(windows)] + return format!("copy LTO artifacts for {}", m.name); + #[cfg(not(windows))] + return format!("copy {}", m.name); + } + WorkItem::LTO(m) => { + #[cfg(windows)] + return format!("LTO module {}", m.name()); + #[cfg(not(windows))] + return format!("LTO {}", m.name()); + } + } + } +} + +enum WorkItemResult { + Compiled(CompiledModule), + NeedsLink(ModuleCodegen), + NeedsFatLTO(FatLTOInput), + NeedsThinLTO(String, B::ThinBuffer), +} + +pub enum FatLTOInput { + Serialized { name: String, buffer: B::ModuleBuffer }, + InMemory(ModuleCodegen), +} + +fn execute_work_item( + cgcx: &CodegenContext, + work_item: WorkItem, +) -> Result, FatalError> { + let module_config = cgcx.config(work_item.module_kind()); + + match work_item { + WorkItem::Optimize(module) => execute_optimize_work_item(cgcx, module, module_config), + WorkItem::CopyPostLtoArtifacts(module) => { + Ok(execute_copy_from_cache_work_item(cgcx, module, module_config)) + } + WorkItem::LTO(module) => execute_lto_work_item(cgcx, module, module_config), + } +} + +// Actual LTO type we end up choosing based on multiple factors. +pub enum ComputedLtoType { + No, + Thin, + Fat, +} + +pub fn compute_per_cgu_lto_type( + sess_lto: &Lto, + opts: &config::Options, + sess_crate_types: &[CrateType], + module_kind: ModuleKind, +) -> ComputedLtoType { + // Metadata modules never participate in LTO regardless of the lto + // settings. + if module_kind == ModuleKind::Metadata { + return ComputedLtoType::No; + } + + // If the linker does LTO, we don't have to do it. Note that we + // keep doing full LTO, if it is requested, as not to break the + // assumption that the output will be a single module. + let linker_does_lto = opts.cg.linker_plugin_lto.enabled(); + + // When we're automatically doing ThinLTO for multi-codegen-unit + // builds we don't actually want to LTO the allocator modules if + // it shows up. This is due to various linker shenanigans that + // we'll encounter later. + let is_allocator = module_kind == ModuleKind::Allocator; + + // We ignore a request for full crate graph LTO if the crate type + // is only an rlib, as there is no full crate graph to process, + // that'll happen later. + // + // This use case currently comes up primarily for targets that + // require LTO so the request for LTO is always unconditionally + // passed down to the backend, but we don't actually want to do + // anything about it yet until we've got a final product. + let is_rlib = sess_crate_types.len() == 1 && sess_crate_types[0] == CrateType::Rlib; + + match sess_lto { + Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin, + Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin, + Lto::Fat if !is_rlib => ComputedLtoType::Fat, + _ => ComputedLtoType::No, + } +} + +fn execute_optimize_work_item( + cgcx: &CodegenContext, + module: ModuleCodegen, + module_config: &ModuleConfig, +) -> Result, FatalError> { + let diag_handler = cgcx.create_diag_handler(); + + unsafe { + B::optimize(cgcx, &diag_handler, &module, module_config)?; + } + + // After we've done the initial round of optimizations we need to + // decide whether to synchronously codegen this module or ship it + // back to the coordinator thread for further LTO processing (which + // has to wait for all the initial modules to be optimized). + + let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind); + + // If we're doing some form of incremental LTO then we need to be sure to + // save our module to disk first. + let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc { + let filename = pre_lto_bitcode_filename(&module.name); + cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename)) + } else { + None + }; + + match lto_type { + ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config), + ComputedLtoType::Thin => { + let (name, thin_buffer) = B::prepare_thin(module); + if let Some(path) = bitcode { + fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| { + panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); + }); + } + Ok(WorkItemResult::NeedsThinLTO(name, thin_buffer)) + } + ComputedLtoType::Fat => match bitcode { + Some(path) => { + let (name, buffer) = B::serialize_module(module); + fs::write(&path, buffer.data()).unwrap_or_else(|e| { + panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); + }); + Ok(WorkItemResult::NeedsFatLTO(FatLTOInput::Serialized { name, buffer })) + } + None => Ok(WorkItemResult::NeedsFatLTO(FatLTOInput::InMemory(module))), + }, + } +} + +fn execute_copy_from_cache_work_item( + cgcx: &CodegenContext, + module: CachedModuleCodegen, + module_config: &ModuleConfig, +) -> WorkItemResult { + assert!(module_config.emit_obj != EmitObj::None); + + let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap(); + + let load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| { + let source_file = in_incr_comp_dir(&incr_comp_session_dir, saved_path); + debug!( + "copying pre-existing module `{}` from {:?} to {}", + module.name, + source_file, + output_path.display() + ); + match link_or_copy(&source_file, &output_path) { + Ok(_) => Some(output_path), + Err(err) => { + let diag_handler = cgcx.create_diag_handler(); + diag_handler.err(&format!( + "unable to copy {} to {}: {}", + source_file.display(), + output_path.display(), + err + )); + None + } + } + }; + + let object = load_from_incr_comp_dir( + cgcx.output_filenames.temp_path(OutputType::Object, Some(&module.name)), + &module.source.saved_files.get("o").expect("no saved object file in work product"), + ); + let dwarf_object = + module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| { + let dwarf_obj_out = cgcx + .output_filenames + .split_dwarf_path(cgcx.split_debuginfo, cgcx.split_dwarf_kind, Some(&module.name)) + .expect( + "saved dwarf object in work product but `split_dwarf_path` returned `None`", + ); + load_from_incr_comp_dir(dwarf_obj_out, &saved_dwarf_object_file) + }); + + WorkItemResult::Compiled(CompiledModule { + name: module.name, + kind: ModuleKind::Regular, + object, + dwarf_object, + bytecode: None, + }) +} + +fn execute_lto_work_item( + cgcx: &CodegenContext, + module: lto::LtoModuleCodegen, + module_config: &ModuleConfig, +) -> Result, FatalError> { + let module = unsafe { module.optimize(cgcx)? }; + finish_intra_module_work(cgcx, module, module_config) +} + +fn finish_intra_module_work( + cgcx: &CodegenContext, + module: ModuleCodegen, + module_config: &ModuleConfig, +) -> Result, FatalError> { + let diag_handler = cgcx.create_diag_handler(); + + if !cgcx.opts.unstable_opts.combine_cgu + || module.kind == ModuleKind::Metadata + || module.kind == ModuleKind::Allocator + { + let module = unsafe { B::codegen(cgcx, &diag_handler, module, module_config)? }; + Ok(WorkItemResult::Compiled(module)) + } else { + Ok(WorkItemResult::NeedsLink(module)) + } +} + +pub enum Message { + Token(io::Result), + NeedsFatLTO { + result: FatLTOInput, + worker_id: usize, + }, + NeedsThinLTO { + name: String, + thin_buffer: B::ThinBuffer, + worker_id: usize, + }, + NeedsLink { + module: ModuleCodegen, + worker_id: usize, + }, + Done { + result: Result>, + worker_id: usize, + }, + CodegenDone { + llvm_work_item: WorkItem, + cost: u64, + }, + AddImportOnlyModule { + module_data: SerializedModule, + work_product: WorkProduct, + }, + CodegenComplete, + CodegenItem, + CodegenAborted, +} + +struct Diagnostic { + msg: String, + code: Option, + lvl: Level, +} + +#[derive(PartialEq, Clone, Copy, Debug)] +enum MainThreadWorkerState { + Idle, + Codegenning, + LLVMing, +} + +fn start_executing_work( + backend: B, + tcx: TyCtxt<'_>, + crate_info: &CrateInfo, + shared_emitter: SharedEmitter, + codegen_worker_send: Sender>, + coordinator_receive: Receiver>, + total_cgus: usize, + jobserver: Client, + regular_config: Arc, + metadata_config: Arc, + allocator_config: Arc, + tx_to_llvm_workers: Sender>, +) -> thread::JoinHandle> { + let coordinator_send = tx_to_llvm_workers; + let sess = tcx.sess; + + // Compute the set of symbols we need to retain when doing LTO (if we need to) + let exported_symbols = { + let mut exported_symbols = FxHashMap::default(); + + let copy_symbols = |cnum| { + let symbols = tcx + .exported_symbols(cnum) + .iter() + .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl)) + .collect(); + Arc::new(symbols) + }; + + match sess.lto() { + Lto::No => None, + Lto::ThinLocal => { + exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE)); + Some(Arc::new(exported_symbols)) + } + Lto::Fat | Lto::Thin => { + exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE)); + for &cnum in tcx.crates(()).iter() { + exported_symbols.insert(cnum, copy_symbols(cnum)); + } + Some(Arc::new(exported_symbols)) + } + } + }; + + // First up, convert our jobserver into a helper thread so we can use normal + // mpsc channels to manage our messages and such. + // After we've requested tokens then we'll, when we can, + // get tokens on `coordinator_receive` which will + // get managed in the main loop below. + let coordinator_send2 = coordinator_send.clone(); + let helper = jobserver + .into_helper_thread(move |token| { + drop(coordinator_send2.send(Box::new(Message::Token::(token)))); + }) + .expect("failed to spawn helper thread"); + + let mut each_linked_rlib_for_lto = Vec::new(); + drop(link::each_linked_rlib(crate_info, &mut |cnum, path| { + if link::ignored_for_lto(sess, crate_info, cnum) { + return; + } + each_linked_rlib_for_lto.push((cnum, path.to_path_buf())); + })); + + let ol = + if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { + // If we know that we won’t be doing codegen, create target machines without optimisation. + config::OptLevel::No + } else { + tcx.backend_optimization_level(()) + }; + let backend_features = tcx.global_backend_features(()); + let cgcx = CodegenContext:: { + backend: backend.clone(), + crate_types: sess.crate_types().to_vec(), + each_linked_rlib_for_lto, + lto: sess.lto(), + fewer_names: sess.fewer_names(), + save_temps: sess.opts.cg.save_temps, + time_trace: sess.opts.unstable_opts.llvm_time_trace, + opts: Arc::new(sess.opts.clone()), + prof: sess.prof.clone(), + exported_symbols, + remark: sess.opts.cg.remark.clone(), + worker: 0, + incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()), + cgu_reuse_tracker: sess.cgu_reuse_tracker.clone(), + coordinator_send, + diag_emitter: shared_emitter.clone(), + output_filenames: tcx.output_filenames(()).clone(), + regular_module_config: regular_config, + metadata_module_config: metadata_config, + allocator_module_config: allocator_config, + tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features), + total_cgus, + msvc_imps_needed: msvc_imps_needed(tcx), + is_pe_coff: tcx.sess.target.is_like_windows, + target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(), + target_pointer_width: tcx.sess.target.pointer_width, + target_arch: tcx.sess.target.arch.to_string(), + debuginfo: tcx.sess.opts.debuginfo, + split_debuginfo: tcx.sess.split_debuginfo(), + split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind, + }; + + // This is the "main loop" of parallel work happening for parallel codegen. + // It's here that we manage parallelism, schedule work, and work with + // messages coming from clients. + // + // There are a few environmental pre-conditions that shape how the system + // is set up: + // + // - Error reporting only can happen on the main thread because that's the + // only place where we have access to the compiler `Session`. + // - LLVM work can be done on any thread. + // - Codegen can only happen on the main thread. + // - Each thread doing substantial work must be in possession of a `Token` + // from the `Jobserver`. + // - The compiler process always holds one `Token`. Any additional `Tokens` + // have to be requested from the `Jobserver`. + // + // Error Reporting + // =============== + // The error reporting restriction is handled separately from the rest: We + // set up a `SharedEmitter` the holds an open channel to the main thread. + // When an error occurs on any thread, the shared emitter will send the + // error message to the receiver main thread (`SharedEmitterMain`). The + // main thread will periodically query this error message queue and emit + // any error messages it has received. It might even abort compilation if + // has received a fatal error. In this case we rely on all other threads + // being torn down automatically with the main thread. + // Since the main thread will often be busy doing codegen work, error + // reporting will be somewhat delayed, since the message queue can only be + // checked in between to work packages. + // + // Work Processing Infrastructure + // ============================== + // The work processing infrastructure knows three major actors: + // + // - the coordinator thread, + // - the main thread, and + // - LLVM worker threads + // + // The coordinator thread is running a message loop. It instructs the main + // thread about what work to do when, and it will spawn off LLVM worker + // threads as open LLVM WorkItems become available. + // + // The job of the main thread is to codegen CGUs into LLVM work package + // (since the main thread is the only thread that can do this). The main + // thread will block until it receives a message from the coordinator, upon + // which it will codegen one CGU, send it to the coordinator and block + // again. This way the coordinator can control what the main thread is + // doing. + // + // The coordinator keeps a queue of LLVM WorkItems, and when a `Token` is + // available, it will spawn off a new LLVM worker thread and let it process + // that a WorkItem. When a LLVM worker thread is done with its WorkItem, + // it will just shut down, which also frees all resources associated with + // the given LLVM module, and sends a message to the coordinator that the + // has been completed. + // + // Work Scheduling + // =============== + // The scheduler's goal is to minimize the time it takes to complete all + // work there is, however, we also want to keep memory consumption low + // if possible. These two goals are at odds with each other: If memory + // consumption were not an issue, we could just let the main thread produce + // LLVM WorkItems at full speed, assuring maximal utilization of + // Tokens/LLVM worker threads. However, since codegen is usually faster + // than LLVM processing, the queue of LLVM WorkItems would fill up and each + // WorkItem potentially holds on to a substantial amount of memory. + // + // So the actual goal is to always produce just enough LLVM WorkItems as + // not to starve our LLVM worker threads. That means, once we have enough + // WorkItems in our queue, we can block the main thread, so it does not + // produce more until we need them. + // + // Doing LLVM Work on the Main Thread + // ---------------------------------- + // Since the main thread owns the compiler processes implicit `Token`, it is + // wasteful to keep it blocked without doing any work. Therefore, what we do + // in this case is: We spawn off an additional LLVM worker thread that helps + // reduce the queue. The work it is doing corresponds to the implicit + // `Token`. The coordinator will mark the main thread as being busy with + // LLVM work. (The actual work happens on another OS thread but we just care + // about `Tokens`, not actual threads). + // + // When any LLVM worker thread finishes while the main thread is marked as + // "busy with LLVM work", we can do a little switcheroo: We give the Token + // of the just finished thread to the LLVM worker thread that is working on + // behalf of the main thread's implicit Token, thus freeing up the main + // thread again. The coordinator can then again decide what the main thread + // should do. This allows the coordinator to make decisions at more points + // in time. + // + // Striking a Balance between Throughput and Memory Consumption + // ------------------------------------------------------------ + // Since our two goals, (1) use as many Tokens as possible and (2) keep + // memory consumption as low as possible, are in conflict with each other, + // we have to find a trade off between them. Right now, the goal is to keep + // all workers busy, which means that no worker should find the queue empty + // when it is ready to start. + // How do we do achieve this? Good question :) We actually never know how + // many `Tokens` are potentially available so it's hard to say how much to + // fill up the queue before switching the main thread to LLVM work. Also we + // currently don't have a means to estimate how long a running LLVM worker + // will still be busy with it's current WorkItem. However, we know the + // maximal count of available Tokens that makes sense (=the number of CPU + // cores), so we can take a conservative guess. The heuristic we use here + // is implemented in the `queue_full_enough()` function. + // + // Some Background on Jobservers + // ----------------------------- + // It's worth also touching on the management of parallelism here. We don't + // want to just spawn a thread per work item because while that's optimal + // parallelism it may overload a system with too many threads or violate our + // configuration for the maximum amount of cpu to use for this process. To + // manage this we use the `jobserver` crate. + // + // Job servers are an artifact of GNU make and are used to manage + // parallelism between processes. A jobserver is a glorified IPC semaphore + // basically. Whenever we want to run some work we acquire the semaphore, + // and whenever we're done with that work we release the semaphore. In this + // manner we can ensure that the maximum number of parallel workers is + // capped at any one point in time. + // + // LTO and the coordinator thread + // ------------------------------ + // + // The final job the coordinator thread is responsible for is managing LTO + // and how that works. When LTO is requested what we'll to is collect all + // optimized LLVM modules into a local vector on the coordinator. Once all + // modules have been codegened and optimized we hand this to the `lto` + // module for further optimization. The `lto` module will return back a list + // of more modules to work on, which the coordinator will continue to spawn + // work for. + // + // Each LLVM module is automatically sent back to the coordinator for LTO if + // necessary. There's already optimizations in place to avoid sending work + // back to the coordinator if LTO isn't requested. + return B::spawn_thread(cgcx.time_trace, move || { + let mut worker_id_counter = 0; + let mut free_worker_ids = Vec::new(); + let mut get_worker_id = |free_worker_ids: &mut Vec| { + if let Some(id) = free_worker_ids.pop() { + id + } else { + let id = worker_id_counter; + worker_id_counter += 1; + id + } + }; + + // This is where we collect codegen units that have gone all the way + // through codegen and LLVM. + let mut compiled_modules = vec![]; + let mut compiled_allocator_module = None; + let mut needs_link = Vec::new(); + let mut needs_fat_lto = Vec::new(); + let mut needs_thin_lto = Vec::new(); + let mut lto_import_only_modules = Vec::new(); + let mut started_lto = false; + let mut codegen_aborted = false; + + // This flag tracks whether all items have gone through codegens + let mut codegen_done = false; + + // This is the queue of LLVM work items that still need processing. + let mut work_items = Vec::<(WorkItem, u64)>::new(); + + // This are the Jobserver Tokens we currently hold. Does not include + // the implicit Token the compiler process owns no matter what. + let mut tokens = Vec::new(); + + let mut main_thread_worker_state = MainThreadWorkerState::Idle; + let mut running = 0; + + let prof = &cgcx.prof; + let mut llvm_start_time: Option> = None; + + // Run the message loop while there's still anything that needs message + // processing. Note that as soon as codegen is aborted we simply want to + // wait for all existing work to finish, so many of the conditions here + // only apply if codegen hasn't been aborted as they represent pending + // work to be done. + while !codegen_done + || running > 0 + || main_thread_worker_state == MainThreadWorkerState::LLVMing + || (!codegen_aborted + && !(work_items.is_empty() + && needs_fat_lto.is_empty() + && needs_thin_lto.is_empty() + && lto_import_only_modules.is_empty() + && main_thread_worker_state == MainThreadWorkerState::Idle)) + { + // While there are still CGUs to be codegened, the coordinator has + // to decide how to utilize the compiler processes implicit Token: + // For codegenning more CGU or for running them through LLVM. + if !codegen_done { + if main_thread_worker_state == MainThreadWorkerState::Idle { + // Compute the number of workers that will be running once we've taken as many + // items from the work queue as we can, plus one for the main thread. It's not + // critically important that we use this instead of just `running`, but it + // prevents the `queue_full_enough` heuristic from fluctuating just because a + // worker finished up and we decreased the `running` count, even though we're + // just going to increase it right after this when we put a new worker to work. + let extra_tokens = tokens.len().checked_sub(running).unwrap(); + let additional_running = std::cmp::min(extra_tokens, work_items.len()); + let anticipated_running = running + additional_running + 1; + + if !queue_full_enough(work_items.len(), anticipated_running) { + // The queue is not full enough, codegen more items: + if codegen_worker_send.send(Message::CodegenItem).is_err() { + panic!("Could not send Message::CodegenItem to main thread") + } + main_thread_worker_state = MainThreadWorkerState::Codegenning; + } else { + // The queue is full enough to not let the worker + // threads starve. Use the implicit Token to do some + // LLVM work too. + let (item, _) = + work_items.pop().expect("queue empty - queue_full_enough() broken?"); + let cgcx = CodegenContext { + worker: get_worker_id(&mut free_worker_ids), + ..cgcx.clone() + }; + maybe_start_llvm_timer( + prof, + cgcx.config(item.module_kind()), + &mut llvm_start_time, + ); + main_thread_worker_state = MainThreadWorkerState::LLVMing; + spawn_work(cgcx, item); + } + } + } else if codegen_aborted { + // don't queue up any more work if codegen was aborted, we're + // just waiting for our existing children to finish + } else { + // If we've finished everything related to normal codegen + // then it must be the case that we've got some LTO work to do. + // Perform the serial work here of figuring out what we're + // going to LTO and then push a bunch of work items onto our + // queue to do LTO + if work_items.is_empty() + && running == 0 + && main_thread_worker_state == MainThreadWorkerState::Idle + { + assert!(!started_lto); + started_lto = true; + + let needs_fat_lto = mem::take(&mut needs_fat_lto); + let needs_thin_lto = mem::take(&mut needs_thin_lto); + let import_only_modules = mem::take(&mut lto_import_only_modules); + + for (work, cost) in + generate_lto_work(&cgcx, needs_fat_lto, needs_thin_lto, import_only_modules) + { + let insertion_index = work_items + .binary_search_by_key(&cost, |&(_, cost)| cost) + .unwrap_or_else(|e| e); + work_items.insert(insertion_index, (work, cost)); + if !cgcx.opts.unstable_opts.no_parallel_llvm { + helper.request_token(); + } + } + } + + // In this branch, we know that everything has been codegened, + // so it's just a matter of determining whether the implicit + // Token is free to use for LLVM work. + match main_thread_worker_state { + MainThreadWorkerState::Idle => { + if let Some((item, _)) = work_items.pop() { + let cgcx = CodegenContext { + worker: get_worker_id(&mut free_worker_ids), + ..cgcx.clone() + }; + maybe_start_llvm_timer( + prof, + cgcx.config(item.module_kind()), + &mut llvm_start_time, + ); + main_thread_worker_state = MainThreadWorkerState::LLVMing; + spawn_work(cgcx, item); + } else { + // There is no unstarted work, so let the main thread + // take over for a running worker. Otherwise the + // implicit token would just go to waste. + // We reduce the `running` counter by one. The + // `tokens.truncate()` below will take care of + // giving the Token back. + debug_assert!(running > 0); + running -= 1; + main_thread_worker_state = MainThreadWorkerState::LLVMing; + } + } + MainThreadWorkerState::Codegenning => bug!( + "codegen worker should not be codegenning after \ + codegen was already completed" + ), + MainThreadWorkerState::LLVMing => { + // Already making good use of that token + } + } + } + + // Spin up what work we can, only doing this while we've got available + // parallelism slots and work left to spawn. + while !codegen_aborted && !work_items.is_empty() && running < tokens.len() { + let (item, _) = work_items.pop().unwrap(); + + maybe_start_llvm_timer(prof, cgcx.config(item.module_kind()), &mut llvm_start_time); + + let cgcx = + CodegenContext { worker: get_worker_id(&mut free_worker_ids), ..cgcx.clone() }; + + spawn_work(cgcx, item); + running += 1; + } + + // Relinquish accidentally acquired extra tokens + tokens.truncate(running); + + // If a thread exits successfully then we drop a token associated + // with that worker and update our `running` count. We may later + // re-acquire a token to continue running more work. We may also not + // actually drop a token here if the worker was running with an + // "ephemeral token" + let mut free_worker = |worker_id| { + if main_thread_worker_state == MainThreadWorkerState::LLVMing { + main_thread_worker_state = MainThreadWorkerState::Idle; + } else { + running -= 1; + } + + free_worker_ids.push(worker_id); + }; + + let msg = coordinator_receive.recv().unwrap(); + match *msg.downcast::>().ok().unwrap() { + // Save the token locally and the next turn of the loop will use + // this to spawn a new unit of work, or it may get dropped + // immediately if we have no more work to spawn. + Message::Token(token) => { + match token { + Ok(token) => { + tokens.push(token); + + if main_thread_worker_state == MainThreadWorkerState::LLVMing { + // If the main thread token is used for LLVM work + // at the moment, we turn that thread into a regular + // LLVM worker thread, so the main thread is free + // to react to codegen demand. + main_thread_worker_state = MainThreadWorkerState::Idle; + running += 1; + } + } + Err(e) => { + let msg = &format!("failed to acquire jobserver token: {}", e); + shared_emitter.fatal(msg); + // Exit the coordinator thread + panic!("{}", msg) + } + } + } + + Message::CodegenDone { llvm_work_item, cost } => { + // We keep the queue sorted by estimated processing cost, + // so that more expensive items are processed earlier. This + // is good for throughput as it gives the main thread more + // time to fill up the queue and it avoids scheduling + // expensive items to the end. + // Note, however, that this is not ideal for memory + // consumption, as LLVM module sizes are not evenly + // distributed. + let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost); + let insertion_index = match insertion_index { + Ok(idx) | Err(idx) => idx, + }; + work_items.insert(insertion_index, (llvm_work_item, cost)); + + if !cgcx.opts.unstable_opts.no_parallel_llvm { + helper.request_token(); + } + assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning); + main_thread_worker_state = MainThreadWorkerState::Idle; + } + + Message::CodegenComplete => { + codegen_done = true; + assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning); + main_thread_worker_state = MainThreadWorkerState::Idle; + } + + // If codegen is aborted that means translation was aborted due + // to some normal-ish compiler error. In this situation we want + // to exit as soon as possible, but we want to make sure all + // existing work has finished. Flag codegen as being done, and + // then conditions above will ensure no more work is spawned but + // we'll keep executing this loop until `running` hits 0. + Message::CodegenAborted => { + codegen_done = true; + codegen_aborted = true; + } + Message::Done { result: Ok(compiled_module), worker_id } => { + free_worker(worker_id); + match compiled_module.kind { + ModuleKind::Regular => { + compiled_modules.push(compiled_module); + } + ModuleKind::Allocator => { + assert!(compiled_allocator_module.is_none()); + compiled_allocator_module = Some(compiled_module); + } + ModuleKind::Metadata => bug!("Should be handled separately"), + } + } + Message::NeedsLink { module, worker_id } => { + free_worker(worker_id); + needs_link.push(module); + } + Message::NeedsFatLTO { result, worker_id } => { + assert!(!started_lto); + free_worker(worker_id); + needs_fat_lto.push(result); + } + Message::NeedsThinLTO { name, thin_buffer, worker_id } => { + assert!(!started_lto); + free_worker(worker_id); + needs_thin_lto.push((name, thin_buffer)); + } + Message::AddImportOnlyModule { module_data, work_product } => { + assert!(!started_lto); + assert!(!codegen_done); + assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning); + lto_import_only_modules.push((module_data, work_product)); + main_thread_worker_state = MainThreadWorkerState::Idle; + } + // If the thread failed that means it panicked, so we abort immediately. + Message::Done { result: Err(None), worker_id: _ } => { + bug!("worker thread panicked"); + } + Message::Done { result: Err(Some(WorkerFatalError)), worker_id } => { + // Similar to CodegenAborted, wait for remaining work to finish. + free_worker(worker_id); + codegen_done = true; + codegen_aborted = true; + } + Message::CodegenItem => bug!("the coordinator should not receive codegen requests"), + } + } + + if codegen_aborted { + return Err(()); + } + + let needs_link = mem::take(&mut needs_link); + if !needs_link.is_empty() { + assert!(compiled_modules.is_empty()); + let diag_handler = cgcx.create_diag_handler(); + let module = B::run_link(&cgcx, &diag_handler, needs_link).map_err(|_| ())?; + let module = unsafe { + B::codegen(&cgcx, &diag_handler, module, cgcx.config(ModuleKind::Regular)) + .map_err(|_| ())? + }; + compiled_modules.push(module); + } + + // Drop to print timings + drop(llvm_start_time); + + // Regardless of what order these modules completed in, report them to + // the backend in the same order every time to ensure that we're handing + // out deterministic results. + compiled_modules.sort_by(|a, b| a.name.cmp(&b.name)); + + Ok(CompiledModules { + modules: compiled_modules, + allocator_module: compiled_allocator_module, + }) + }); + + // A heuristic that determines if we have enough LLVM WorkItems in the + // queue so that the main thread can do LLVM work instead of codegen + fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool { + // This heuristic scales ahead-of-time codegen according to available + // concurrency, as measured by `workers_running`. The idea is that the + // more concurrency we have available, the more demand there will be for + // work items, and the fuller the queue should be kept to meet demand. + // An important property of this approach is that we codegen ahead of + // time only as much as necessary, so as to keep fewer LLVM modules in + // memory at once, thereby reducing memory consumption. + // + // When the number of workers running is less than the max concurrency + // available to us, this heuristic can cause us to instruct the main + // thread to work on an LLVM item (that is, tell it to "LLVM") instead + // of codegen, even though it seems like it *should* be codegenning so + // that we can create more work items and spawn more LLVM workers. + // + // But this is not a problem. When the main thread is told to LLVM, + // according to this heuristic and how work is scheduled, there is + // always at least one item in the queue, and therefore at least one + // pending jobserver token request. If there *is* more concurrency + // available, we will immediately receive a token, which will upgrade + // the main thread's LLVM worker to a real one (conceptually), and free + // up the main thread to codegen if necessary. On the other hand, if + // there isn't more concurrency, then the main thread working on an LLVM + // item is appropriate, as long as the queue is full enough for demand. + // + // Speaking of which, how full should we keep the queue? Probably less + // full than you'd think. A lot has to go wrong for the queue not to be + // full enough and for that to have a negative effect on compile times. + // + // Workers are unlikely to finish at exactly the same time, so when one + // finishes and takes another work item off the queue, we often have + // ample time to codegen at that point before the next worker finishes. + // But suppose that codegen takes so long that the workers exhaust the + // queue, and we have one or more workers that have nothing to work on. + // Well, it might not be so bad. Of all the LLVM modules we create and + // optimize, one has to finish last. It's not necessarily the case that + // by losing some concurrency for a moment, we delay the point at which + // that last LLVM module is finished and the rest of compilation can + // proceed. Also, when we can't take advantage of some concurrency, we + // give tokens back to the job server. That enables some other rustc to + // potentially make use of the available concurrency. That could even + // *decrease* overall compile time if we're lucky. But yes, if no other + // rustc can make use of the concurrency, then we've squandered it. + // + // However, keeping the queue full is also beneficial when we have a + // surge in available concurrency. Then items can be taken from the + // queue immediately, without having to wait for codegen. + // + // So, the heuristic below tries to keep one item in the queue for every + // four running workers. Based on limited benchmarking, this appears to + // be more than sufficient to avoid increasing compilation times. + let quarter_of_workers = workers_running - 3 * workers_running / 4; + items_in_queue > 0 && items_in_queue >= quarter_of_workers + } + + fn maybe_start_llvm_timer<'a>( + prof: &'a SelfProfilerRef, + config: &ModuleConfig, + llvm_start_time: &mut Option>, + ) { + if config.time_module && llvm_start_time.is_none() { + *llvm_start_time = Some(prof.extra_verbose_generic_activity("LLVM_passes", "crate")); + } + } +} + +/// `FatalError` is explicitly not `Send`. +#[must_use] +pub struct WorkerFatalError; + +fn spawn_work(cgcx: CodegenContext, work: WorkItem) { + B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || { + // Set up a destructor which will fire off a message that we're done as + // we exit. + struct Bomb { + coordinator_send: Sender>, + result: Option, FatalError>>, + worker_id: usize, + } + impl Drop for Bomb { + fn drop(&mut self) { + let worker_id = self.worker_id; + let msg = match self.result.take() { + Some(Ok(WorkItemResult::Compiled(m))) => { + Message::Done:: { result: Ok(m), worker_id } + } + Some(Ok(WorkItemResult::NeedsLink(m))) => { + Message::NeedsLink:: { module: m, worker_id } + } + Some(Ok(WorkItemResult::NeedsFatLTO(m))) => { + Message::NeedsFatLTO:: { result: m, worker_id } + } + Some(Ok(WorkItemResult::NeedsThinLTO(name, thin_buffer))) => { + Message::NeedsThinLTO:: { name, thin_buffer, worker_id } + } + Some(Err(FatalError)) => { + Message::Done:: { result: Err(Some(WorkerFatalError)), worker_id } + } + None => Message::Done:: { result: Err(None), worker_id }, + }; + drop(self.coordinator_send.send(Box::new(msg))); + } + } + + let mut bomb = Bomb:: { + coordinator_send: cgcx.coordinator_send.clone(), + result: None, + worker_id: cgcx.worker, + }; + + // Execute the work itself, and if it finishes successfully then flag + // ourselves as a success as well. + // + // Note that we ignore any `FatalError` coming out of `execute_work_item`, + // as a diagnostic was already sent off to the main thread - just + // surface that there was an error in this worker. + bomb.result = { + let _prof_timer = work.start_profiling(&cgcx); + Some(execute_work_item(&cgcx, work)) + }; + }) + .expect("failed to spawn thread"); +} + +enum SharedEmitterMessage { + Diagnostic(Diagnostic), + InlineAsmError(u32, String, Level, Option<(String, Vec)>), + AbortIfErrors, + Fatal(String), +} + +#[derive(Clone)] +pub struct SharedEmitter { + sender: Sender, +} + +pub struct SharedEmitterMain { + receiver: Receiver, +} + +impl SharedEmitter { + pub fn new() -> (SharedEmitter, SharedEmitterMain) { + let (sender, receiver) = channel(); + + (SharedEmitter { sender }, SharedEmitterMain { receiver }) + } + + pub fn inline_asm_error( + &self, + cookie: u32, + msg: String, + level: Level, + source: Option<(String, Vec)>, + ) { + drop(self.sender.send(SharedEmitterMessage::InlineAsmError(cookie, msg, level, source))); + } + + pub fn fatal(&self, msg: &str) { + drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string()))); + } +} + +impl Emitter for SharedEmitter { + fn emit_diagnostic(&mut self, diag: &rustc_errors::Diagnostic) { + let fluent_args = self.to_fluent_args(diag.args()); + drop(self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic { + msg: self.translate_messages(&diag.message, &fluent_args).to_string(), + code: diag.code.clone(), + lvl: diag.level(), + }))); + for child in &diag.children { + drop(self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic { + msg: self.translate_messages(&child.message, &fluent_args).to_string(), + code: None, + lvl: child.level, + }))); + } + drop(self.sender.send(SharedEmitterMessage::AbortIfErrors)); + } + + fn source_map(&self) -> Option<&Lrc> { + None + } + + fn fluent_bundle(&self) -> Option<&Lrc> { + None + } + + fn fallback_fluent_bundle(&self) -> &rustc_errors::FluentBundle { + panic!("shared emitter attempted to translate a diagnostic"); + } +} + +impl SharedEmitterMain { + pub fn check(&self, sess: &Session, blocking: bool) { + loop { + let message = if blocking { + match self.receiver.recv() { + Ok(message) => Ok(message), + Err(_) => Err(()), + } + } else { + match self.receiver.try_recv() { + Ok(message) => Ok(message), + Err(_) => Err(()), + } + }; + + match message { + Ok(SharedEmitterMessage::Diagnostic(diag)) => { + let handler = sess.diagnostic(); + let mut d = rustc_errors::Diagnostic::new(diag.lvl, &diag.msg); + if let Some(code) = diag.code { + d.code(code); + } + handler.emit_diagnostic(&mut d); + } + Ok(SharedEmitterMessage::InlineAsmError(cookie, msg, level, source)) => { + let msg = msg.strip_prefix("error: ").unwrap_or(&msg); + + let mut err = match level { + Level::Error { lint: false } => sess.struct_err(msg).forget_guarantee(), + Level::Warning(_) => sess.struct_warn(msg), + Level::Note => sess.struct_note_without_error(msg), + _ => bug!("Invalid inline asm diagnostic level"), + }; + + // If the cookie is 0 then we don't have span information. + if cookie != 0 { + let pos = BytePos::from_u32(cookie); + let span = Span::with_root_ctxt(pos, pos); + err.set_span(span); + }; + + // Point to the generated assembly if it is available. + if let Some((buffer, spans)) = source { + let source = sess + .source_map() + .new_source_file(FileName::inline_asm_source_code(&buffer), buffer); + let source_span = Span::with_root_ctxt(source.start_pos, source.end_pos); + let spans: Vec<_> = + spans.iter().map(|sp| source_span.from_inner(*sp)).collect(); + err.span_note(spans, "instantiated into assembly here"); + } + + err.emit(); + } + Ok(SharedEmitterMessage::AbortIfErrors) => { + sess.abort_if_errors(); + } + Ok(SharedEmitterMessage::Fatal(msg)) => { + sess.fatal(&msg); + } + Err(_) => { + break; + } + } + } + } +} + +pub struct Coordinator { + pub sender: Sender>, + future: Option>>, + // Only used for the Message type. + phantom: PhantomData, +} + +impl Coordinator { + fn join(mut self) -> std::thread::Result> { + self.future.take().unwrap().join() + } +} + +impl Drop for Coordinator { + fn drop(&mut self) { + if let Some(future) = self.future.take() { + // If we haven't joined yet, signal to the coordinator that it should spawn no more + // work, and wait for worker threads to finish. + drop(self.sender.send(Box::new(Message::CodegenAborted::))); + drop(future.join()); + } + } +} + +pub struct OngoingCodegen { + pub backend: B, + pub metadata: EncodedMetadata, + pub metadata_module: Option, + pub crate_info: CrateInfo, + pub codegen_worker_receive: Receiver>, + pub shared_emitter_main: SharedEmitterMain, + pub output_filenames: Arc, + pub coordinator: Coordinator, +} + +impl OngoingCodegen { + pub fn join(self, sess: &Session) -> (CodegenResults, FxHashMap) { + let _timer = sess.timer("finish_ongoing_codegen"); + + self.shared_emitter_main.check(sess, true); + let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() { + Ok(Ok(compiled_modules)) => compiled_modules, + Ok(Err(())) => { + sess.abort_if_errors(); + panic!("expected abort due to worker thread errors") + } + Err(_) => { + bug!("panic during codegen/LLVM phase"); + } + }); + + sess.cgu_reuse_tracker.check_expected_reuse(sess.diagnostic()); + + sess.abort_if_errors(); + + let work_products = + copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules); + produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames); + + // FIXME: time_llvm_passes support - does this use a global context or + // something? + if sess.codegen_units() == 1 && sess.time_llvm_passes() { + self.backend.print_pass_timings() + } + + ( + CodegenResults { + metadata: self.metadata, + crate_info: self.crate_info, + + modules: compiled_modules.modules, + allocator_module: compiled_modules.allocator_module, + metadata_module: self.metadata_module, + }, + work_products, + ) + } + + pub fn submit_pre_codegened_module_to_llvm( + &self, + tcx: TyCtxt<'_>, + module: ModuleCodegen, + ) { + self.wait_for_signal_to_codegen_item(); + self.check_for_errors(tcx.sess); + + // These are generally cheap and won't throw off scheduling. + let cost = 0; + submit_codegened_module_to_llvm(&self.backend, &self.coordinator.sender, module, cost); + } + + pub fn codegen_finished(&self, tcx: TyCtxt<'_>) { + self.wait_for_signal_to_codegen_item(); + self.check_for_errors(tcx.sess); + drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::))); + } + + pub fn check_for_errors(&self, sess: &Session) { + self.shared_emitter_main.check(sess, false); + } + + pub fn wait_for_signal_to_codegen_item(&self) { + match self.codegen_worker_receive.recv() { + Ok(Message::CodegenItem) => { + // Nothing to do + } + Ok(_) => panic!("unexpected message"), + Err(_) => { + // One of the LLVM threads must have panicked, fall through so + // error handling can be reached. + } + } + } +} + +pub fn submit_codegened_module_to_llvm( + _backend: &B, + tx_to_llvm_workers: &Sender>, + module: ModuleCodegen, + cost: u64, +) { + let llvm_work_item = WorkItem::Optimize(module); + drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone:: { llvm_work_item, cost }))); +} + +pub fn submit_post_lto_module_to_llvm( + _backend: &B, + tx_to_llvm_workers: &Sender>, + module: CachedModuleCodegen, +) { + let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module); + drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone:: { llvm_work_item, cost: 0 }))); +} + +pub fn submit_pre_lto_module_to_llvm( + _backend: &B, + tcx: TyCtxt<'_>, + tx_to_llvm_workers: &Sender>, + module: CachedModuleCodegen, +) { + let filename = pre_lto_bitcode_filename(&module.name); + let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename); + let file = fs::File::open(&bc_path) + .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e)); + + let mmap = unsafe { + Mmap::map(file).unwrap_or_else(|e| { + panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e) + }) + }; + // Schedule the module to be loaded + drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule:: { + module_data: SerializedModule::FromUncompressedFile(mmap), + work_product: module.source, + }))); +} + +pub fn pre_lto_bitcode_filename(module_name: &str) -> String { + format!("{}.{}", module_name, PRE_LTO_BC_EXT) +} + +fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool { + // This should never be true (because it's not supported). If it is true, + // something is wrong with commandline arg validation. + assert!( + !(tcx.sess.opts.cg.linker_plugin_lto.enabled() + && tcx.sess.target.is_like_windows + && tcx.sess.opts.cg.prefer_dynamic) + ); + + tcx.sess.target.is_like_windows && + tcx.sess.crate_types().iter().any(|ct| *ct == CrateType::Rlib) && + // ThinLTO can't handle this workaround in all cases, so we don't + // emit the `__imp_` symbols. Instead we make them unnecessary by disallowing + // dynamic linking when linker plugin LTO is enabled. + !tcx.sess.opts.cg.linker_plugin_lto.enabled() +} diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs new file mode 100644 index 00000000000..a840b270974 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -0,0 +1,961 @@ +use crate::back::metadata::create_compressed_metadata_file; +use crate::back::write::{ + compute_per_cgu_lto_type, start_async_codegen, submit_codegened_module_to_llvm, + submit_post_lto_module_to_llvm, submit_pre_lto_module_to_llvm, ComputedLtoType, OngoingCodegen, +}; +use crate::common::{IntPredicate, RealPredicate, TypeKind}; +use crate::meth; +use crate::mir; +use crate::mir::operand::OperandValue; +use crate::mir::place::PlaceRef; +use crate::traits::*; +use crate::{CachedModuleCodegen, CompiledModule, CrateInfo, MemFlags, ModuleCodegen, ModuleKind}; + +use rustc_attr as attr; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry}; + +use rustc_data_structures::sync::par_iter; +#[cfg(parallel_compiler)] +use rustc_data_structures::sync::ParallelIterator; +use rustc_hir as hir; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_hir::lang_items::LangItem; +use rustc_index::vec::Idx; +use rustc_metadata::EncodedMetadata; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; +use rustc_middle::middle::exported_symbols; +use rustc_middle::middle::lang_items; +use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem}; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; +use rustc_middle::ty::query::Providers; +use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; +use rustc_session::cgu_reuse_tracker::CguReuse; +use rustc_session::config::{self, CrateType, EntryFnType, OutputType}; +use rustc_session::Session; +use rustc_span::symbol::sym; +use rustc_span::{DebuggerVisualizerFile, DebuggerVisualizerType}; +use rustc_target::abi::{Align, VariantIdx}; + +use std::collections::BTreeSet; +use std::convert::TryFrom; +use std::time::{Duration, Instant}; + +use itertools::Itertools; + +pub fn bin_op_to_icmp_predicate(op: hir::BinOpKind, signed: bool) -> IntPredicate { + match op { + hir::BinOpKind::Eq => IntPredicate::IntEQ, + hir::BinOpKind::Ne => IntPredicate::IntNE, + hir::BinOpKind::Lt => { + if signed { + IntPredicate::IntSLT + } else { + IntPredicate::IntULT + } + } + hir::BinOpKind::Le => { + if signed { + IntPredicate::IntSLE + } else { + IntPredicate::IntULE + } + } + hir::BinOpKind::Gt => { + if signed { + IntPredicate::IntSGT + } else { + IntPredicate::IntUGT + } + } + hir::BinOpKind::Ge => { + if signed { + IntPredicate::IntSGE + } else { + IntPredicate::IntUGE + } + } + op => bug!( + "comparison_op_to_icmp_predicate: expected comparison operator, \ + found {:?}", + op + ), + } +} + +pub fn bin_op_to_fcmp_predicate(op: hir::BinOpKind) -> RealPredicate { + match op { + hir::BinOpKind::Eq => RealPredicate::RealOEQ, + hir::BinOpKind::Ne => RealPredicate::RealUNE, + hir::BinOpKind::Lt => RealPredicate::RealOLT, + hir::BinOpKind::Le => RealPredicate::RealOLE, + hir::BinOpKind::Gt => RealPredicate::RealOGT, + hir::BinOpKind::Ge => RealPredicate::RealOGE, + op => { + bug!( + "comparison_op_to_fcmp_predicate: expected comparison operator, \ + found {:?}", + op + ); + } + } +} + +pub fn compare_simd_types<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + lhs: Bx::Value, + rhs: Bx::Value, + t: Ty<'tcx>, + ret_ty: Bx::Type, + op: hir::BinOpKind, +) -> Bx::Value { + let signed = match t.kind() { + ty::Float(_) => { + let cmp = bin_op_to_fcmp_predicate(op); + let cmp = bx.fcmp(cmp, lhs, rhs); + return bx.sext(cmp, ret_ty); + } + ty::Uint(_) => false, + ty::Int(_) => true, + _ => bug!("compare_simd_types: invalid SIMD type"), + }; + + let cmp = bin_op_to_icmp_predicate(op, signed); + let cmp = bx.icmp(cmp, lhs, rhs); + // LLVM outputs an `< size x i1 >`, so we need to perform a sign extension + // to get the correctly sized type. This will compile to a single instruction + // once the IR is converted to assembly if the SIMD instruction is supported + // by the target architecture. + bx.sext(cmp, ret_ty) +} + +/// Retrieves the information we are losing (making dynamic) in an unsizing +/// adjustment. +/// +/// The `old_info` argument is a bit odd. It is intended for use in an upcast, +/// where the new vtable for an object will be derived from the old one. +pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + source: Ty<'tcx>, + target: Ty<'tcx>, + old_info: Option, +) -> Bx::Value { + let cx = bx.cx(); + let (source, target) = + cx.tcx().struct_lockstep_tails_erasing_lifetimes(source, target, bx.param_env()); + match (source.kind(), target.kind()) { + (&ty::Array(_, len), &ty::Slice(_)) => { + cx.const_usize(len.eval_usize(cx.tcx(), ty::ParamEnv::reveal_all())) + } + (&ty::Dynamic(ref data_a, ..), &ty::Dynamic(ref data_b, ..)) => { + let old_info = + old_info.expect("unsized_info: missing old info for trait upcasting coercion"); + if data_a.principal_def_id() == data_b.principal_def_id() { + return old_info; + } + + // trait upcasting coercion + + let vptr_entry_idx = + cx.tcx().vtable_trait_upcasting_coercion_new_vptr_slot((source, target)); + + if let Some(entry_idx) = vptr_entry_idx { + let ptr_ty = cx.type_i8p(); + let ptr_align = cx.tcx().data_layout.pointer_align.abi; + let llvtable = bx.pointercast(old_info, bx.type_ptr_to(ptr_ty)); + let gep = bx.inbounds_gep( + ptr_ty, + llvtable, + &[bx.const_usize(u64::try_from(entry_idx).unwrap())], + ); + let new_vptr = bx.load(ptr_ty, gep, ptr_align); + bx.nonnull_metadata(new_vptr); + // VTable loads are invariant. + bx.set_invariant_load(new_vptr); + new_vptr + } else { + old_info + } + } + (_, &ty::Dynamic(ref data, ..)) => { + let vtable_ptr_ty = cx.scalar_pair_element_backend_type( + cx.layout_of(cx.tcx().mk_mut_ptr(target)), + 1, + true, + ); + cx.const_ptrcast(meth::get_vtable(cx, source, data.principal()), vtable_ptr_ty) + } + _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target), + } +} + +/// Coerces `src` to `dst_ty`. `src_ty` must be a pointer. +pub fn unsize_ptr<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + src: Bx::Value, + src_ty: Ty<'tcx>, + dst_ty: Ty<'tcx>, + old_info: Option, +) -> (Bx::Value, Bx::Value) { + debug!("unsize_ptr: {:?} => {:?}", src_ty, dst_ty); + match (src_ty.kind(), dst_ty.kind()) { + (&ty::Ref(_, a, _), &ty::Ref(_, b, _) | &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) + | (&ty::RawPtr(ty::TypeAndMut { ty: a, .. }), &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) => { + assert_eq!(bx.cx().type_is_sized(a), old_info.is_none()); + let ptr_ty = bx.cx().type_ptr_to(bx.cx().backend_type(bx.cx().layout_of(b))); + (bx.pointercast(src, ptr_ty), unsized_info(bx, a, b, old_info)) + } + (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { + assert_eq!(def_a, def_b); + let src_layout = bx.cx().layout_of(src_ty); + let dst_layout = bx.cx().layout_of(dst_ty); + if src_ty == dst_ty { + return (src, old_info.unwrap()); + } + let mut result = None; + for i in 0..src_layout.fields.count() { + let src_f = src_layout.field(bx.cx(), i); + if src_f.is_zst() { + continue; + } + + assert_eq!(src_layout.fields.offset(i).bytes(), 0); + assert_eq!(dst_layout.fields.offset(i).bytes(), 0); + assert_eq!(src_layout.size, src_f.size); + + let dst_f = dst_layout.field(bx.cx(), i); + assert_ne!(src_f.ty, dst_f.ty); + assert_eq!(result, None); + result = Some(unsize_ptr(bx, src, src_f.ty, dst_f.ty, old_info)); + } + let (lldata, llextra) = result.unwrap(); + let lldata_ty = bx.cx().scalar_pair_element_backend_type(dst_layout, 0, true); + let llextra_ty = bx.cx().scalar_pair_element_backend_type(dst_layout, 1, true); + // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. + (bx.bitcast(lldata, lldata_ty), bx.bitcast(llextra, llextra_ty)) + } + _ => bug!("unsize_ptr: called on bad types"), + } +} + +/// Coerces `src`, which is a reference to a value of type `src_ty`, +/// to a value of type `dst_ty`, and stores the result in `dst`. +pub fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + src: PlaceRef<'tcx, Bx::Value>, + dst: PlaceRef<'tcx, Bx::Value>, +) { + let src_ty = src.layout.ty; + let dst_ty = dst.layout.ty; + match (src_ty.kind(), dst_ty.kind()) { + (&ty::Ref(..), &ty::Ref(..) | &ty::RawPtr(..)) | (&ty::RawPtr(..), &ty::RawPtr(..)) => { + let (base, info) = match bx.load_operand(src).val { + OperandValue::Pair(base, info) => unsize_ptr(bx, base, src_ty, dst_ty, Some(info)), + OperandValue::Immediate(base) => unsize_ptr(bx, base, src_ty, dst_ty, None), + OperandValue::Ref(..) => bug!(), + }; + OperandValue::Pair(base, info).store(bx, dst); + } + + (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { + assert_eq!(def_a, def_b); + + for i in 0..def_a.variant(VariantIdx::new(0)).fields.len() { + let src_f = src.project_field(bx, i); + let dst_f = dst.project_field(bx, i); + + if dst_f.layout.is_zst() { + continue; + } + + if src_f.layout.ty == dst_f.layout.ty { + memcpy_ty( + bx, + dst_f.llval, + dst_f.align, + src_f.llval, + src_f.align, + src_f.layout, + MemFlags::empty(), + ); + } else { + coerce_unsized_into(bx, src_f, dst_f); + } + } + } + _ => bug!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty, dst_ty,), + } +} + +pub fn cast_shift_expr_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + op: hir::BinOpKind, + lhs: Bx::Value, + rhs: Bx::Value, +) -> Bx::Value { + cast_shift_rhs(bx, op, lhs, rhs) +} + +fn cast_shift_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + op: hir::BinOpKind, + lhs: Bx::Value, + rhs: Bx::Value, +) -> Bx::Value { + // Shifts may have any size int on the rhs + if op.is_shift() { + let mut rhs_llty = bx.cx().val_ty(rhs); + let mut lhs_llty = bx.cx().val_ty(lhs); + if bx.cx().type_kind(rhs_llty) == TypeKind::Vector { + rhs_llty = bx.cx().element_type(rhs_llty) + } + if bx.cx().type_kind(lhs_llty) == TypeKind::Vector { + lhs_llty = bx.cx().element_type(lhs_llty) + } + let rhs_sz = bx.cx().int_width(rhs_llty); + let lhs_sz = bx.cx().int_width(lhs_llty); + if lhs_sz < rhs_sz { + bx.trunc(rhs, lhs_llty) + } else if lhs_sz > rhs_sz { + // FIXME (#1877: If in the future shifting by negative + // values is no longer undefined then this is wrong. + bx.zext(rhs, lhs_llty) + } else { + rhs + } + } else { + rhs + } +} + +/// Returns `true` if this session's target will use SEH-based unwinding. +/// +/// This is only true for MSVC targets, and even then the 64-bit MSVC target +/// currently uses SEH-ish unwinding with DWARF info tables to the side (same as +/// 64-bit MinGW) instead of "full SEH". +pub fn wants_msvc_seh(sess: &Session) -> bool { + sess.target.is_like_msvc +} + +pub fn memcpy_ty<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + dst: Bx::Value, + dst_align: Align, + src: Bx::Value, + src_align: Align, + layout: TyAndLayout<'tcx>, + flags: MemFlags, +) { + let size = layout.size.bytes(); + if size == 0 { + return; + } + + bx.memcpy(dst, dst_align, src, src_align, bx.cx().const_usize(size), flags); +} + +pub fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>( + cx: &'a Bx::CodegenCx, + instance: Instance<'tcx>, +) { + // this is an info! to allow collecting monomorphization statistics + // and to allow finding the last function before LLVM aborts from + // release builds. + info!("codegen_instance({})", instance); + + mir::codegen_mir::(cx, instance); +} + +/// Creates the `main` function which will initialize the rust runtime and call +/// users main function. +pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + cx: &'a Bx::CodegenCx, +) -> Option { + let (main_def_id, entry_type) = cx.tcx().entry_fn(())?; + let main_is_local = main_def_id.is_local(); + let instance = Instance::mono(cx.tcx(), main_def_id); + + if main_is_local { + // We want to create the wrapper in the same codegen unit as Rust's main + // function. + if !cx.codegen_unit().contains_item(&MonoItem::Fn(instance)) { + return None; + } + } else if !cx.codegen_unit().is_primary() { + // We want to create the wrapper only when the codegen unit is the primary one + return None; + } + + let main_llfn = cx.get_fn_addr(instance); + + let use_start_lang_item = EntryFnType::Start != entry_type; + let entry_fn = create_entry_fn::(cx, main_llfn, main_def_id, use_start_lang_item); + return Some(entry_fn); + + fn create_entry_fn<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + cx: &'a Bx::CodegenCx, + rust_main: Bx::Value, + rust_main_def_id: DefId, + use_start_lang_item: bool, + ) -> Bx::Function { + // The entry function is either `int main(void)` or `int main(int argc, char **argv)`, + // depending on whether the target needs `argc` and `argv` to be passed in. + let llfty = if cx.sess().target.main_needs_argc_argv { + cx.type_func(&[cx.type_int(), cx.type_ptr_to(cx.type_i8p())], cx.type_int()) + } else { + cx.type_func(&[], cx.type_int()) + }; + + let main_ret_ty = cx.tcx().fn_sig(rust_main_def_id).output(); + // Given that `main()` has no arguments, + // then its return type cannot have + // late-bound regions, since late-bound + // regions must appear in the argument + // listing. + let main_ret_ty = cx.tcx().normalize_erasing_regions( + ty::ParamEnv::reveal_all(), + main_ret_ty.no_bound_vars().unwrap(), + ); + + let Some(llfn) = cx.declare_c_main(llfty) else { + // FIXME: We should be smart and show a better diagnostic here. + let span = cx.tcx().def_span(rust_main_def_id); + cx.sess() + .struct_span_err(span, "entry symbol `main` declared multiple times") + .help("did you use `#[no_mangle]` on `fn main`? Use `#[start]` instead") + .emit(); + cx.sess().abort_if_errors(); + bug!(); + }; + + // `main` should respect same config for frame pointer elimination as rest of code + cx.set_frame_pointer_type(llfn); + cx.apply_target_cpu_attr(llfn); + + let llbb = Bx::append_block(&cx, llfn, "top"); + let mut bx = Bx::build(&cx, llbb); + + bx.insert_reference_to_gdb_debug_scripts_section_global(); + + let isize_ty = cx.type_isize(); + let i8pp_ty = cx.type_ptr_to(cx.type_i8p()); + let (arg_argc, arg_argv) = get_argc_argv(cx, &mut bx); + + let (start_fn, start_ty, args) = if use_start_lang_item { + let start_def_id = cx.tcx().require_lang_item(LangItem::Start, None); + let start_fn = cx.get_fn_addr( + ty::Instance::resolve( + cx.tcx(), + ty::ParamEnv::reveal_all(), + start_def_id, + cx.tcx().intern_substs(&[main_ret_ty.into()]), + ) + .unwrap() + .unwrap(), + ); + let start_ty = cx.type_func(&[cx.val_ty(rust_main), isize_ty, i8pp_ty], isize_ty); + (start_fn, start_ty, vec![rust_main, arg_argc, arg_argv]) + } else { + debug!("using user-defined start fn"); + let start_ty = cx.type_func(&[isize_ty, i8pp_ty], isize_ty); + (rust_main, start_ty, vec![arg_argc, arg_argv]) + }; + + let result = bx.call(start_ty, start_fn, &args, None); + let cast = bx.intcast(result, cx.type_int(), true); + bx.ret(cast); + + llfn + } +} + +/// Obtain the `argc` and `argv` values to pass to the rust start function. +fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + cx: &'a Bx::CodegenCx, + bx: &mut Bx, +) -> (Bx::Value, Bx::Value) { + if cx.sess().target.main_needs_argc_argv { + // Params from native `main()` used as args for rust start function + let param_argc = bx.get_param(0); + let param_argv = bx.get_param(1); + let arg_argc = bx.intcast(param_argc, cx.type_isize(), true); + let arg_argv = param_argv; + (arg_argc, arg_argv) + } else { + // The Rust start function doesn't need `argc` and `argv`, so just pass zeros. + let arg_argc = bx.const_int(cx.type_int(), 0); + let arg_argv = bx.const_null(cx.type_ptr_to(cx.type_i8p())); + (arg_argc, arg_argv) + } +} + +/// This function returns all of the debugger visualizers specified for the +/// current crate as well as all upstream crates transitively that match the +/// `visualizer_type` specified. +pub fn collect_debugger_visualizers_transitive( + tcx: TyCtxt<'_>, + visualizer_type: DebuggerVisualizerType, +) -> BTreeSet { + tcx.debugger_visualizers(LOCAL_CRATE) + .iter() + .chain( + tcx.crates(()) + .iter() + .filter(|&cnum| { + let used_crate_source = tcx.used_crate_source(*cnum); + used_crate_source.rlib.is_some() || used_crate_source.rmeta.is_some() + }) + .flat_map(|&cnum| tcx.debugger_visualizers(cnum)), + ) + .filter(|visualizer| visualizer.visualizer_type == visualizer_type) + .cloned() + .collect::>() +} + +pub fn codegen_crate( + backend: B, + tcx: TyCtxt<'_>, + target_cpu: String, + metadata: EncodedMetadata, + need_metadata_module: bool, +) -> OngoingCodegen { + // Skip crate items and just output metadata in -Z no-codegen mode. + if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { + let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, metadata, None, 1); + + ongoing_codegen.codegen_finished(tcx); + + ongoing_codegen.check_for_errors(tcx.sess); + + return ongoing_codegen; + } + + let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx); + + // Run the monomorphization collector and partition the collected items into + // codegen units. + let codegen_units = tcx.collect_and_partition_mono_items(()).1; + + // Force all codegen_unit queries so they are already either red or green + // when compile_codegen_unit accesses them. We are not able to re-execute + // the codegen_unit query from just the DepNode, so an unknown color would + // lead to having to re-execute compile_codegen_unit, possibly + // unnecessarily. + if tcx.dep_graph.is_fully_enabled() { + for cgu in codegen_units { + tcx.ensure().codegen_unit(cgu.name()); + } + } + + let metadata_module = if need_metadata_module { + // Emit compressed metadata object. + let metadata_cgu_name = + cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("metadata")).to_string(); + tcx.sess.time("write_compressed_metadata", || { + let file_name = + tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); + let data = create_compressed_metadata_file( + tcx.sess, + &metadata, + &exported_symbols::metadata_symbol_name(tcx), + ); + if let Err(err) = std::fs::write(&file_name, data) { + tcx.sess.fatal(&format!("error writing metadata object file: {}", err)); + } + Some(CompiledModule { + name: metadata_cgu_name, + kind: ModuleKind::Metadata, + object: Some(file_name), + dwarf_object: None, + bytecode: None, + }) + }) + } else { + None + }; + + let ongoing_codegen = start_async_codegen( + backend.clone(), + tcx, + target_cpu, + metadata, + metadata_module, + codegen_units.len(), + ); + + // Codegen an allocator shim, if necessary. + // + // If the crate doesn't have an `allocator_kind` set then there's definitely + // no shim to generate. Otherwise we also check our dependency graph for all + // our output crate types. If anything there looks like its a `Dynamic` + // linkage, then it's already got an allocator shim and we'll be using that + // one instead. If nothing exists then it's our job to generate the + // allocator! + let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| { + use rustc_middle::middle::dependency_format::Linkage; + list.iter().any(|&linkage| linkage == Linkage::Dynamic) + }); + let allocator_module = if any_dynamic_crate { + None + } else if let Some(kind) = tcx.allocator_kind(()) { + let llmod_id = + cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("allocator")).to_string(); + let module_llvm = tcx.sess.time("write_allocator_module", || { + backend.codegen_allocator(tcx, &llmod_id, kind, tcx.lang_items().oom().is_some()) + }); + + Some(ModuleCodegen { name: llmod_id, module_llvm, kind: ModuleKind::Allocator }) + } else { + None + }; + + if let Some(allocator_module) = allocator_module { + ongoing_codegen.submit_pre_codegened_module_to_llvm(tcx, allocator_module); + } + + // For better throughput during parallel processing by LLVM, we used to sort + // CGUs largest to smallest. This would lead to better thread utilization + // by, for example, preventing a large CGU from being processed last and + // having only one LLVM thread working while the rest remained idle. + // + // However, this strategy would lead to high memory usage, as it meant the + // LLVM-IR for all of the largest CGUs would be resident in memory at once. + // + // Instead, we can compromise by ordering CGUs such that the largest and + // smallest are first, second largest and smallest are next, etc. If there + // are large size variations, this can reduce memory usage significantly. + let codegen_units: Vec<_> = { + let mut sorted_cgus = codegen_units.iter().collect::>(); + sorted_cgus.sort_by_cached_key(|cgu| cgu.size_estimate()); + + let (first_half, second_half) = sorted_cgus.split_at(sorted_cgus.len() / 2); + second_half.iter().rev().interleave(first_half).copied().collect() + }; + + // Calculate the CGU reuse + let cgu_reuse = tcx.sess.time("find_cgu_reuse", || { + codegen_units.iter().map(|cgu| determine_cgu_reuse(tcx, &cgu)).collect::>() + }); + + let mut total_codegen_time = Duration::new(0, 0); + let start_rss = tcx.sess.time_passes().then(|| get_resident_set_size()); + + // The non-parallel compiler can only translate codegen units to LLVM IR + // on a single thread, leading to a staircase effect where the N LLVM + // threads have to wait on the single codegen threads to generate work + // for them. The parallel compiler does not have this restriction, so + // we can pre-load the LLVM queue in parallel before handing off + // coordination to the OnGoingCodegen scheduler. + // + // This likely is a temporary measure. Once we don't have to support the + // non-parallel compiler anymore, we can compile CGUs end-to-end in + // parallel and get rid of the complicated scheduling logic. + let mut pre_compiled_cgus = if cfg!(parallel_compiler) { + tcx.sess.time("compile_first_CGU_batch", || { + // Try to find one CGU to compile per thread. + let cgus: Vec<_> = cgu_reuse + .iter() + .enumerate() + .filter(|&(_, reuse)| reuse == &CguReuse::No) + .take(tcx.sess.threads()) + .collect(); + + // Compile the found CGUs in parallel. + let start_time = Instant::now(); + + let pre_compiled_cgus = par_iter(cgus) + .map(|(i, _)| { + let module = backend.compile_codegen_unit(tcx, codegen_units[i].name()); + (i, module) + }) + .collect(); + + total_codegen_time += start_time.elapsed(); + + pre_compiled_cgus + }) + } else { + FxHashMap::default() + }; + + for (i, cgu) in codegen_units.iter().enumerate() { + ongoing_codegen.wait_for_signal_to_codegen_item(); + ongoing_codegen.check_for_errors(tcx.sess); + + let cgu_reuse = cgu_reuse[i]; + tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse); + + match cgu_reuse { + CguReuse::No => { + let (module, cost) = if let Some(cgu) = pre_compiled_cgus.remove(&i) { + cgu + } else { + let start_time = Instant::now(); + let module = backend.compile_codegen_unit(tcx, cgu.name()); + total_codegen_time += start_time.elapsed(); + module + }; + // This will unwind if there are errors, which triggers our `AbortCodegenOnDrop` + // guard. Unfortunately, just skipping the `submit_codegened_module_to_llvm` makes + // compilation hang on post-monomorphization errors. + tcx.sess.abort_if_errors(); + + submit_codegened_module_to_llvm( + &backend, + &ongoing_codegen.coordinator.sender, + module, + cost, + ); + false + } + CguReuse::PreLto => { + submit_pre_lto_module_to_llvm( + &backend, + tcx, + &ongoing_codegen.coordinator.sender, + CachedModuleCodegen { + name: cgu.name().to_string(), + source: cgu.previous_work_product(tcx), + }, + ); + true + } + CguReuse::PostLto => { + submit_post_lto_module_to_llvm( + &backend, + &ongoing_codegen.coordinator.sender, + CachedModuleCodegen { + name: cgu.name().to_string(), + source: cgu.previous_work_product(tcx), + }, + ); + true + } + }; + } + + ongoing_codegen.codegen_finished(tcx); + + // Since the main thread is sometimes blocked during codegen, we keep track + // -Ztime-passes output manually. + if tcx.sess.time_passes() { + let end_rss = get_resident_set_size(); + + print_time_passes_entry( + "codegen_to_LLVM_IR", + total_codegen_time, + start_rss.unwrap(), + end_rss, + ); + } + + ongoing_codegen.check_for_errors(tcx.sess); + ongoing_codegen +} + +impl CrateInfo { + pub fn new(tcx: TyCtxt<'_>, target_cpu: String) -> CrateInfo { + let exported_symbols = tcx + .sess + .crate_types() + .iter() + .map(|&c| (c, crate::back::linker::exported_symbols(tcx, c))) + .collect(); + let linked_symbols = tcx + .sess + .crate_types() + .iter() + .map(|&c| (c, crate::back::linker::linked_symbols(tcx, c))) + .collect(); + let local_crate_name = tcx.crate_name(LOCAL_CRATE); + let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID); + let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem); + let windows_subsystem = subsystem.map(|subsystem| { + if subsystem != sym::windows && subsystem != sym::console { + tcx.sess.fatal(&format!( + "invalid windows subsystem `{}`, only \ + `windows` and `console` are allowed", + subsystem + )); + } + subsystem.to_string() + }); + + // This list is used when generating the command line to pass through to + // system linker. The linker expects undefined symbols on the left of the + // command line to be defined in libraries on the right, not the other way + // around. For more info, see some comments in the add_used_library function + // below. + // + // In order to get this left-to-right dependency ordering, we use the reverse + // postorder of all crates putting the leaves at the right-most positions. + let used_crates = tcx + .postorder_cnums(()) + .iter() + .rev() + .copied() + .filter(|&cnum| !tcx.dep_kind(cnum).macros_only()) + .collect(); + + let mut info = CrateInfo { + target_cpu, + exported_symbols, + linked_symbols, + local_crate_name, + compiler_builtins: None, + profiler_runtime: None, + is_no_builtins: Default::default(), + native_libraries: Default::default(), + used_libraries: tcx.native_libraries(LOCAL_CRATE).iter().map(Into::into).collect(), + crate_name: Default::default(), + used_crates, + used_crate_source: Default::default(), + lang_item_to_crate: Default::default(), + missing_lang_items: Default::default(), + dependency_formats: tcx.dependency_formats(()).clone(), + windows_subsystem, + natvis_debugger_visualizers: Default::default(), + }; + let lang_items = tcx.lang_items(); + + let crates = tcx.crates(()); + + let n_crates = crates.len(); + info.native_libraries.reserve(n_crates); + info.crate_name.reserve(n_crates); + info.used_crate_source.reserve(n_crates); + info.missing_lang_items.reserve(n_crates); + + for &cnum in crates.iter() { + info.native_libraries + .insert(cnum, tcx.native_libraries(cnum).iter().map(Into::into).collect()); + info.crate_name.insert(cnum, tcx.crate_name(cnum)); + + let used_crate_source = tcx.used_crate_source(cnum); + info.used_crate_source.insert(cnum, used_crate_source.clone()); + if tcx.is_compiler_builtins(cnum) { + info.compiler_builtins = Some(cnum); + } + if tcx.is_profiler_runtime(cnum) { + info.profiler_runtime = Some(cnum); + } + if tcx.is_no_builtins(cnum) { + info.is_no_builtins.insert(cnum); + } + let missing = tcx.missing_lang_items(cnum); + for &item in missing.iter() { + if let Ok(id) = lang_items.require(item) { + info.lang_item_to_crate.insert(item, id.krate); + } + } + + // No need to look for lang items that don't actually need to exist. + let missing = + missing.iter().cloned().filter(|&l| lang_items::required(tcx, l)).collect(); + info.missing_lang_items.insert(cnum, missing); + } + + let embed_visualizers = tcx.sess.crate_types().iter().any(|&crate_type| match crate_type { + CrateType::Executable | CrateType::Dylib | CrateType::Cdylib => { + // These are crate types for which we invoke the linker and can embed + // NatVis visualizers. + true + } + CrateType::ProcMacro => { + // We could embed NatVis for proc macro crates too (to improve the debugging + // experience for them) but it does not seem like a good default, since + // this is a rare use case and we don't want to slow down the common case. + false + } + CrateType::Staticlib | CrateType::Rlib => { + // We don't invoke the linker for these, so we don't need to collect the NatVis for them. + false + } + }); + + if tcx.sess.target.is_like_msvc && embed_visualizers { + info.natvis_debugger_visualizers = + collect_debugger_visualizers_transitive(tcx, DebuggerVisualizerType::Natvis); + } + + info + } +} + +pub fn provide(providers: &mut Providers) { + providers.backend_optimization_level = |tcx, cratenum| { + let for_speed = match tcx.sess.opts.optimize { + // If globally no optimisation is done, #[optimize] has no effect. + // + // This is done because if we ended up "upgrading" to `-O2` here, we’d populate the + // pass manager and it is likely that some module-wide passes (such as inliner or + // cross-function constant propagation) would ignore the `optnone` annotation we put + // on the functions, thus necessarily involving these functions into optimisations. + config::OptLevel::No => return config::OptLevel::No, + // If globally optimise-speed is already specified, just use that level. + config::OptLevel::Less => return config::OptLevel::Less, + config::OptLevel::Default => return config::OptLevel::Default, + config::OptLevel::Aggressive => return config::OptLevel::Aggressive, + // If globally optimize-for-size has been requested, use -O2 instead (if optimize(size) + // are present). + config::OptLevel::Size => config::OptLevel::Default, + config::OptLevel::SizeMin => config::OptLevel::Default, + }; + + let (defids, _) = tcx.collect_and_partition_mono_items(cratenum); + for id in &*defids { + let CodegenFnAttrs { optimize, .. } = tcx.codegen_fn_attrs(*id); + match optimize { + attr::OptimizeAttr::None => continue, + attr::OptimizeAttr::Size => continue, + attr::OptimizeAttr::Speed => { + return for_speed; + } + } + } + tcx.sess.opts.optimize + }; +} + +fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguReuse { + if !tcx.dep_graph.is_fully_enabled() { + return CguReuse::No; + } + + let work_product_id = &cgu.work_product_id(); + if tcx.dep_graph.previous_work_product(work_product_id).is_none() { + // We don't have anything cached for this CGU. This can happen + // if the CGU did not exist in the previous session. + return CguReuse::No; + } + + // Try to mark the CGU as green. If it we can do so, it means that nothing + // affecting the LLVM module has changed and we can re-use a cached version. + // If we compile with any kind of LTO, this means we can re-use the bitcode + // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only + // know that later). If we are not doing LTO, there is only one optimized + // version of each module, so we re-use that. + let dep_node = cgu.codegen_dep_node(tcx); + assert!( + !tcx.dep_graph.dep_node_exists(&dep_node), + "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", + cgu.name() + ); + + if tcx.try_mark_green(&dep_node) { + // We can re-use either the pre- or the post-thinlto state. If no LTO is + // being performed then we can use post-LTO artifacts, otherwise we must + // reuse pre-LTO artifacts + match compute_per_cgu_lto_type( + &tcx.sess.lto(), + &tcx.sess.opts, + &tcx.sess.crate_types(), + ModuleKind::Regular, + ) { + ComputedLtoType::No => CguReuse::PostLto, + _ => CguReuse::PreLto, + } + } else { + CguReuse::No + } +} diff --git a/compiler/rustc_codegen_ssa/src/common.rs b/compiler/rustc_codegen_ssa/src/common.rs new file mode 100644 index 00000000000..8ca1a6084cf --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/common.rs @@ -0,0 +1,223 @@ +#![allow(non_camel_case_types)] + +use rustc_errors::struct_span_err; +use rustc_hir as hir; +use rustc_hir::LangItem; +use rustc_middle::mir::interpret::ConstValue; +use rustc_middle::ty::{self, layout::TyAndLayout, Ty, TyCtxt}; +use rustc_session::Session; +use rustc_span::Span; + +use crate::base; +use crate::traits::*; + +#[derive(Copy, Clone)] +pub enum IntPredicate { + IntEQ, + IntNE, + IntUGT, + IntUGE, + IntULT, + IntULE, + IntSGT, + IntSGE, + IntSLT, + IntSLE, +} + +#[derive(Copy, Clone)] +pub enum RealPredicate { + RealPredicateFalse, + RealOEQ, + RealOGT, + RealOGE, + RealOLT, + RealOLE, + RealONE, + RealORD, + RealUNO, + RealUEQ, + RealUGT, + RealUGE, + RealULT, + RealULE, + RealUNE, + RealPredicateTrue, +} + +#[derive(Copy, Clone)] +pub enum AtomicRmwBinOp { + AtomicXchg, + AtomicAdd, + AtomicSub, + AtomicAnd, + AtomicNand, + AtomicOr, + AtomicXor, + AtomicMax, + AtomicMin, + AtomicUMax, + AtomicUMin, +} + +#[derive(Copy, Clone)] +pub enum AtomicOrdering { + Unordered, + Relaxed, + Acquire, + Release, + AcquireRelease, + SequentiallyConsistent, +} + +#[derive(Copy, Clone)] +pub enum SynchronizationScope { + SingleThread, + CrossThread, +} + +#[derive(Copy, Clone, PartialEq, Debug)] +pub enum TypeKind { + Void, + Half, + Float, + Double, + X86_FP80, + FP128, + PPC_FP128, + Label, + Integer, + Function, + Struct, + Array, + Pointer, + Vector, + Metadata, + X86_MMX, + Token, + ScalableVector, + BFloat, + X86_AMX, +} + +// FIXME(mw): Anything that is produced via DepGraph::with_task() must implement +// the HashStable trait. Normally DepGraph::with_task() calls are +// hidden behind queries, but CGU creation is a special case in two +// ways: (1) it's not a query and (2) CGU are output nodes, so their +// Fingerprints are not actually needed. It remains to be clarified +// how exactly this case will be handled in the red/green system but +// for now we content ourselves with providing a no-op HashStable +// implementation for CGUs. +mod temp_stable_hash_impls { + use crate::ModuleCodegen; + use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; + + impl HashStable for ModuleCodegen { + fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) { + // do nothing + } + } +} + +pub fn build_langcall<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &Bx, + span: Option, + li: LangItem, +) -> (Bx::FnAbiOfResult, Bx::Value) { + let tcx = bx.tcx(); + let def_id = tcx.require_lang_item(li, span); + let instance = ty::Instance::mono(tcx, def_id); + (bx.fn_abi_of_instance(instance, ty::List::empty()), bx.get_fn_addr(instance)) +} + +// To avoid UB from LLVM, these two functions mask RHS with an +// appropriate mask unconditionally (i.e., the fallback behavior for +// all shifts). For 32- and 64-bit types, this matches the semantics +// of Java. (See related discussion on #1877 and #10183.) + +pub fn build_unchecked_lshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + lhs: Bx::Value, + rhs: Bx::Value, +) -> Bx::Value { + let rhs = base::cast_shift_expr_rhs(bx, hir::BinOpKind::Shl, lhs, rhs); + // #1877, #10183: Ensure that input is always valid + let rhs = shift_mask_rhs(bx, rhs); + bx.shl(lhs, rhs) +} + +pub fn build_unchecked_rshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + lhs_t: Ty<'tcx>, + lhs: Bx::Value, + rhs: Bx::Value, +) -> Bx::Value { + let rhs = base::cast_shift_expr_rhs(bx, hir::BinOpKind::Shr, lhs, rhs); + // #1877, #10183: Ensure that input is always valid + let rhs = shift_mask_rhs(bx, rhs); + let is_signed = lhs_t.is_signed(); + if is_signed { bx.ashr(lhs, rhs) } else { bx.lshr(lhs, rhs) } +} + +fn shift_mask_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + rhs: Bx::Value, +) -> Bx::Value { + let rhs_llty = bx.val_ty(rhs); + let shift_val = shift_mask_val(bx, rhs_llty, rhs_llty, false); + bx.and(rhs, shift_val) +} + +pub fn shift_mask_val<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + llty: Bx::Type, + mask_llty: Bx::Type, + invert: bool, +) -> Bx::Value { + let kind = bx.type_kind(llty); + match kind { + TypeKind::Integer => { + // i8/u8 can shift by at most 7, i16/u16 by at most 15, etc. + let val = bx.int_width(llty) - 1; + if invert { + bx.const_int(mask_llty, !val as i64) + } else { + bx.const_uint(mask_llty, val) + } + } + TypeKind::Vector => { + let mask = + shift_mask_val(bx, bx.element_type(llty), bx.element_type(mask_llty), invert); + bx.vector_splat(bx.vector_length(mask_llty), mask) + } + _ => bug!("shift_mask_val: expected Integer or Vector, found {:?}", kind), + } +} + +pub fn span_invalid_monomorphization_error(a: &Session, b: Span, c: &str) { + struct_span_err!(a, b, E0511, "{}", c).emit(); +} + +pub fn asm_const_to_str<'tcx>( + tcx: TyCtxt<'tcx>, + sp: Span, + const_value: ConstValue<'tcx>, + ty_and_layout: TyAndLayout<'tcx>, +) -> String { + let ConstValue::Scalar(scalar) = const_value else { + span_bug!(sp, "expected Scalar for promoted asm const, but got {:#?}", const_value) + }; + let value = scalar.assert_bits(ty_and_layout.size); + match ty_and_layout.ty.kind() { + ty::Uint(_) => value.to_string(), + ty::Int(int_ty) => match int_ty.normalize(tcx.sess.target.pointer_width) { + ty::IntTy::I8 => (value as i8).to_string(), + ty::IntTy::I16 => (value as i16).to_string(), + ty::IntTy::I32 => (value as i32).to_string(), + ty::IntTy::I64 => (value as i64).to_string(), + ty::IntTy::I128 => (value as i128).to_string(), + ty::IntTy::Isize => unreachable!(), + }, + _ => span_bug!(sp, "asm const has bad type {}", ty_and_layout.ty), + } +} diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs new file mode 100644 index 00000000000..e288760a02b --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs @@ -0,0 +1,85 @@ +use rustc_middle::mir::coverage::{CounterValueReference, MappedExpressionIndex}; + +/// Aligns with [llvm::coverage::Counter::CounterKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L95) +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub enum CounterKind { + Zero = 0, + CounterValueReference = 1, + Expression = 2, +} + +/// A reference to an instance of an abstract "counter" that will yield a value in a coverage +/// report. Note that `id` has different interpretations, depending on the `kind`: +/// * For `CounterKind::Zero`, `id` is assumed to be `0` +/// * For `CounterKind::CounterValueReference`, `id` matches the `counter_id` of the injected +/// instrumentation counter (the `index` argument to the LLVM intrinsic +/// `instrprof.increment()`) +/// * For `CounterKind::Expression`, `id` is the index into the coverage map's array of +/// counter expressions. +/// Aligns with [llvm::coverage::Counter](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L102-L103) +/// Important: The Rust struct layout (order and types of fields) must match its C++ counterpart. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub struct Counter { + // Important: The layout (order and types of fields) must match its C++ counterpart. + pub kind: CounterKind, + id: u32, +} + +impl Counter { + /// Constructs a new `Counter` of kind `Zero`. For this `CounterKind`, the + /// `id` is not used. + pub fn zero() -> Self { + Self { kind: CounterKind::Zero, id: 0 } + } + + /// Constructs a new `Counter` of kind `CounterValueReference`, and converts + /// the given 1-based counter_id to the required 0-based equivalent for + /// the `Counter` encoding. + pub fn counter_value_reference(counter_id: CounterValueReference) -> Self { + Self { kind: CounterKind::CounterValueReference, id: counter_id.zero_based_index() } + } + + /// Constructs a new `Counter` of kind `Expression`. + pub fn expression(mapped_expression_index: MappedExpressionIndex) -> Self { + Self { kind: CounterKind::Expression, id: mapped_expression_index.into() } + } + + /// Returns true if the `Counter` kind is `Zero`. + pub fn is_zero(&self) -> bool { + matches!(self.kind, CounterKind::Zero) + } + + /// An explicitly-named function to get the ID value, making it more obvious + /// that the stored value is now 0-based. + pub fn zero_based_id(&self) -> u32 { + debug_assert!(!self.is_zero(), "`id` is undefined for CounterKind::Zero"); + self.id + } +} + +/// Aligns with [llvm::coverage::CounterExpression::ExprKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L150) +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub enum ExprKind { + Subtract = 0, + Add = 1, +} + +/// Aligns with [llvm::coverage::CounterExpression](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L152) +/// Important: The Rust struct layout (order and types of fields) must match its C++ +/// counterpart. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub struct CounterExpression { + pub kind: ExprKind, + pub lhs: Counter, + pub rhs: Counter, +} + +impl CounterExpression { + pub fn new(lhs: Counter, kind: ExprKind, rhs: Counter) -> Self { + Self { kind, lhs, rhs } + } +} diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs new file mode 100644 index 00000000000..1a6495cb15c --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs @@ -0,0 +1,347 @@ +pub use super::ffi::*; + +use rustc_index::vec::IndexVec; +use rustc_middle::mir::coverage::{ + CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, + InjectedExpressionIndex, MappedExpressionIndex, Op, +}; +use rustc_middle::ty::Instance; +use rustc_middle::ty::TyCtxt; + +#[derive(Clone, Debug, PartialEq)] +pub struct Expression { + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + region: Option, +} + +/// Collects all of the coverage regions associated with (a) injected counters, (b) counter +/// expressions (additions or subtraction), and (c) unreachable regions (always counted as zero), +/// for a given Function. Counters and counter expressions have non-overlapping `id`s because they +/// can both be operands in an expression. This struct also stores the `function_source_hash`, +/// computed during instrumentation, and forwarded with counters. +/// +/// Note, it may be important to understand LLVM's definitions of `unreachable` regions versus "gap +/// regions" (or "gap areas"). A gap region is a code region within a counted region (either counter +/// or expression), but the line or lines in the gap region are not executable (such as lines with +/// only whitespace or comments). According to LLVM Code Coverage Mapping documentation, "A count +/// for a gap area is only used as the line execution count if there are no other regions on a +/// line." +#[derive(Debug)] +pub struct FunctionCoverage<'tcx> { + instance: Instance<'tcx>, + source_hash: u64, + is_used: bool, + counters: IndexVec>, + expressions: IndexVec>, + unreachable_regions: Vec, +} + +impl<'tcx> FunctionCoverage<'tcx> { + /// Creates a new set of coverage data for a used (called) function. + pub fn new(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { + Self::create(tcx, instance, true) + } + + /// Creates a new set of coverage data for an unused (never called) function. + pub fn unused(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { + Self::create(tcx, instance, false) + } + + fn create(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, is_used: bool) -> Self { + let coverageinfo = tcx.coverageinfo(instance.def); + debug!( + "FunctionCoverage::create(instance={:?}) has coverageinfo={:?}. is_used={}", + instance, coverageinfo, is_used + ); + Self { + instance, + source_hash: 0, // will be set with the first `add_counter()` + is_used, + counters: IndexVec::from_elem_n(None, coverageinfo.num_counters as usize), + expressions: IndexVec::from_elem_n(None, coverageinfo.num_expressions as usize), + unreachable_regions: Vec::new(), + } + } + + /// Returns true for a used (called) function, and false for an unused function. + pub fn is_used(&self) -> bool { + self.is_used + } + + /// Sets the function source hash value. If called multiple times for the same function, all + /// calls should have the same hash value. + pub fn set_function_source_hash(&mut self, source_hash: u64) { + if self.source_hash == 0 { + self.source_hash = source_hash; + } else { + debug_assert_eq!(source_hash, self.source_hash); + } + } + + /// Adds a code region to be counted by an injected counter intrinsic. + pub fn add_counter(&mut self, id: CounterValueReference, region: CodeRegion) { + if let Some(previous_region) = self.counters[id].replace(region.clone()) { + assert_eq!(previous_region, region, "add_counter: code region for id changed"); + } + } + + /// Both counters and "counter expressions" (or simply, "expressions") can be operands in other + /// expressions. Expression IDs start from `u32::MAX` and go down, so the range of expression + /// IDs will not overlap with the range of counter IDs. Counters and expressions can be added in + /// any order, and expressions can still be assigned contiguous (though descending) IDs, without + /// knowing what the last counter ID will be. + /// + /// When storing the expression data in the `expressions` vector in the `FunctionCoverage` + /// struct, its vector index is computed, from the given expression ID, by subtracting from + /// `u32::MAX`. + /// + /// Since the expression operands (`lhs` and `rhs`) can reference either counters or + /// expressions, an operand that references an expression also uses its original ID, descending + /// from `u32::MAX`. Theses operands are translated only during code generation, after all + /// counters and expressions have been added. + pub fn add_counter_expression( + &mut self, + expression_id: InjectedExpressionId, + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + region: Option, + ) { + debug!( + "add_counter_expression({:?}, lhs={:?}, op={:?}, rhs={:?} at {:?}", + expression_id, lhs, op, rhs, region + ); + let expression_index = self.expression_index(u32::from(expression_id)); + debug_assert!( + expression_index.as_usize() < self.expressions.len(), + "expression_index {} is out of range for expressions.len() = {} + for {:?}", + expression_index.as_usize(), + self.expressions.len(), + self, + ); + if let Some(previous_expression) = self.expressions[expression_index].replace(Expression { + lhs, + op, + rhs, + region: region.clone(), + }) { + assert_eq!( + previous_expression, + Expression { lhs, op, rhs, region }, + "add_counter_expression: expression for id changed" + ); + } + } + + /// Add a region that will be marked as "unreachable", with a constant "zero counter". + pub fn add_unreachable_region(&mut self, region: CodeRegion) { + self.unreachable_regions.push(region) + } + + /// Return the source hash, generated from the HIR node structure, and used to indicate whether + /// or not the source code structure changed between different compilations. + pub fn source_hash(&self) -> u64 { + self.source_hash + } + + /// Generate an array of CounterExpressions, and an iterator over all `Counter`s and their + /// associated `Regions` (from which the LLVM-specific `CoverageMapGenerator` will create + /// `CounterMappingRegion`s. + pub fn get_expressions_and_counter_regions( + &self, + ) -> (Vec, impl Iterator) { + assert!( + self.source_hash != 0 || !self.is_used, + "No counters provided the source_hash for used function: {:?}", + self.instance + ); + + let counter_regions = self.counter_regions(); + let (counter_expressions, expression_regions) = self.expressions_with_regions(); + let unreachable_regions = self.unreachable_regions(); + + let counter_regions = + counter_regions.chain(expression_regions.into_iter().chain(unreachable_regions)); + (counter_expressions, counter_regions) + } + + fn counter_regions(&self) -> impl Iterator { + self.counters.iter_enumerated().filter_map(|(index, entry)| { + // Option::map() will return None to filter out missing counters. This may happen + // if, for example, a MIR-instrumented counter is removed during an optimization. + entry.as_ref().map(|region| (Counter::counter_value_reference(index), region)) + }) + } + + fn expressions_with_regions( + &self, + ) -> (Vec, impl Iterator) { + let mut counter_expressions = Vec::with_capacity(self.expressions.len()); + let mut expression_regions = Vec::with_capacity(self.expressions.len()); + let mut new_indexes = IndexVec::from_elem_n(None, self.expressions.len()); + + // This closure converts any `Expression` operand (`lhs` or `rhs` of the `Op::Add` or + // `Op::Subtract` operation) into its native `llvm::coverage::Counter::CounterKind` type + // and value. Operand ID value `0` maps to `CounterKind::Zero`; values in the known range + // of injected LLVM counters map to `CounterKind::CounterValueReference` (and the value + // matches the injected counter index); and any other value is converted into a + // `CounterKind::Expression` with the expression's `new_index`. + // + // Expressions will be returned from this function in a sequential vector (array) of + // `CounterExpression`, so the expression IDs must be mapped from their original, + // potentially sparse set of indexes, originally in reverse order from `u32::MAX`. + // + // An `Expression` as an operand will have already been encountered as an `Expression` with + // operands, so its new_index will already have been generated (as a 1-up index value). + // (If an `Expression` as an operand does not have a corresponding new_index, it was + // probably optimized out, after the expression was injected into the MIR, so it will + // get a `CounterKind::Zero` instead.) + // + // In other words, an `Expression`s at any given index can include other expressions as + // operands, but expression operands can only come from the subset of expressions having + // `expression_index`s lower than the referencing `Expression`. Therefore, it is + // reasonable to look up the new index of an expression operand while the `new_indexes` + // vector is only complete up to the current `ExpressionIndex`. + let id_to_counter = |new_indexes: &IndexVec< + InjectedExpressionIndex, + Option, + >, + id: ExpressionOperandId| { + if id == ExpressionOperandId::ZERO { + Some(Counter::zero()) + } else if id.index() < self.counters.len() { + debug_assert!( + id.index() > 0, + "ExpressionOperandId indexes for counters are 1-based, but this id={}", + id.index() + ); + // Note: Some codegen-injected Counters may be only referenced by `Expression`s, + // and may not have their own `CodeRegion`s, + let index = CounterValueReference::from(id.index()); + // Note, the conversion to LLVM `Counter` adjusts the index to be zero-based. + Some(Counter::counter_value_reference(index)) + } else { + let index = self.expression_index(u32::from(id)); + self.expressions + .get(index) + .expect("expression id is out of range") + .as_ref() + // If an expression was optimized out, assume it would have produced a count + // of zero. This ensures that expressions dependent on optimized-out + // expressions are still valid. + .map_or(Some(Counter::zero()), |_| new_indexes[index].map(Counter::expression)) + } + }; + + for (original_index, expression) in + self.expressions.iter_enumerated().filter_map(|(original_index, entry)| { + // Option::map() will return None to filter out missing expressions. This may happen + // if, for example, a MIR-instrumented expression is removed during an optimization. + entry.as_ref().map(|expression| (original_index, expression)) + }) + { + let optional_region = &expression.region; + let Expression { lhs, op, rhs, .. } = *expression; + + if let Some(Some((lhs_counter, mut rhs_counter))) = id_to_counter(&new_indexes, lhs) + .map(|lhs_counter| { + id_to_counter(&new_indexes, rhs).map(|rhs_counter| (lhs_counter, rhs_counter)) + }) + { + if lhs_counter.is_zero() && op.is_subtract() { + // The left side of a subtraction was probably optimized out. As an example, + // a branch condition might be evaluated as a constant expression, and the + // branch could be removed, dropping unused counters in the process. + // + // Since counters are unsigned, we must assume the result of the expression + // can be no more and no less than zero. An expression known to evaluate to zero + // does not need to be added to the coverage map. + // + // Coverage test `loops_branches.rs` includes multiple variations of branches + // based on constant conditional (literal `true` or `false`), and demonstrates + // that the expected counts are still correct. + debug!( + "Expression subtracts from zero (assume unreachable): \ + original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}", + original_index, lhs, op, rhs, optional_region, + ); + rhs_counter = Counter::zero(); + } + debug_assert!( + lhs_counter.is_zero() + // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16` + || ((lhs_counter.zero_based_id() as usize) + <= usize::max(self.counters.len(), self.expressions.len())), + "lhs id={} > both counters.len()={} and expressions.len()={} + ({:?} {:?} {:?})", + lhs_counter.zero_based_id(), + self.counters.len(), + self.expressions.len(), + lhs_counter, + op, + rhs_counter, + ); + + debug_assert!( + rhs_counter.is_zero() + // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16` + || ((rhs_counter.zero_based_id() as usize) + <= usize::max(self.counters.len(), self.expressions.len())), + "rhs id={} > both counters.len()={} and expressions.len()={} + ({:?} {:?} {:?})", + rhs_counter.zero_based_id(), + self.counters.len(), + self.expressions.len(), + lhs_counter, + op, + rhs_counter, + ); + + // Both operands exist. `Expression` operands exist in `self.expressions` and have + // been assigned a `new_index`. + let mapped_expression_index = + MappedExpressionIndex::from(counter_expressions.len()); + let expression = CounterExpression::new( + lhs_counter, + match op { + Op::Add => ExprKind::Add, + Op::Subtract => ExprKind::Subtract, + }, + rhs_counter, + ); + debug!( + "Adding expression {:?} = {:?}, region: {:?}", + mapped_expression_index, expression, optional_region + ); + counter_expressions.push(expression); + new_indexes[original_index] = Some(mapped_expression_index); + if let Some(region) = optional_region { + expression_regions.push((Counter::expression(mapped_expression_index), region)); + } + } else { + bug!( + "expression has one or more missing operands \ + original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}", + original_index, + lhs, + op, + rhs, + optional_region, + ); + } + } + (counter_expressions, expression_regions.into_iter()) + } + + fn unreachable_regions(&self) -> impl Iterator { + self.unreachable_regions.iter().map(|region| (Counter::zero(), region)) + } + + fn expression_index(&self, id_descending_from_max: u32) -> InjectedExpressionIndex { + debug_assert!(id_descending_from_max >= self.counters.len() as u32); + InjectedExpressionIndex::from(u32::MAX - id_descending_from_max) + } +} diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs new file mode 100644 index 00000000000..569fd3f1a51 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs @@ -0,0 +1,2 @@ +pub mod ffi; +pub mod map; diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/mod.rs b/compiler/rustc_codegen_ssa/src/debuginfo/mod.rs new file mode 100644 index 00000000000..6e3f4f0b8ef --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/debuginfo/mod.rs @@ -0,0 +1,34 @@ +use rustc_middle::ty::{self, layout::TyAndLayout}; +use rustc_target::abi::Size; + +// FIXME(eddyb) find a place for this (or a way to replace it). +pub mod type_names; + +/// Returns true if we want to generate a DW_TAG_enumeration_type description for +/// this instead of a DW_TAG_struct_type with DW_TAG_variant_part. +/// +/// NOTE: This is somewhat inconsistent right now: For empty enums and enums with a single +/// fieldless variant, we generate DW_TAG_struct_type, although a +/// DW_TAG_enumeration_type would be a better fit. +pub fn wants_c_like_enum_debuginfo<'tcx>(enum_type_and_layout: TyAndLayout<'tcx>) -> bool { + match enum_type_and_layout.ty.kind() { + ty::Adt(adt_def, _) => { + if !adt_def.is_enum() { + return false; + } + + match adt_def.variants().len() { + 0 => false, + 1 => { + // Univariant enums unless they are zero-sized + enum_type_and_layout.size != Size::ZERO && adt_def.all_fields().count() == 0 + } + _ => { + // Enums with more than one variant if they have no fields + adt_def.all_fields().count() == 0 + } + } + } + _ => false, + } +} diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs new file mode 100644 index 00000000000..8cd5a0fc247 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs @@ -0,0 +1,821 @@ +// Type Names for Debug Info. + +// Notes on targeting MSVC: +// In general, MSVC's debugger attempts to parse all arguments as C++ expressions, +// even if the argument is explicitly a symbol name. +// As such, there are many things that cause parsing issues: +// * `#` is treated as a special character for macros. +// * `{` or `<` at the beginning of a name is treated as an operator. +// * `>>` is always treated as a right-shift. +// * `[` in a name is treated like a regex bracket expression (match any char +// within the brackets). +// * `"` is treated as the start of a string. + +use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_hir::def_id::DefId; +use rustc_hir::definitions::{DefPathData, DefPathDataName, DisambiguatedDefPathData}; +use rustc_hir::{AsyncGeneratorKind, GeneratorKind, Mutability}; +use rustc_middle::ty::layout::{IntegerExt, TyAndLayout}; +use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; +use rustc_middle::ty::{self, ExistentialProjection, GeneratorSubsts, ParamEnv, Ty, TyCtxt}; +use rustc_target::abi::{Integer, TagEncoding, Variants}; +use smallvec::SmallVec; + +use std::borrow::Cow; +use std::fmt::Write; + +use crate::debuginfo::wants_c_like_enum_debuginfo; + +// Compute the name of the type as it should be stored in debuginfo. Does not do +// any caching, i.e., calling the function twice with the same type will also do +// the work twice. The `qualified` parameter only affects the first level of the +// type name, further levels (i.e., type parameters) are always fully qualified. +pub fn compute_debuginfo_type_name<'tcx>( + tcx: TyCtxt<'tcx>, + t: Ty<'tcx>, + qualified: bool, +) -> String { + let _prof = tcx.prof.generic_activity("compute_debuginfo_type_name"); + + let mut result = String::with_capacity(64); + let mut visited = FxHashSet::default(); + push_debuginfo_type_name(tcx, t, qualified, &mut result, &mut visited); + result +} + +// Pushes the name of the type as it should be stored in debuginfo on the +// `output` String. See also compute_debuginfo_type_name(). +fn push_debuginfo_type_name<'tcx>( + tcx: TyCtxt<'tcx>, + t: Ty<'tcx>, + qualified: bool, + output: &mut String, + visited: &mut FxHashSet>, +) { + // When targeting MSVC, emit C++ style type names for compatibility with + // .natvis visualizers (and perhaps other existing native debuggers?) + let cpp_like_debuginfo = cpp_like_debuginfo(tcx); + + match *t.kind() { + ty::Bool => output.push_str("bool"), + ty::Char => output.push_str("char"), + ty::Str => output.push_str("str"), + ty::Never => { + if cpp_like_debuginfo { + output.push_str("never$"); + } else { + output.push('!'); + } + } + ty::Int(int_ty) => output.push_str(int_ty.name_str()), + ty::Uint(uint_ty) => output.push_str(uint_ty.name_str()), + ty::Float(float_ty) => output.push_str(float_ty.name_str()), + ty::Foreign(def_id) => push_item_name(tcx, def_id, qualified, output), + ty::Adt(def, substs) => { + // `layout_for_cpp_like_fallback` will be `Some` if we want to use the fallback encoding. + let layout_for_cpp_like_fallback = if cpp_like_debuginfo && def.is_enum() { + match tcx.layout_of(ParamEnv::reveal_all().and(t)) { + Ok(layout) => { + if !wants_c_like_enum_debuginfo(layout) { + Some(layout) + } else { + // This is a C-like enum so we don't want to use the fallback encoding + // for the name. + None + } + } + Err(e) => { + // Computing the layout can still fail here, e.g. if the target architecture + // cannot represent the type. See https://github.com/rust-lang/rust/issues/94961. + tcx.sess.fatal(&format!("{}", e)); + } + } + } else { + // We are not emitting cpp-like debuginfo or this isn't even an enum. + None + }; + + if let Some(ty_and_layout) = layout_for_cpp_like_fallback { + msvc_enum_fallback( + tcx, + ty_and_layout, + &|output, visited| { + push_item_name(tcx, def.did(), true, output); + push_generic_params_internal(tcx, substs, output, visited); + }, + output, + visited, + ); + } else { + push_item_name(tcx, def.did(), qualified, output); + push_generic_params_internal(tcx, substs, output, visited); + } + } + ty::Tuple(component_types) => { + if cpp_like_debuginfo { + output.push_str("tuple$<"); + } else { + output.push('('); + } + + for component_type in component_types { + push_debuginfo_type_name(tcx, component_type, true, output, visited); + push_arg_separator(cpp_like_debuginfo, output); + } + if !component_types.is_empty() { + pop_arg_separator(output); + } + + if cpp_like_debuginfo { + push_close_angle_bracket(cpp_like_debuginfo, output); + } else { + output.push(')'); + } + } + ty::RawPtr(ty::TypeAndMut { ty: inner_type, mutbl }) => { + if cpp_like_debuginfo { + match mutbl { + Mutability::Not => output.push_str("ptr_const$<"), + Mutability::Mut => output.push_str("ptr_mut$<"), + } + } else { + output.push('*'); + match mutbl { + Mutability::Not => output.push_str("const "), + Mutability::Mut => output.push_str("mut "), + } + } + + push_debuginfo_type_name(tcx, inner_type, qualified, output, visited); + + if cpp_like_debuginfo { + push_close_angle_bracket(cpp_like_debuginfo, output); + } + } + ty::Ref(_, inner_type, mutbl) => { + // Slices and `&str` are treated like C++ pointers when computing debug + // info for MSVC debugger. However, wrapping these types' names in a synthetic type + // causes the .natvis engine for WinDbg to fail to display their data, so we opt these + // types out to aid debugging in MSVC. + let is_slice_or_str = matches!(*inner_type.kind(), ty::Slice(_) | ty::Str); + + if !cpp_like_debuginfo { + output.push('&'); + output.push_str(mutbl.prefix_str()); + } else if !is_slice_or_str { + match mutbl { + Mutability::Not => output.push_str("ref$<"), + Mutability::Mut => output.push_str("ref_mut$<"), + } + } + + push_debuginfo_type_name(tcx, inner_type, qualified, output, visited); + + if cpp_like_debuginfo && !is_slice_or_str { + push_close_angle_bracket(cpp_like_debuginfo, output); + } + } + ty::Array(inner_type, len) => { + if cpp_like_debuginfo { + output.push_str("array$<"); + push_debuginfo_type_name(tcx, inner_type, true, output, visited); + match len.kind() { + ty::ConstKind::Param(param) => write!(output, ",{}>", param.name).unwrap(), + _ => write!(output, ",{}>", len.eval_usize(tcx, ty::ParamEnv::reveal_all())) + .unwrap(), + } + } else { + output.push('['); + push_debuginfo_type_name(tcx, inner_type, true, output, visited); + match len.kind() { + ty::ConstKind::Param(param) => write!(output, "; {}]", param.name).unwrap(), + _ => write!(output, "; {}]", len.eval_usize(tcx, ty::ParamEnv::reveal_all())) + .unwrap(), + } + } + } + ty::Slice(inner_type) => { + if cpp_like_debuginfo { + output.push_str("slice$<"); + } else { + output.push('['); + } + + push_debuginfo_type_name(tcx, inner_type, true, output, visited); + + if cpp_like_debuginfo { + push_close_angle_bracket(cpp_like_debuginfo, output); + } else { + output.push(']'); + } + } + ty::Dynamic(ref trait_data, ..) => { + let auto_traits: SmallVec<[DefId; 4]> = trait_data.auto_traits().collect(); + + let has_enclosing_parens = if cpp_like_debuginfo { + output.push_str("dyn$<"); + false + } else { + if trait_data.len() > 1 && auto_traits.len() != 0 { + // We need enclosing parens because there is more than one trait + output.push_str("(dyn "); + true + } else { + output.push_str("dyn "); + false + } + }; + + if let Some(principal) = trait_data.principal() { + let principal = + tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), principal); + push_item_name(tcx, principal.def_id, qualified, output); + let principal_has_generic_params = + push_generic_params_internal(tcx, principal.substs, output, visited); + + let projection_bounds: SmallVec<[_; 4]> = trait_data + .projection_bounds() + .map(|bound| { + let ExistentialProjection { item_def_id, term, .. } = + tcx.erase_late_bound_regions(bound); + // FIXME(associated_const_equality): allow for consts here + (item_def_id, term.ty().unwrap()) + }) + .collect(); + + if projection_bounds.len() != 0 { + if principal_has_generic_params { + // push_generic_params_internal() above added a `>` but we actually + // want to add more items to that list, so remove that again... + pop_close_angle_bracket(output); + // .. and add a comma to separate the regular generic args from the + // associated types. + push_arg_separator(cpp_like_debuginfo, output); + } else { + // push_generic_params_internal() did not add `<...>`, so we open + // angle brackets here. + output.push('<'); + } + + for (item_def_id, ty) in projection_bounds { + if cpp_like_debuginfo { + output.push_str("assoc$<"); + push_item_name(tcx, item_def_id, false, output); + push_arg_separator(cpp_like_debuginfo, output); + push_debuginfo_type_name(tcx, ty, true, output, visited); + push_close_angle_bracket(cpp_like_debuginfo, output); + } else { + push_item_name(tcx, item_def_id, false, output); + output.push('='); + push_debuginfo_type_name(tcx, ty, true, output, visited); + } + push_arg_separator(cpp_like_debuginfo, output); + } + + pop_arg_separator(output); + push_close_angle_bracket(cpp_like_debuginfo, output); + } + + if auto_traits.len() != 0 { + push_auto_trait_separator(cpp_like_debuginfo, output); + } + } + + if auto_traits.len() != 0 { + let mut auto_traits: SmallVec<[String; 4]> = auto_traits + .into_iter() + .map(|def_id| { + let mut name = String::with_capacity(20); + push_item_name(tcx, def_id, true, &mut name); + name + }) + .collect(); + auto_traits.sort_unstable(); + + for auto_trait in auto_traits { + output.push_str(&auto_trait); + push_auto_trait_separator(cpp_like_debuginfo, output); + } + + pop_auto_trait_separator(output); + } + + if cpp_like_debuginfo { + push_close_angle_bracket(cpp_like_debuginfo, output); + } else if has_enclosing_parens { + output.push(')'); + } + } + ty::FnDef(..) | ty::FnPtr(_) => { + // We've encountered a weird 'recursive type' + // Currently, the only way to generate such a type + // is by using 'impl trait': + // + // fn foo() -> impl Copy { foo } + // + // There's not really a sensible name we can generate, + // since we don't include 'impl trait' types (e.g. ty::Opaque) + // in the output + // + // Since we need to generate *something*, we just + // use a dummy string that should make it clear + // that something unusual is going on + if !visited.insert(t) { + output.push_str(if cpp_like_debuginfo { + "recursive_type$" + } else { + "" + }); + return; + } + + let sig = + tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), t.fn_sig(tcx)); + + if cpp_like_debuginfo { + // Format as a C++ function pointer: return_type (*)(params...) + if sig.output().is_unit() { + output.push_str("void"); + } else { + push_debuginfo_type_name(tcx, sig.output(), true, output, visited); + } + output.push_str(" (*)("); + } else { + output.push_str(sig.unsafety.prefix_str()); + + if sig.abi != rustc_target::spec::abi::Abi::Rust { + output.push_str("extern \""); + output.push_str(sig.abi.name()); + output.push_str("\" "); + } + + output.push_str("fn("); + } + + if !sig.inputs().is_empty() { + for ¶meter_type in sig.inputs() { + push_debuginfo_type_name(tcx, parameter_type, true, output, visited); + push_arg_separator(cpp_like_debuginfo, output); + } + pop_arg_separator(output); + } + + if sig.c_variadic { + if !sig.inputs().is_empty() { + output.push_str(", ..."); + } else { + output.push_str("..."); + } + } + + output.push(')'); + + if !cpp_like_debuginfo && !sig.output().is_unit() { + output.push_str(" -> "); + push_debuginfo_type_name(tcx, sig.output(), true, output, visited); + } + + // We only keep the type in 'visited' + // for the duration of the body of this method. + // It's fine for a particular function type + // to show up multiple times in one overall type + // (e.g. MyType u8, fn() -> u8> + // + // We only care about avoiding recursing + // directly back to the type we're currently + // processing + visited.remove(&t); + } + ty::Closure(def_id, substs) | ty::Generator(def_id, substs, ..) => { + // Name will be "{closure_env#0}", "{generator_env#0}", or + // "{async_fn_env#0}", etc. + // In the case of cpp-like debuginfo, the name additionally gets wrapped inside of + // an artificial `enum$<>` type, as defined in msvc_enum_fallback(). + if cpp_like_debuginfo && t.is_generator() { + let ty_and_layout = tcx.layout_of(ParamEnv::reveal_all().and(t)).unwrap(); + msvc_enum_fallback( + tcx, + ty_and_layout, + &|output, visited| { + push_closure_or_generator_name(tcx, def_id, substs, true, output, visited); + }, + output, + visited, + ); + } else { + push_closure_or_generator_name(tcx, def_id, substs, qualified, output, visited); + } + } + // Type parameters from polymorphized functions. + ty::Param(_) => { + write!(output, "{:?}", t).unwrap(); + } + ty::Error(_) + | ty::Infer(_) + | ty::Placeholder(..) + | ty::Projection(..) + | ty::Bound(..) + | ty::Opaque(..) + | ty::GeneratorWitness(..) => { + bug!( + "debuginfo: Trying to create type name for \ + unexpected type: {:?}", + t + ); + } + } + + /// MSVC names enums differently than other platforms so that the debugging visualization + // format (natvis) is able to understand enums and render the active variant correctly in the + // debugger. For more information, look in `src/etc/natvis/intrinsic.natvis` and + // `EnumMemberDescriptionFactor::create_member_descriptions`. + fn msvc_enum_fallback<'tcx>( + tcx: TyCtxt<'tcx>, + ty_and_layout: TyAndLayout<'tcx>, + push_inner: &dyn Fn(/*output*/ &mut String, /*visited*/ &mut FxHashSet>), + output: &mut String, + visited: &mut FxHashSet>, + ) { + debug_assert!(!wants_c_like_enum_debuginfo(ty_and_layout)); + let ty = ty_and_layout.ty; + + output.push_str("enum$<"); + push_inner(output, visited); + + let variant_name = |variant_index| match ty.kind() { + ty::Adt(adt_def, _) => { + debug_assert!(adt_def.is_enum()); + Cow::from(adt_def.variant(variant_index).name.as_str()) + } + ty::Generator(..) => GeneratorSubsts::variant_name(variant_index), + _ => unreachable!(), + }; + + if let Variants::Multiple { + tag_encoding: TagEncoding::Niche { dataful_variant, .. }, + tag, + variants, + .. + } = &ty_and_layout.variants + { + let dataful_variant_layout = &variants[*dataful_variant]; + + // calculate the range of values for the dataful variant + let dataful_discriminant_range = + dataful_variant_layout.largest_niche().unwrap().valid_range; + + let min = dataful_discriminant_range.start; + let min = tag.size(&tcx).truncate(min); + + let max = dataful_discriminant_range.end; + let max = tag.size(&tcx).truncate(max); + + let dataful_variant_name = variant_name(*dataful_variant); + write!(output, ", {}, {}, {}", min, max, dataful_variant_name).unwrap(); + } else if let Variants::Single { index: variant_idx } = &ty_and_layout.variants { + // Uninhabited enums can't be constructed and should never need to be visualized so + // skip this step for them. + if !ty_and_layout.abi.is_uninhabited() { + write!(output, ", {}", variant_name(*variant_idx)).unwrap(); + } + } + push_close_angle_bracket(true, output); + } + + const NON_CPP_AUTO_TRAIT_SEPARATOR: &str = " + "; + + fn push_auto_trait_separator(cpp_like_debuginfo: bool, output: &mut String) { + if cpp_like_debuginfo { + push_arg_separator(cpp_like_debuginfo, output); + } else { + output.push_str(NON_CPP_AUTO_TRAIT_SEPARATOR); + } + } + + fn pop_auto_trait_separator(output: &mut String) { + if output.ends_with(NON_CPP_AUTO_TRAIT_SEPARATOR) { + output.truncate(output.len() - NON_CPP_AUTO_TRAIT_SEPARATOR.len()); + } else { + pop_arg_separator(output); + } + } +} + +pub enum VTableNameKind { + // Is the name for the const/static holding the vtable? + GlobalVariable, + // Is the name for the type of the vtable? + Type, +} + +/// Computes a name for the global variable storing a vtable (or the type of that global variable). +/// +/// The name is of the form: +/// +/// `::{vtable}` +/// +/// or, when generating C++-like names: +/// +/// `impl$::vtable$` +/// +/// If `kind` is `VTableNameKind::Type` then the last component is `{vtable_ty}` instead of just +/// `{vtable}`, so that the type and the corresponding global variable get assigned different +/// names. +pub fn compute_debuginfo_vtable_name<'tcx>( + tcx: TyCtxt<'tcx>, + t: Ty<'tcx>, + trait_ref: Option>, + kind: VTableNameKind, +) -> String { + let cpp_like_debuginfo = cpp_like_debuginfo(tcx); + + let mut vtable_name = String::with_capacity(64); + + if cpp_like_debuginfo { + vtable_name.push_str("impl$<"); + } else { + vtable_name.push('<'); + } + + let mut visited = FxHashSet::default(); + push_debuginfo_type_name(tcx, t, true, &mut vtable_name, &mut visited); + + if cpp_like_debuginfo { + vtable_name.push_str(", "); + } else { + vtable_name.push_str(" as "); + } + + if let Some(trait_ref) = trait_ref { + let trait_ref = + tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), trait_ref); + push_item_name(tcx, trait_ref.def_id, true, &mut vtable_name); + visited.clear(); + push_generic_params_internal(tcx, trait_ref.substs, &mut vtable_name, &mut visited); + } else { + vtable_name.push_str("_"); + } + + push_close_angle_bracket(cpp_like_debuginfo, &mut vtable_name); + + let suffix = match (cpp_like_debuginfo, kind) { + (true, VTableNameKind::GlobalVariable) => "::vtable$", + (false, VTableNameKind::GlobalVariable) => "::{vtable}", + (true, VTableNameKind::Type) => "::vtable_type$", + (false, VTableNameKind::Type) => "::{vtable_type}", + }; + + vtable_name.reserve_exact(suffix.len()); + vtable_name.push_str(suffix); + + vtable_name +} + +pub fn push_item_name(tcx: TyCtxt<'_>, def_id: DefId, qualified: bool, output: &mut String) { + let def_key = tcx.def_key(def_id); + if qualified { + if let Some(parent) = def_key.parent { + push_item_name(tcx, DefId { krate: def_id.krate, index: parent }, true, output); + output.push_str("::"); + } + } + + push_unqualified_item_name(tcx, def_id, def_key.disambiguated_data, output); +} + +fn generator_kind_label(generator_kind: Option) -> &'static str { + match generator_kind { + Some(GeneratorKind::Async(AsyncGeneratorKind::Block)) => "async_block", + Some(GeneratorKind::Async(AsyncGeneratorKind::Closure)) => "async_closure", + Some(GeneratorKind::Async(AsyncGeneratorKind::Fn)) => "async_fn", + Some(GeneratorKind::Gen) => "generator", + None => "closure", + } +} + +fn push_disambiguated_special_name( + label: &str, + disambiguator: u32, + cpp_like_debuginfo: bool, + output: &mut String, +) { + if cpp_like_debuginfo { + write!(output, "{}${}", label, disambiguator).unwrap(); + } else { + write!(output, "{{{}#{}}}", label, disambiguator).unwrap(); + } +} + +fn push_unqualified_item_name( + tcx: TyCtxt<'_>, + def_id: DefId, + disambiguated_data: DisambiguatedDefPathData, + output: &mut String, +) { + match disambiguated_data.data { + DefPathData::CrateRoot => { + output.push_str(tcx.crate_name(def_id.krate).as_str()); + } + DefPathData::ClosureExpr => { + let label = generator_kind_label(tcx.generator_kind(def_id)); + + push_disambiguated_special_name( + label, + disambiguated_data.disambiguator, + cpp_like_debuginfo(tcx), + output, + ); + } + _ => match disambiguated_data.data.name() { + DefPathDataName::Named(name) => { + output.push_str(name.as_str()); + } + DefPathDataName::Anon { namespace } => { + push_disambiguated_special_name( + namespace.as_str(), + disambiguated_data.disambiguator, + cpp_like_debuginfo(tcx), + output, + ); + } + }, + }; +} + +fn push_generic_params_internal<'tcx>( + tcx: TyCtxt<'tcx>, + substs: SubstsRef<'tcx>, + output: &mut String, + visited: &mut FxHashSet>, +) -> bool { + if substs.non_erasable_generics().next().is_none() { + return false; + } + + debug_assert_eq!(substs, tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs)); + + let cpp_like_debuginfo = cpp_like_debuginfo(tcx); + + output.push('<'); + + for type_parameter in substs.non_erasable_generics() { + match type_parameter { + GenericArgKind::Type(type_parameter) => { + push_debuginfo_type_name(tcx, type_parameter, true, output, visited); + } + GenericArgKind::Const(ct) => { + push_const_param(tcx, ct, output); + } + other => bug!("Unexpected non-erasable generic: {:?}", other), + } + + push_arg_separator(cpp_like_debuginfo, output); + } + pop_arg_separator(output); + push_close_angle_bracket(cpp_like_debuginfo, output); + + true +} + +fn push_const_param<'tcx>(tcx: TyCtxt<'tcx>, ct: ty::Const<'tcx>, output: &mut String) { + match ct.kind() { + ty::ConstKind::Param(param) => { + write!(output, "{}", param.name) + } + _ => match ct.ty().kind() { + ty::Int(ity) => { + let bits = ct.eval_bits(tcx, ty::ParamEnv::reveal_all(), ct.ty()); + let val = Integer::from_int_ty(&tcx, *ity).size().sign_extend(bits) as i128; + write!(output, "{}", val) + } + ty::Uint(_) => { + let val = ct.eval_bits(tcx, ty::ParamEnv::reveal_all(), ct.ty()); + write!(output, "{}", val) + } + ty::Bool => { + let val = ct.try_eval_bool(tcx, ty::ParamEnv::reveal_all()).unwrap(); + write!(output, "{}", val) + } + _ => { + // If we cannot evaluate the constant to a known type, we fall back + // to emitting a stable hash value of the constant. This isn't very pretty + // but we get a deterministic, virtually unique value for the constant. + // + // Let's only emit 64 bits of the hash value. That should be plenty for + // avoiding collisions and will make the emitted type names shorter. + let hash_short = tcx.with_stable_hashing_context(|mut hcx| { + let mut hasher = StableHasher::new(); + let ct = ct.eval(tcx, ty::ParamEnv::reveal_all()); + hcx.while_hashing_spans(false, |hcx| { + ct.to_valtree().hash_stable(hcx, &mut hasher) + }); + // Note: Don't use `StableHashResult` impl of `u64` here directly, since that + // would lead to endianness problems. + let hash: u128 = hasher.finish(); + (hash.to_le() as u64).to_le() + }); + + if cpp_like_debuginfo(tcx) { + write!(output, "CONST${:x}", hash_short) + } else { + write!(output, "{{CONST#{:x}}}", hash_short) + } + } + }, + } + .unwrap(); +} + +pub fn push_generic_params<'tcx>(tcx: TyCtxt<'tcx>, substs: SubstsRef<'tcx>, output: &mut String) { + let _prof = tcx.prof.generic_activity("compute_debuginfo_type_name"); + let mut visited = FxHashSet::default(); + push_generic_params_internal(tcx, substs, output, &mut visited); +} + +fn push_closure_or_generator_name<'tcx>( + tcx: TyCtxt<'tcx>, + def_id: DefId, + substs: SubstsRef<'tcx>, + qualified: bool, + output: &mut String, + visited: &mut FxHashSet>, +) { + // Name will be "{closure_env#0}", "{generator_env#0}", or + // "{async_fn_env#0}", etc. + let def_key = tcx.def_key(def_id); + let generator_kind = tcx.generator_kind(def_id); + + if qualified { + let parent_def_id = DefId { index: def_key.parent.unwrap(), ..def_id }; + push_item_name(tcx, parent_def_id, true, output); + output.push_str("::"); + } + + let mut label = String::with_capacity(20); + write!(&mut label, "{}_env", generator_kind_label(generator_kind)).unwrap(); + + push_disambiguated_special_name( + &label, + def_key.disambiguated_data.disambiguator, + cpp_like_debuginfo(tcx), + output, + ); + + // We also need to add the generic arguments of the async fn/generator or + // the enclosing function (for closures or async blocks), so that we end + // up with a unique name for every instantiation. + + // Find the generics of the enclosing function, as defined in the source code. + let enclosing_fn_def_id = tcx.typeck_root_def_id(def_id); + let generics = tcx.generics_of(enclosing_fn_def_id); + + // Truncate the substs to the length of the above generics. This will cut off + // anything closure- or generator-specific. + let substs = substs.truncate_to(tcx, generics); + push_generic_params_internal(tcx, substs, output, visited); +} + +fn push_close_angle_bracket(cpp_like_debuginfo: bool, output: &mut String) { + // MSVC debugger always treats `>>` as a shift, even when parsing templates, + // so add a space to avoid confusion. + if cpp_like_debuginfo && output.ends_with('>') { + output.push(' ') + }; + + output.push('>'); +} + +fn pop_close_angle_bracket(output: &mut String) { + assert!(output.ends_with('>'), "'output' does not end with '>': {}", output); + output.pop(); + if output.ends_with(' ') { + output.pop(); + } +} + +fn push_arg_separator(cpp_like_debuginfo: bool, output: &mut String) { + // Natvis does not always like having spaces between parts of the type name + // and this causes issues when we need to write a typename in natvis, for example + // as part of a cast like the `HashMap` visualizer does. + if cpp_like_debuginfo { + output.push(','); + } else { + output.push_str(", "); + }; +} + +fn pop_arg_separator(output: &mut String) { + if output.ends_with(' ') { + output.pop(); + } + + assert!(output.ends_with(',')); + + output.pop(); +} + +/// Check if we should generate C++ like names and debug information. +pub fn cpp_like_debuginfo(tcx: TyCtxt<'_>) -> bool { + tcx.sess.target.is_like_msvc +} diff --git a/compiler/rustc_codegen_ssa/src/glue.rs b/compiler/rustc_codegen_ssa/src/glue.rs new file mode 100644 index 00000000000..e6f402ef19d --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/glue.rs @@ -0,0 +1,123 @@ +//! +// +// Code relating to drop glue. + +use crate::common::IntPredicate; +use crate::meth; +use crate::traits::*; +use rustc_middle::ty::{self, Ty}; +use rustc_target::abi::WrappingRange; + +pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + t: Ty<'tcx>, + info: Option, +) -> (Bx::Value, Bx::Value) { + let layout = bx.layout_of(t); + debug!("size_and_align_of_dst(ty={}, info={:?}): layout: {:?}", t, info, layout); + if !layout.is_unsized() { + let size = bx.const_usize(layout.size.bytes()); + let align = bx.const_usize(layout.align.abi.bytes()); + return (size, align); + } + match t.kind() { + ty::Dynamic(..) => { + // Load size/align from vtable. + let vtable = info.unwrap(); + let size = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_SIZE) + .get_usize(bx, vtable); + let align = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_ALIGN) + .get_usize(bx, vtable); + + // Alignment is always nonzero. + bx.range_metadata(align, WrappingRange { start: 1, end: !0 }); + + (size, align) + } + ty::Slice(_) | ty::Str => { + let unit = layout.field(bx, 0); + // The info in this case is the length of the str, so the size is that + // times the unit size. + ( + // All slice sizes must fit into `isize`, so this multiplication cannot (signed) wrap. + // NOTE: ideally, we want the effects of both `unchecked_smul` and `unchecked_umul` + // (resulting in `mul nsw nuw` in LLVM IR), since we know that the multiplication + // cannot signed wrap, and that both operands are non-negative. But at the time of writing, + // `BuilderMethods` can't do this, and it doesn't seem to enable any further optimizations. + bx.unchecked_smul(info.unwrap(), bx.const_usize(unit.size.bytes())), + bx.const_usize(unit.align.abi.bytes()), + ) + } + _ => { + // First get the size of all statically known fields. + // Don't use size_of because it also rounds up to alignment, which we + // want to avoid, as the unsized field's alignment could be smaller. + assert!(!t.is_simd()); + debug!("DST {} layout: {:?}", t, layout); + + let i = layout.fields.count() - 1; + let sized_size = layout.fields.offset(i).bytes(); + let sized_align = layout.align.abi.bytes(); + debug!("DST {} statically sized prefix size: {} align: {}", t, sized_size, sized_align); + let sized_size = bx.const_usize(sized_size); + let sized_align = bx.const_usize(sized_align); + + // Recurse to get the size of the dynamically sized field (must be + // the last field). + let field_ty = layout.field(bx, i).ty; + let (unsized_size, mut unsized_align) = size_and_align_of_dst(bx, field_ty, info); + + // FIXME (#26403, #27023): We should be adding padding + // to `sized_size` (to accommodate the `unsized_align` + // required of the unsized field that follows) before + // summing it with `sized_size`. (Note that since #26403 + // is unfixed, we do not yet add the necessary padding + // here. But this is where the add would go.) + + // Return the sum of sizes and max of aligns. + let size = bx.add(sized_size, unsized_size); + + // Packed types ignore the alignment of their fields. + if let ty::Adt(def, _) = t.kind() { + if def.repr().packed() { + unsized_align = sized_align; + } + } + + // Choose max of two known alignments (combined value must + // be aligned according to more restrictive of the two). + let align = match ( + bx.const_to_opt_u128(sized_align, false), + bx.const_to_opt_u128(unsized_align, false), + ) { + (Some(sized_align), Some(unsized_align)) => { + // If both alignments are constant, (the sized_align should always be), then + // pick the correct alignment statically. + bx.const_usize(std::cmp::max(sized_align, unsized_align) as u64) + } + _ => { + let cmp = bx.icmp(IntPredicate::IntUGT, sized_align, unsized_align); + bx.select(cmp, sized_align, unsized_align) + } + }; + + // Issue #27023: must add any necessary padding to `size` + // (to make it a multiple of `align`) before returning it. + // + // Namely, the returned size should be, in C notation: + // + // `size + ((size & (align-1)) ? align : 0)` + // + // emulated via the semi-standard fast bit trick: + // + // `(size + (align-1)) & -align` + let one = bx.const_usize(1); + let addend = bx.sub(align, one); + let add = bx.add(size, addend); + let neg = bx.neg(align); + let size = bx.and(add, neg); + + (size, align) + } + } +} diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs new file mode 100644 index 00000000000..1802eedf193 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -0,0 +1,244 @@ +#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] +#![feature(box_patterns)] +#![feature(try_blocks)] +#![feature(let_else)] +#![feature(once_cell)] +#![feature(associated_type_bounds)] +#![feature(strict_provenance)] +#![feature(int_roundings)] +#![feature(if_let_guard)] +#![recursion_limit = "256"] +#![allow(rustc::potential_query_instability)] + +//! This crate contains codegen code that is used by all codegen backends (LLVM and others). +//! The backend-agnostic functions of this crate use functions defined in various traits that +//! have to be implemented by each backends. + +#[macro_use] +extern crate rustc_macros; +#[macro_use] +extern crate tracing; +#[macro_use] +extern crate rustc_middle; + +use rustc_ast as ast; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::sync::Lrc; +use rustc_hir::def_id::CrateNum; +use rustc_hir::LangItem; +use rustc_middle::dep_graph::WorkProduct; +use rustc_middle::middle::dependency_format::Dependencies; +use rustc_middle::middle::exported_symbols::SymbolExportKind; +use rustc_middle::ty::query::{ExternProviders, Providers}; +use rustc_serialize::opaque::{MemDecoder, MemEncoder}; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; +use rustc_session::config::{CrateType, OutputFilenames, OutputType, RUST_CGU_EXT}; +use rustc_session::cstore::{self, CrateSource}; +use rustc_session::utils::NativeLibKind; +use rustc_span::symbol::Symbol; +use rustc_span::DebuggerVisualizerFile; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +pub mod back; +pub mod base; +pub mod common; +pub mod coverageinfo; +pub mod debuginfo; +pub mod glue; +pub mod meth; +pub mod mir; +pub mod mono_item; +pub mod target_features; +pub mod traits; + +pub struct ModuleCodegen { + /// The name of the module. When the crate may be saved between + /// compilations, incremental compilation requires that name be + /// unique amongst **all** crates. Therefore, it should contain + /// something unique to this crate (e.g., a module path) as well + /// as the crate name and disambiguator. + /// We currently generate these names via CodegenUnit::build_cgu_name(). + pub name: String, + pub module_llvm: M, + pub kind: ModuleKind, +} + +impl ModuleCodegen { + pub fn into_compiled_module( + self, + emit_obj: bool, + emit_dwarf_obj: bool, + emit_bc: bool, + outputs: &OutputFilenames, + ) -> CompiledModule { + let object = emit_obj.then(|| outputs.temp_path(OutputType::Object, Some(&self.name))); + let dwarf_object = emit_dwarf_obj.then(|| outputs.temp_path_dwo(Some(&self.name))); + let bytecode = emit_bc.then(|| outputs.temp_path(OutputType::Bitcode, Some(&self.name))); + + CompiledModule { name: self.name.clone(), kind: self.kind, object, dwarf_object, bytecode } + } +} + +#[derive(Debug, Encodable, Decodable)] +pub struct CompiledModule { + pub name: String, + pub kind: ModuleKind, + pub object: Option, + pub dwarf_object: Option, + pub bytecode: Option, +} + +pub struct CachedModuleCodegen { + pub name: String, + pub source: WorkProduct, +} + +#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable)] +pub enum ModuleKind { + Regular, + Metadata, + Allocator, +} + +bitflags::bitflags! { + pub struct MemFlags: u8 { + const VOLATILE = 1 << 0; + const NONTEMPORAL = 1 << 1; + const UNALIGNED = 1 << 2; + } +} + +#[derive(Clone, Debug, Encodable, Decodable, HashStable)] +pub struct NativeLib { + pub kind: NativeLibKind, + pub name: Option, + pub cfg: Option, + pub verbatim: Option, + pub dll_imports: Vec, +} + +impl From<&cstore::NativeLib> for NativeLib { + fn from(lib: &cstore::NativeLib) -> Self { + NativeLib { + kind: lib.kind, + name: lib.name, + cfg: lib.cfg.clone(), + verbatim: lib.verbatim, + dll_imports: lib.dll_imports.clone(), + } + } +} + +/// Misc info we load from metadata to persist beyond the tcx. +/// +/// Note: though `CrateNum` is only meaningful within the same tcx, information within `CrateInfo` +/// is self-contained. `CrateNum` can be viewed as a unique identifier within a `CrateInfo`, where +/// `used_crate_source` contains all `CrateSource` of the dependents, and maintains a mapping from +/// identifiers (`CrateNum`) to `CrateSource`. The other fields map `CrateNum` to the crate's own +/// additional properties, so that effectively we can retrieve each dependent crate's `CrateSource` +/// and the corresponding properties without referencing information outside of a `CrateInfo`. +#[derive(Debug, Encodable, Decodable)] +pub struct CrateInfo { + pub target_cpu: String, + pub exported_symbols: FxHashMap>, + pub linked_symbols: FxHashMap>, + pub local_crate_name: Symbol, + pub compiler_builtins: Option, + pub profiler_runtime: Option, + pub is_no_builtins: FxHashSet, + pub native_libraries: FxHashMap>, + pub crate_name: FxHashMap, + pub used_libraries: Vec, + pub used_crate_source: FxHashMap>, + pub used_crates: Vec, + pub lang_item_to_crate: FxHashMap, + pub missing_lang_items: FxHashMap>, + pub dependency_formats: Lrc, + pub windows_subsystem: Option, + pub natvis_debugger_visualizers: BTreeSet, +} + +#[derive(Encodable, Decodable)] +pub struct CodegenResults { + pub modules: Vec, + pub allocator_module: Option, + pub metadata_module: Option, + pub metadata: rustc_metadata::EncodedMetadata, + pub crate_info: CrateInfo, +} + +pub fn provide(providers: &mut Providers) { + crate::back::symbol_export::provide(providers); + crate::base::provide(providers); + crate::target_features::provide(providers); +} + +pub fn provide_extern(providers: &mut ExternProviders) { + crate::back::symbol_export::provide_extern(providers); +} + +/// Checks if the given filename ends with the `.rcgu.o` extension that `rustc` +/// uses for the object files it generates. +pub fn looks_like_rust_object_file(filename: &str) -> bool { + let path = Path::new(filename); + let ext = path.extension().and_then(|s| s.to_str()); + if ext != Some(OutputType::Object.extension()) { + // The file name does not end with ".o", so it can't be an object file. + return false; + } + + // Strip the ".o" at the end + let ext2 = path.file_stem().and_then(|s| Path::new(s).extension()).and_then(|s| s.to_str()); + + // Check if the "inner" extension + ext2 == Some(RUST_CGU_EXT) +} + +const RLINK_VERSION: u32 = 1; +const RLINK_MAGIC: &[u8] = b"rustlink"; + +const RUSTC_VERSION: Option<&str> = option_env!("CFG_VERSION"); + +impl CodegenResults { + pub fn serialize_rlink(codegen_results: &CodegenResults) -> Vec { + let mut encoder = MemEncoder::new(); + encoder.emit_raw_bytes(RLINK_MAGIC); + // `emit_raw_bytes` is used to make sure that the version representation does not depend on + // Encoder's inner representation of `u32`. + encoder.emit_raw_bytes(&RLINK_VERSION.to_be_bytes()); + encoder.emit_str(RUSTC_VERSION.unwrap()); + Encodable::encode(codegen_results, &mut encoder); + encoder.finish() + } + + pub fn deserialize_rlink(data: Vec) -> Result { + // The Decodable machinery is not used here because it panics if the input data is invalid + // and because its internal representation may change. + if !data.starts_with(RLINK_MAGIC) { + return Err("The input does not look like a .rlink file".to_string()); + } + let data = &data[RLINK_MAGIC.len()..]; + if data.len() < 4 { + return Err("The input does not contain version number".to_string()); + } + + let mut version_array: [u8; 4] = Default::default(); + version_array.copy_from_slice(&data[..4]); + if u32::from_be_bytes(version_array) != RLINK_VERSION { + return Err(".rlink file was produced with encoding version {version_array}, but the current version is {RLINK_VERSION}".to_string()); + } + + let mut decoder = MemDecoder::new(&data[4..], 0); + let rustc_version = decoder.read_str(); + let current_version = RUSTC_VERSION.unwrap(); + if rustc_version != current_version { + return Err(format!( + ".rlink file was produced by rustc version {rustc_version}, but the current version is {current_version}." + )); + } + + let codegen_results = CodegenResults::decode(&mut decoder); + Ok(codegen_results) + } +} diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs new file mode 100644 index 00000000000..27d791d90a5 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -0,0 +1,116 @@ +use crate::traits::*; + +use rustc_middle::ty::{self, subst::GenericArgKind, ExistentialPredicate, Ty, TyCtxt}; +use rustc_session::config::Lto; +use rustc_symbol_mangling::typeid_for_trait_ref; +use rustc_target::abi::call::FnAbi; + +#[derive(Copy, Clone, Debug)] +pub struct VirtualIndex(u64); + +impl<'a, 'tcx> VirtualIndex { + pub fn from_index(index: usize) -> Self { + VirtualIndex(index as u64) + } + + pub fn get_fn>( + self, + bx: &mut Bx, + llvtable: Bx::Value, + ty: Ty<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + ) -> Bx::Value { + // Load the data pointer from the object. + debug!("get_fn({llvtable:?}, {ty:?}, {self:?})"); + let llty = bx.fn_ptr_backend_type(fn_abi); + let llvtable = bx.pointercast(llvtable, bx.type_ptr_to(llty)); + + if bx.cx().sess().opts.unstable_opts.virtual_function_elimination + && bx.cx().sess().lto() == Lto::Fat + { + let typeid = + bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), get_trait_ref(bx.tcx(), ty))); + let vtable_byte_offset = self.0 * bx.data_layout().pointer_size.bytes(); + let type_checked_load = bx.type_checked_load(llvtable, vtable_byte_offset, typeid); + let func = bx.extract_value(type_checked_load, 0); + bx.pointercast(func, llty) + } else { + let ptr_align = bx.tcx().data_layout.pointer_align.abi; + let gep = bx.inbounds_gep(llty, llvtable, &[bx.const_usize(self.0)]); + let ptr = bx.load(llty, gep, ptr_align); + bx.nonnull_metadata(ptr); + // VTable loads are invariant. + bx.set_invariant_load(ptr); + ptr + } + } + + pub fn get_usize>( + self, + bx: &mut Bx, + llvtable: Bx::Value, + ) -> Bx::Value { + // Load the data pointer from the object. + debug!("get_int({:?}, {:?})", llvtable, self); + + let llty = bx.type_isize(); + let llvtable = bx.pointercast(llvtable, bx.type_ptr_to(llty)); + let usize_align = bx.tcx().data_layout.pointer_align.abi; + let gep = bx.inbounds_gep(llty, llvtable, &[bx.const_usize(self.0)]); + let ptr = bx.load(llty, gep, usize_align); + // VTable loads are invariant. + bx.set_invariant_load(ptr); + ptr + } +} + +fn get_trait_ref<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> { + for arg in ty.peel_refs().walk() { + if let GenericArgKind::Type(ty) = arg.unpack() { + if let ty::Dynamic(trait_refs, _) = ty.kind() { + return trait_refs[0].map_bound(|trait_ref| match trait_ref { + ExistentialPredicate::Trait(tr) => tr, + ExistentialPredicate::Projection(proj) => proj.trait_ref(tcx), + ExistentialPredicate::AutoTrait(_) => { + bug!("auto traits don't have functions") + } + }); + } + } + } + + bug!("expected a `dyn Trait` ty, found {ty:?}") +} + +/// Creates a dynamic vtable for the given type and vtable origin. +/// This is used only for objects. +/// +/// The vtables are cached instead of created on every call. +/// +/// The `trait_ref` encodes the erased self type. Hence if we are +/// making an object `Foo` from a value of type `Foo`, then +/// `trait_ref` would map `T: Trait`. +pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( + cx: &Cx, + ty: Ty<'tcx>, + trait_ref: Option>, +) -> Cx::Value { + let tcx = cx.tcx(); + + debug!("get_vtable(ty={:?}, trait_ref={:?})", ty, trait_ref); + + // Check the cache. + if let Some(&val) = cx.vtables().borrow().get(&(ty, trait_ref)) { + return val; + } + + let vtable_alloc_id = tcx.vtable_allocation((ty, trait_ref)); + let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory(); + let vtable_const = cx.const_data_from_alloc(vtable_allocation); + let align = cx.data_layout().pointer_align.abi; + let vtable = cx.static_addr_of(vtable_const, align, Some("vtable")); + + cx.create_vtable_debuginfo(ty, trait_ref, vtable); + cx.vtables().borrow_mut().insert((ty, trait_ref), vtable); + vtable +} diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs new file mode 100644 index 00000000000..24da48ead63 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -0,0 +1,368 @@ +//! An analysis to determine which locals require allocas and +//! which do not. + +use super::FunctionCx; +use crate::traits::*; +use rustc_data_structures::graph::dominators::Dominators; +use rustc_index::bit_set::BitSet; +use rustc_index::vec::IndexVec; +use rustc_middle::mir::traversal; +use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; +use rustc_middle::mir::{self, Location, TerminatorKind}; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; + +pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + fx: &FunctionCx<'a, 'tcx, Bx>, +) -> BitSet { + let mir = fx.mir; + let dominators = mir.basic_blocks.dominators(); + let locals = mir + .local_decls + .iter() + .map(|decl| { + let ty = fx.monomorphize(decl.ty); + let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span); + if layout.is_zst() { + LocalKind::ZST + } else if fx.cx.is_backend_immediate(layout) || fx.cx.is_backend_scalar_pair(layout) { + LocalKind::Unused + } else { + LocalKind::Memory + } + }) + .collect(); + + let mut analyzer = LocalAnalyzer { fx, dominators, locals }; + + // Arguments get assigned to by means of the function being called + for arg in mir.args_iter() { + analyzer.assign(arg, mir::START_BLOCK.start_location()); + } + + // If there exists a local definition that dominates all uses of that local, + // the definition should be visited first. Traverse blocks in an order that + // is a topological sort of dominance partial order. + for (bb, data) in traversal::reverse_postorder(&mir) { + analyzer.visit_basic_block_data(bb, data); + } + + let mut non_ssa_locals = BitSet::new_empty(analyzer.locals.len()); + for (local, kind) in analyzer.locals.iter_enumerated() { + if matches!(kind, LocalKind::Memory) { + non_ssa_locals.insert(local); + } + } + + non_ssa_locals +} + +#[derive(Copy, Clone, PartialEq, Eq)] +enum LocalKind { + ZST, + /// A local that requires an alloca. + Memory, + /// A scalar or a scalar pair local that is neither defined nor used. + Unused, + /// A scalar or a scalar pair local with a single definition that dominates all uses. + SSA(mir::Location), +} + +struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { + fx: &'mir FunctionCx<'a, 'tcx, Bx>, + dominators: Dominators, + locals: IndexVec, +} + +impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, Bx> { + fn assign(&mut self, local: mir::Local, location: Location) { + let kind = &mut self.locals[local]; + match *kind { + LocalKind::ZST => {} + LocalKind::Memory => {} + LocalKind::Unused => { + *kind = LocalKind::SSA(location); + } + LocalKind::SSA(_) => { + *kind = LocalKind::Memory; + } + } + } + + fn process_place( + &mut self, + place_ref: &mir::PlaceRef<'tcx>, + context: PlaceContext, + location: Location, + ) { + let cx = self.fx.cx; + + if let Some((place_base, elem)) = place_ref.last_projection() { + let mut base_context = if context.is_mutating_use() { + PlaceContext::MutatingUse(MutatingUseContext::Projection) + } else { + PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection) + }; + + // Allow uses of projections that are ZSTs or from scalar fields. + let is_consume = matches!( + context, + PlaceContext::NonMutatingUse( + NonMutatingUseContext::Copy | NonMutatingUseContext::Move, + ) + ); + if is_consume { + let base_ty = place_base.ty(self.fx.mir, cx.tcx()); + let base_ty = self.fx.monomorphize(base_ty); + + // ZSTs don't require any actual memory access. + let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(elem)).ty; + let span = self.fx.mir.local_decls[place_ref.local].source_info.span; + if cx.spanned_layout_of(elem_ty, span).is_zst() { + return; + } + + if let mir::ProjectionElem::Field(..) = elem { + let layout = cx.spanned_layout_of(base_ty.ty, span); + if cx.is_backend_immediate(layout) || cx.is_backend_scalar_pair(layout) { + // Recurse with the same context, instead of `Projection`, + // potentially stopping at non-operand projections, + // which would trigger `not_ssa` on locals. + base_context = context; + } + } + } + + if let mir::ProjectionElem::Deref = elem { + // Deref projections typically only read the pointer. + base_context = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy); + } + + self.process_place(&place_base, base_context, location); + // HACK(eddyb) this emulates the old `visit_projection_elem`, this + // entire `visit_place`-like `process_place` method should be rewritten, + // now that we have moved to the "slice of projections" representation. + if let mir::ProjectionElem::Index(local) = elem { + self.visit_local( + local, + PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy), + location, + ); + } + } else { + self.visit_local(place_ref.local, context, location); + } + } +} + +impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx> + for LocalAnalyzer<'mir, 'a, 'tcx, Bx> +{ + fn visit_assign( + &mut self, + place: &mir::Place<'tcx>, + rvalue: &mir::Rvalue<'tcx>, + location: Location, + ) { + debug!("visit_assign(place={:?}, rvalue={:?})", place, rvalue); + + if let Some(local) = place.as_local() { + self.assign(local, location); + if self.locals[local] != LocalKind::Memory { + let decl_span = self.fx.mir.local_decls[local].source_info.span; + if !self.fx.rvalue_creates_operand(rvalue, decl_span) { + self.locals[local] = LocalKind::Memory; + } + } + } else { + self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), location); + } + + self.visit_rvalue(rvalue, location); + } + + fn visit_place(&mut self, place: &mir::Place<'tcx>, context: PlaceContext, location: Location) { + debug!("visit_place(place={:?}, context={:?})", place, context); + self.process_place(&place.as_ref(), context, location); + } + + fn visit_local(&mut self, local: mir::Local, context: PlaceContext, location: Location) { + match context { + PlaceContext::MutatingUse(MutatingUseContext::Call) + | PlaceContext::MutatingUse(MutatingUseContext::Yield) => { + self.assign(local, location); + } + + PlaceContext::NonUse(_) | PlaceContext::MutatingUse(MutatingUseContext::Retag) => {} + + PlaceContext::NonMutatingUse( + NonMutatingUseContext::Copy | NonMutatingUseContext::Move, + ) => match &mut self.locals[local] { + LocalKind::ZST => {} + LocalKind::Memory => {} + LocalKind::SSA(def) if def.dominates(location, &self.dominators) => {} + // Reads from uninitialized variables (e.g., in dead code, after + // optimizations) require locals to be in (uninitialized) memory. + // N.B., there can be uninitialized reads of a local visited after + // an assignment to that local, if they happen on disjoint paths. + kind @ (LocalKind::Unused | LocalKind::SSA(_)) => { + *kind = LocalKind::Memory; + } + }, + + PlaceContext::MutatingUse( + MutatingUseContext::Store + | MutatingUseContext::Deinit + | MutatingUseContext::SetDiscriminant + | MutatingUseContext::AsmOutput + | MutatingUseContext::Borrow + | MutatingUseContext::AddressOf + | MutatingUseContext::Projection, + ) + | PlaceContext::NonMutatingUse( + NonMutatingUseContext::Inspect + | NonMutatingUseContext::SharedBorrow + | NonMutatingUseContext::UniqueBorrow + | NonMutatingUseContext::ShallowBorrow + | NonMutatingUseContext::AddressOf + | NonMutatingUseContext::Projection, + ) => { + self.locals[local] = LocalKind::Memory; + } + + PlaceContext::MutatingUse(MutatingUseContext::Drop) => { + let kind = &mut self.locals[local]; + if *kind != LocalKind::Memory { + let ty = self.fx.mir.local_decls[local].ty; + let ty = self.fx.monomorphize(ty); + if self.fx.cx.type_needs_drop(ty) { + // Only need the place if we're actually dropping it. + *kind = LocalKind::Memory; + } + } + } + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum CleanupKind { + NotCleanup, + Funclet, + Internal { funclet: mir::BasicBlock }, +} + +impl CleanupKind { + pub fn funclet_bb(self, for_bb: mir::BasicBlock) -> Option { + match self { + CleanupKind::NotCleanup => None, + CleanupKind::Funclet => Some(for_bb), + CleanupKind::Internal { funclet } => Some(funclet), + } + } +} + +pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec { + fn discover_masters<'tcx>( + result: &mut IndexVec, + mir: &mir::Body<'tcx>, + ) { + for (bb, data) in mir.basic_blocks().iter_enumerated() { + match data.terminator().kind { + TerminatorKind::Goto { .. } + | TerminatorKind::Resume + | TerminatorKind::Abort + | TerminatorKind::Return + | TerminatorKind::GeneratorDrop + | TerminatorKind::Unreachable + | TerminatorKind::SwitchInt { .. } + | TerminatorKind::Yield { .. } + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ } + TerminatorKind::Call { cleanup: unwind, .. } + | TerminatorKind::InlineAsm { cleanup: unwind, .. } + | TerminatorKind::Assert { cleanup: unwind, .. } + | TerminatorKind::DropAndReplace { unwind, .. } + | TerminatorKind::Drop { unwind, .. } => { + if let Some(unwind) = unwind { + debug!( + "cleanup_kinds: {:?}/{:?} registering {:?} as funclet", + bb, data, unwind + ); + result[unwind] = CleanupKind::Funclet; + } + } + } + } + } + + fn propagate<'tcx>(result: &mut IndexVec, mir: &mir::Body<'tcx>) { + let mut funclet_succs = IndexVec::from_elem(None, mir.basic_blocks()); + + let mut set_successor = |funclet: mir::BasicBlock, succ| match funclet_succs[funclet] { + ref mut s @ None => { + debug!("set_successor: updating successor of {:?} to {:?}", funclet, succ); + *s = Some(succ); + } + Some(s) => { + if s != succ { + span_bug!( + mir.span, + "funclet {:?} has 2 parents - {:?} and {:?}", + funclet, + s, + succ + ); + } + } + }; + + for (bb, data) in traversal::reverse_postorder(mir) { + let funclet = match result[bb] { + CleanupKind::NotCleanup => continue, + CleanupKind::Funclet => bb, + CleanupKind::Internal { funclet } => funclet, + }; + + debug!( + "cleanup_kinds: {:?}/{:?}/{:?} propagating funclet {:?}", + bb, data, result[bb], funclet + ); + + for succ in data.terminator().successors() { + let kind = result[succ]; + debug!("cleanup_kinds: propagating {:?} to {:?}/{:?}", funclet, succ, kind); + match kind { + CleanupKind::NotCleanup => { + result[succ] = CleanupKind::Internal { funclet }; + } + CleanupKind::Funclet => { + if funclet != succ { + set_successor(funclet, succ); + } + } + CleanupKind::Internal { funclet: succ_funclet } => { + if funclet != succ_funclet { + // `succ` has 2 different funclet going into it, so it must + // be a funclet by itself. + + debug!( + "promoting {:?} to a funclet and updating {:?}", + succ, succ_funclet + ); + result[succ] = CleanupKind::Funclet; + set_successor(succ_funclet, succ); + set_successor(funclet, succ); + } + } + } + } + } + } + + let mut result = IndexVec::from_elem(CleanupKind::NotCleanup, mir.basic_blocks()); + + discover_masters(&mut result, mir); + propagate(&mut result, mir); + debug!("cleanup_kinds: result={:?}", result); + result +} diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs new file mode 100644 index 00000000000..3eee58d9d1c --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -0,0 +1,1654 @@ +use super::operand::OperandRef; +use super::operand::OperandValue::{Immediate, Pair, Ref}; +use super::place::PlaceRef; +use super::{FunctionCx, LocalRef}; + +use crate::base; +use crate::common::{self, IntPredicate}; +use crate::meth; +use crate::traits::*; +use crate::MemFlags; + +use rustc_ast as ast; +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_hir::lang_items::LangItem; +use rustc_index::vec::Idx; +use rustc_middle::mir::AssertKind; +use rustc_middle::mir::{self, SwitchTargets}; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; +use rustc_middle::ty::{self, Instance, Ty, TypeVisitable}; +use rustc_span::source_map::Span; +use rustc_span::{sym, Symbol}; +use rustc_symbol_mangling::typeid::typeid_for_fnabi; +use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode}; +use rustc_target::abi::{self, HasDataLayout, WrappingRange}; +use rustc_target::spec::abi::Abi; + +/// Used by `FunctionCx::codegen_terminator` for emitting common patterns +/// e.g., creating a basic block, calling a function, etc. +struct TerminatorCodegenHelper<'tcx> { + bb: mir::BasicBlock, + terminator: &'tcx mir::Terminator<'tcx>, + funclet_bb: Option, +} + +impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { + /// Returns the appropriate `Funclet` for the current funclet, if on MSVC, + /// either already previously cached, or newly created, by `landing_pad_for`. + fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>( + &self, + fx: &'b mut FunctionCx<'a, 'tcx, Bx>, + ) -> Option<&'b Bx::Funclet> { + let funclet_bb = self.funclet_bb?; + if base::wants_msvc_seh(fx.cx.tcx().sess) { + // If `landing_pad_for` hasn't been called yet to create the `Funclet`, + // it has to be now. This may not seem necessary, as RPO should lead + // to all the unwind edges being visited (and so to `landing_pad_for` + // getting called for them), before building any of the blocks inside + // the funclet itself - however, if MIR contains edges that end up not + // being needed in the LLVM IR after monomorphization, the funclet may + // be unreachable, and we don't have yet a way to skip building it in + // such an eventuality (which may be a better solution than this). + if fx.funclets[funclet_bb].is_none() { + fx.landing_pad_for(funclet_bb); + } + + Some( + fx.funclets[funclet_bb] + .as_ref() + .expect("landing_pad_for didn't also create funclets entry"), + ) + } else { + None + } + } + + fn lltarget>( + &self, + fx: &mut FunctionCx<'a, 'tcx, Bx>, + target: mir::BasicBlock, + ) -> (Bx::BasicBlock, bool) { + let span = self.terminator.source_info.span; + let lltarget = fx.llbb(target); + let target_funclet = fx.cleanup_kinds[target].funclet_bb(target); + match (self.funclet_bb, target_funclet) { + (None, None) => (lltarget, false), + (Some(f), Some(t_f)) if f == t_f || !base::wants_msvc_seh(fx.cx.tcx().sess) => { + (lltarget, false) + } + // jump *into* cleanup - need a landing pad if GNU, cleanup pad if MSVC + (None, Some(_)) => (fx.landing_pad_for(target), false), + (Some(_), None) => span_bug!(span, "{:?} - jump out of cleanup?", self.terminator), + (Some(_), Some(_)) => (fx.landing_pad_for(target), true), + } + } + + /// Create a basic block. + fn llblock>( + &self, + fx: &mut FunctionCx<'a, 'tcx, Bx>, + target: mir::BasicBlock, + ) -> Bx::BasicBlock { + let (lltarget, is_cleanupret) = self.lltarget(fx, target); + if is_cleanupret { + // MSVC cross-funclet jump - need a trampoline + + debug!("llblock: creating cleanup trampoline for {:?}", target); + let name = &format!("{:?}_cleanup_trampoline_{:?}", self.bb, target); + let trampoline = Bx::append_block(fx.cx, fx.llfn, name); + let mut trampoline_bx = Bx::build(fx.cx, trampoline); + trampoline_bx.cleanup_ret(self.funclet(fx).unwrap(), Some(lltarget)); + trampoline + } else { + lltarget + } + } + + fn funclet_br>( + &self, + fx: &mut FunctionCx<'a, 'tcx, Bx>, + bx: &mut Bx, + target: mir::BasicBlock, + ) { + let (lltarget, is_cleanupret) = self.lltarget(fx, target); + if is_cleanupret { + // micro-optimization: generate a `ret` rather than a jump + // to a trampoline. + bx.cleanup_ret(self.funclet(fx).unwrap(), Some(lltarget)); + } else { + bx.br(lltarget); + } + } + + /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional + /// return destination `destination` and the cleanup function `cleanup`. + fn do_call>( + &self, + fx: &mut FunctionCx<'a, 'tcx, Bx>, + bx: &mut Bx, + fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, + fn_ptr: Bx::Value, + llargs: &[Bx::Value], + destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>, + cleanup: Option, + copied_constant_arguments: &[PlaceRef<'tcx, ::Value>], + ) { + // If there is a cleanup block and the function we're calling can unwind, then + // do an invoke, otherwise do a call. + let fn_ty = bx.fn_decl_backend_type(&fn_abi); + + let unwind_block = if let Some(cleanup) = cleanup.filter(|_| fn_abi.can_unwind) { + Some(self.llblock(fx, cleanup)) + } else if fx.mir[self.bb].is_cleanup + && fn_abi.can_unwind + && !base::wants_msvc_seh(fx.cx.tcx().sess) + { + // Exception must not propagate out of the execution of a cleanup (doing so + // can cause undefined behaviour). We insert a double unwind guard for + // functions that can potentially unwind to protect against this. + // + // This is not necessary for SEH which does not use successive unwinding + // like Itanium EH. EH frames in SEH are different from normal function + // frames and SEH will abort automatically if an exception tries to + // propagate out from cleanup. + Some(fx.double_unwind_guard()) + } else { + None + }; + + if let Some(unwind_block) = unwind_block { + let ret_llbb = if let Some((_, target)) = destination { + fx.llbb(target) + } else { + fx.unreachable_block() + }; + let invokeret = + bx.invoke(fn_ty, fn_ptr, &llargs, ret_llbb, unwind_block, self.funclet(fx)); + bx.apply_attrs_callsite(&fn_abi, invokeret); + if fx.mir[self.bb].is_cleanup { + bx.do_not_inline(invokeret); + } + + if let Some((ret_dest, target)) = destination { + bx.switch_to_block(fx.llbb(target)); + fx.set_debug_loc(bx, self.terminator.source_info); + for tmp in copied_constant_arguments { + bx.lifetime_end(tmp.llval, tmp.layout.size); + } + fx.store_return(bx, ret_dest, &fn_abi.ret, invokeret); + } + } else { + let llret = bx.call(fn_ty, fn_ptr, &llargs, self.funclet(fx)); + bx.apply_attrs_callsite(&fn_abi, llret); + if fx.mir[self.bb].is_cleanup { + // Cleanup is always the cold path. Don't inline + // drop glue. Also, when there is a deeply-nested + // struct, there are "symmetry" issues that cause + // exponential inlining - see issue #41696. + bx.do_not_inline(llret); + } + + if let Some((ret_dest, target)) = destination { + for tmp in copied_constant_arguments { + bx.lifetime_end(tmp.llval, tmp.layout.size); + } + fx.store_return(bx, ret_dest, &fn_abi.ret, llret); + self.funclet_br(fx, bx, target); + } else { + bx.unreachable(); + } + } + } + + /// Generates inline assembly with optional `destination` and `cleanup`. + fn do_inlineasm>( + &self, + fx: &mut FunctionCx<'a, 'tcx, Bx>, + bx: &mut Bx, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperandRef<'tcx, Bx>], + options: InlineAsmOptions, + line_spans: &[Span], + destination: Option, + cleanup: Option, + instance: Instance<'_>, + ) { + if let Some(cleanup) = cleanup { + let ret_llbb = if let Some(target) = destination { + fx.llbb(target) + } else { + fx.unreachable_block() + }; + + bx.codegen_inline_asm( + template, + &operands, + options, + line_spans, + instance, + Some((ret_llbb, self.llblock(fx, cleanup), self.funclet(fx))), + ); + } else { + bx.codegen_inline_asm(template, &operands, options, line_spans, instance, None); + + if let Some(target) = destination { + self.funclet_br(fx, bx, target); + } else { + bx.unreachable(); + } + } + } +} + +/// Codegen implementations for some terminator variants. +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + /// Generates code for a `Resume` terminator. + fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, mut bx: Bx) { + if let Some(funclet) = helper.funclet(self) { + bx.cleanup_ret(funclet, None); + } else { + let slot = self.get_personality_slot(&mut bx); + let lp0 = slot.project_field(&mut bx, 0); + let lp0 = bx.load_operand(lp0).immediate(); + let lp1 = slot.project_field(&mut bx, 1); + let lp1 = bx.load_operand(lp1).immediate(); + slot.storage_dead(&mut bx); + + let mut lp = bx.const_undef(self.landing_pad_type()); + lp = bx.insert_value(lp, lp0, 0); + lp = bx.insert_value(lp, lp1, 1); + bx.resume(lp); + } + } + + fn codegen_switchint_terminator( + &mut self, + helper: TerminatorCodegenHelper<'tcx>, + mut bx: Bx, + discr: &mir::Operand<'tcx>, + switch_ty: Ty<'tcx>, + targets: &SwitchTargets, + ) { + let discr = self.codegen_operand(&mut bx, &discr); + // `switch_ty` is redundant, sanity-check that. + assert_eq!(discr.layout.ty, switch_ty); + let mut target_iter = targets.iter(); + if target_iter.len() == 1 { + // If there are two targets (one conditional, one fallback), emit br instead of switch + let (test_value, target) = target_iter.next().unwrap(); + let lltrue = helper.llblock(self, target); + let llfalse = helper.llblock(self, targets.otherwise()); + if switch_ty == bx.tcx().types.bool { + // Don't generate trivial icmps when switching on bool + match test_value { + 0 => bx.cond_br(discr.immediate(), llfalse, lltrue), + 1 => bx.cond_br(discr.immediate(), lltrue, llfalse), + _ => bug!(), + } + } else { + let switch_llty = bx.immediate_backend_type(bx.layout_of(switch_ty)); + let llval = bx.const_uint_big(switch_llty, test_value); + let cmp = bx.icmp(IntPredicate::IntEQ, discr.immediate(), llval); + bx.cond_br(cmp, lltrue, llfalse); + } + } else { + bx.switch( + discr.immediate(), + helper.llblock(self, targets.otherwise()), + target_iter.map(|(value, target)| (value, helper.llblock(self, target))), + ); + } + } + + fn codegen_return_terminator(&mut self, mut bx: Bx) { + // Call `va_end` if this is the definition of a C-variadic function. + if self.fn_abi.c_variadic { + // The `VaList` "spoofed" argument is just after all the real arguments. + let va_list_arg_idx = self.fn_abi.args.len(); + match self.locals[mir::Local::new(1 + va_list_arg_idx)] { + LocalRef::Place(va_list) => { + bx.va_end(va_list.llval); + } + _ => bug!("C-variadic function must have a `VaList` place"), + } + } + if self.fn_abi.ret.layout.abi.is_uninhabited() { + // Functions with uninhabited return values are marked `noreturn`, + // so we should make sure that we never actually do. + // We play it safe by using a well-defined `abort`, but we could go for immediate UB + // if that turns out to be helpful. + bx.abort(); + // `abort` does not terminate the block, so we still need to generate + // an `unreachable` terminator after it. + bx.unreachable(); + return; + } + let llval = match self.fn_abi.ret.mode { + PassMode::Ignore | PassMode::Indirect { .. } => { + bx.ret_void(); + return; + } + + PassMode::Direct(_) | PassMode::Pair(..) => { + let op = self.codegen_consume(&mut bx, mir::Place::return_place().as_ref()); + if let Ref(llval, _, align) = op.val { + bx.load(bx.backend_type(op.layout), llval, align) + } else { + op.immediate_or_packed_pair(&mut bx) + } + } + + PassMode::Cast(cast_ty) => { + let op = match self.locals[mir::RETURN_PLACE] { + LocalRef::Operand(Some(op)) => op, + LocalRef::Operand(None) => bug!("use of return before def"), + LocalRef::Place(cg_place) => OperandRef { + val: Ref(cg_place.llval, None, cg_place.align), + layout: cg_place.layout, + }, + LocalRef::UnsizedPlace(_) => bug!("return type must be sized"), + }; + let llslot = match op.val { + Immediate(_) | Pair(..) => { + let scratch = PlaceRef::alloca(&mut bx, self.fn_abi.ret.layout); + op.val.store(&mut bx, scratch); + scratch.llval + } + Ref(llval, _, align) => { + assert_eq!(align, op.layout.align.abi, "return place is unaligned!"); + llval + } + }; + let ty = bx.cast_backend_type(&cast_ty); + let addr = bx.pointercast(llslot, bx.type_ptr_to(ty)); + bx.load(ty, addr, self.fn_abi.ret.layout.align.abi) + } + }; + bx.ret(llval); + } + + fn codegen_drop_terminator( + &mut self, + helper: TerminatorCodegenHelper<'tcx>, + mut bx: Bx, + location: mir::Place<'tcx>, + target: mir::BasicBlock, + unwind: Option, + ) { + let ty = location.ty(self.mir, bx.tcx()).ty; + let ty = self.monomorphize(ty); + let drop_fn = Instance::resolve_drop_in_place(bx.tcx(), ty); + + if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def { + // we don't actually need to drop anything. + helper.funclet_br(self, &mut bx, target); + return; + } + + let place = self.codegen_place(&mut bx, location.as_ref()); + let (args1, args2); + let mut args = if let Some(llextra) = place.llextra { + args2 = [place.llval, llextra]; + &args2[..] + } else { + args1 = [place.llval]; + &args1[..] + }; + let (drop_fn, fn_abi) = match ty.kind() { + // FIXME(eddyb) perhaps move some of this logic into + // `Instance::resolve_drop_in_place`? + ty::Dynamic(..) => { + let virtual_drop = Instance { + def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), + substs: drop_fn.substs, + }; + let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty()); + let vtable = args[1]; + args = &args[..1]; + ( + meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE) + .get_fn(&mut bx, vtable, ty, &fn_abi), + fn_abi, + ) + } + _ => (bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty())), + }; + helper.do_call( + self, + &mut bx, + fn_abi, + drop_fn, + args, + Some((ReturnDest::Nothing, target)), + unwind, + &[], + ); + } + + fn codegen_assert_terminator( + &mut self, + helper: TerminatorCodegenHelper<'tcx>, + mut bx: Bx, + terminator: &mir::Terminator<'tcx>, + cond: &mir::Operand<'tcx>, + expected: bool, + msg: &mir::AssertMessage<'tcx>, + target: mir::BasicBlock, + cleanup: Option, + ) { + let span = terminator.source_info.span; + let cond = self.codegen_operand(&mut bx, cond).immediate(); + let mut const_cond = bx.const_to_opt_u128(cond, false).map(|c| c == 1); + + // This case can currently arise only from functions marked + // with #[rustc_inherit_overflow_checks] and inlined from + // another crate (mostly core::num generic/#[inline] fns), + // while the current crate doesn't use overflow checks. + // NOTE: Unlike binops, negation doesn't have its own + // checked operation, just a comparison with the minimum + // value, so we have to check for the assert message. + if !bx.check_overflow() { + if let AssertKind::OverflowNeg(_) = *msg { + const_cond = Some(expected); + } + } + + // Don't codegen the panic block if success if known. + if const_cond == Some(expected) { + helper.funclet_br(self, &mut bx, target); + return; + } + + // Pass the condition through llvm.expect for branch hinting. + let cond = bx.expect(cond, expected); + + // Create the failure block and the conditional branch to it. + let lltarget = helper.llblock(self, target); + let panic_block = bx.append_sibling_block("panic"); + if expected { + bx.cond_br(cond, lltarget, panic_block); + } else { + bx.cond_br(cond, panic_block, lltarget); + } + + // After this point, bx is the block for the call to panic. + bx.switch_to_block(panic_block); + self.set_debug_loc(&mut bx, terminator.source_info); + + // Get the location information. + let location = self.get_caller_location(&mut bx, terminator.source_info).immediate(); + + // Put together the arguments to the panic entry point. + let (lang_item, args) = match msg { + AssertKind::BoundsCheck { ref len, ref index } => { + let len = self.codegen_operand(&mut bx, len).immediate(); + let index = self.codegen_operand(&mut bx, index).immediate(); + // It's `fn panic_bounds_check(index: usize, len: usize)`, + // and `#[track_caller]` adds an implicit third argument. + (LangItem::PanicBoundsCheck, vec![index, len, location]) + } + _ => { + let msg = bx.const_str(msg.description()); + // It's `pub fn panic(expr: &str)`, with the wide reference being passed + // as two arguments, and `#[track_caller]` adds an implicit third argument. + (LangItem::Panic, vec![msg.0, msg.1, location]) + } + }; + + let (fn_abi, llfn) = common::build_langcall(&bx, Some(span), lang_item); + + // Codegen the actual panic invoke/call. + helper.do_call(self, &mut bx, fn_abi, llfn, &args, None, cleanup, &[]); + } + + fn codegen_abort_terminator( + &mut self, + helper: TerminatorCodegenHelper<'tcx>, + mut bx: Bx, + terminator: &mir::Terminator<'tcx>, + ) { + let span = terminator.source_info.span; + self.set_debug_loc(&mut bx, terminator.source_info); + + // Obtain the panic entry point. + let (fn_abi, llfn) = common::build_langcall(&bx, Some(span), LangItem::PanicNoUnwind); + + // Codegen the actual panic invoke/call. + helper.do_call(self, &mut bx, fn_abi, llfn, &[], None, None, &[]); + } + + /// Returns `true` if this is indeed a panic intrinsic and codegen is done. + fn codegen_panic_intrinsic( + &mut self, + helper: &TerminatorCodegenHelper<'tcx>, + bx: &mut Bx, + intrinsic: Option, + instance: Option>, + source_info: mir::SourceInfo, + target: Option, + cleanup: Option, + ) -> bool { + // Emit a panic or a no-op for `assert_*` intrinsics. + // These are intrinsics that compile to panics so that we can get a message + // which mentions the offending type, even from a const context. + #[derive(Debug, PartialEq)] + enum AssertIntrinsic { + Inhabited, + ZeroValid, + UninitValid, + } + let panic_intrinsic = intrinsic.and_then(|i| match i { + sym::assert_inhabited => Some(AssertIntrinsic::Inhabited), + sym::assert_zero_valid => Some(AssertIntrinsic::ZeroValid), + sym::assert_uninit_valid => Some(AssertIntrinsic::UninitValid), + _ => None, + }); + if let Some(intrinsic) = panic_intrinsic { + use AssertIntrinsic::*; + + let ty = instance.unwrap().substs.type_at(0); + let layout = bx.layout_of(ty); + let do_panic = match intrinsic { + Inhabited => layout.abi.is_uninhabited(), + ZeroValid => !bx.tcx().permits_zero_init(layout), + UninitValid => !bx.tcx().permits_uninit_init(layout), + }; + if do_panic { + let msg_str = with_no_visible_paths!({ + with_no_trimmed_paths!({ + if layout.abi.is_uninhabited() { + // Use this error even for the other intrinsics as it is more precise. + format!("attempted to instantiate uninhabited type `{}`", ty) + } else if intrinsic == ZeroValid { + format!("attempted to zero-initialize type `{}`, which is invalid", ty) + } else { + format!( + "attempted to leave type `{}` uninitialized, which is invalid", + ty + ) + } + }) + }); + let msg = bx.const_str(&msg_str); + let location = self.get_caller_location(bx, source_info).immediate(); + + // Obtain the panic entry point. + let (fn_abi, llfn) = + common::build_langcall(bx, Some(source_info.span), LangItem::Panic); + + // Codegen the actual panic invoke/call. + helper.do_call( + self, + bx, + fn_abi, + llfn, + &[msg.0, msg.1, location], + target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)), + cleanup, + &[], + ); + } else { + // a NOP + let target = target.unwrap(); + helper.funclet_br(self, bx, target) + } + true + } else { + false + } + } + + fn codegen_call_terminator( + &mut self, + helper: TerminatorCodegenHelper<'tcx>, + mut bx: Bx, + terminator: &mir::Terminator<'tcx>, + func: &mir::Operand<'tcx>, + args: &[mir::Operand<'tcx>], + destination: mir::Place<'tcx>, + target: Option, + cleanup: Option, + fn_span: Span, + ) { + let source_info = terminator.source_info; + let span = source_info.span; + + // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar. + let callee = self.codegen_operand(&mut bx, func); + + let (instance, mut llfn) = match *callee.layout.ty.kind() { + ty::FnDef(def_id, substs) => ( + Some( + ty::Instance::resolve(bx.tcx(), ty::ParamEnv::reveal_all(), def_id, substs) + .unwrap() + .unwrap() + .polymorphize(bx.tcx()), + ), + None, + ), + ty::FnPtr(_) => (None, Some(callee.immediate())), + _ => bug!("{} is not callable", callee.layout.ty), + }; + let def = instance.map(|i| i.def); + + if let Some(ty::InstanceDef::DropGlue(_, None)) = def { + // Empty drop glue; a no-op. + let target = target.unwrap(); + helper.funclet_br(self, &mut bx, target); + return; + } + + // FIXME(eddyb) avoid computing this if possible, when `instance` is + // available - right now `sig` is only needed for getting the `abi` + // and figuring out how many extra args were passed to a C-variadic `fn`. + let sig = callee.layout.ty.fn_sig(bx.tcx()); + let abi = sig.abi(); + + // Handle intrinsics old codegen wants Expr's for, ourselves. + let intrinsic = match def { + Some(ty::InstanceDef::Intrinsic(def_id)) => Some(bx.tcx().item_name(def_id)), + _ => None, + }; + + let extra_args = &args[sig.inputs().skip_binder().len()..]; + let extra_args = bx.tcx().mk_type_list(extra_args.iter().map(|op_arg| { + let op_ty = op_arg.ty(self.mir, bx.tcx()); + self.monomorphize(op_ty) + })); + + let fn_abi = match instance { + Some(instance) => bx.fn_abi_of_instance(instance, extra_args), + None => bx.fn_abi_of_fn_ptr(sig, extra_args), + }; + + if intrinsic == Some(sym::transmute) { + if let Some(target) = target { + self.codegen_transmute(&mut bx, &args[0], destination); + helper.funclet_br(self, &mut bx, target); + } else { + // If we are trying to transmute to an uninhabited type, + // it is likely there is no allotted destination. In fact, + // transmuting to an uninhabited type is UB, which means + // we can do what we like. Here, we declare that transmuting + // into an uninhabited type is impossible, so anything following + // it must be unreachable. + assert_eq!(fn_abi.ret.layout.abi, abi::Abi::Uninhabited); + bx.unreachable(); + } + return; + } + + if self.codegen_panic_intrinsic( + &helper, + &mut bx, + intrinsic, + instance, + source_info, + target, + cleanup, + ) { + return; + } + + // The arguments we'll be passing. Plus one to account for outptr, if used. + let arg_count = fn_abi.args.len() + fn_abi.ret.is_indirect() as usize; + let mut llargs = Vec::with_capacity(arg_count); + + // Prepare the return value destination + let ret_dest = if target.is_some() { + let is_intrinsic = intrinsic.is_some(); + self.make_return_dest(&mut bx, destination, &fn_abi.ret, &mut llargs, is_intrinsic) + } else { + ReturnDest::Nothing + }; + + if intrinsic == Some(sym::caller_location) { + if let Some(target) = target { + let location = self + .get_caller_location(&mut bx, mir::SourceInfo { span: fn_span, ..source_info }); + + if let ReturnDest::IndirectOperand(tmp, _) = ret_dest { + location.val.store(&mut bx, tmp); + } + self.store_return(&mut bx, ret_dest, &fn_abi.ret, location.immediate()); + helper.funclet_br(self, &mut bx, target); + } + return; + } + + match intrinsic { + None | Some(sym::drop_in_place) => {} + Some(sym::copy_nonoverlapping) => unreachable!(), + Some(intrinsic) => { + let dest = match ret_dest { + _ if fn_abi.ret.is_indirect() => llargs[0], + ReturnDest::Nothing => { + bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret))) + } + ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval, + ReturnDest::DirectOperand(_) => { + bug!("Cannot use direct operand with an intrinsic call") + } + }; + + let args: Vec<_> = args + .iter() + .enumerate() + .map(|(i, arg)| { + // The indices passed to simd_shuffle* in the + // third argument must be constant. This is + // checked by const-qualification, which also + // promotes any complex rvalues to constants. + if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") { + if let mir::Operand::Constant(constant) = arg { + let c = self.eval_mir_constant(constant); + let (llval, ty) = self.simd_shuffle_indices( + &bx, + constant.span, + self.monomorphize(constant.ty()), + c, + ); + return OperandRef { + val: Immediate(llval), + layout: bx.layout_of(ty), + }; + } else { + span_bug!(span, "shuffle indices must be constant"); + } + } + + self.codegen_operand(&mut bx, arg) + }) + .collect(); + + Self::codegen_intrinsic_call( + &mut bx, + *instance.as_ref().unwrap(), + &fn_abi, + &args, + dest, + span, + ); + + if let ReturnDest::IndirectOperand(dst, _) = ret_dest { + self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval); + } + + if let Some(target) = target { + helper.funclet_br(self, &mut bx, target); + } else { + bx.unreachable(); + } + + return; + } + } + + // Split the rust-call tupled arguments off. + let (first_args, untuple) = if abi == Abi::RustCall && !args.is_empty() { + let (tup, args) = args.split_last().unwrap(); + (args, Some(tup)) + } else { + (args, None) + }; + + let mut copied_constant_arguments = vec![]; + 'make_args: for (i, arg) in first_args.iter().enumerate() { + let mut op = self.codegen_operand(&mut bx, arg); + + if let (0, Some(ty::InstanceDef::Virtual(_, idx))) = (i, def) { + if let Pair(..) = op.val { + // In the case of Rc, we need to explicitly pass a + // *mut RcBox with a Scalar (not ScalarPair) ABI. This is a hack + // that is understood elsewhere in the compiler as a method on + // `dyn Trait`. + // To get a `*mut RcBox`, we just keep unwrapping newtypes until + // we get a value of a built-in pointer type + 'descend_newtypes: while !op.layout.ty.is_unsafe_ptr() + && !op.layout.ty.is_region_ptr() + { + for i in 0..op.layout.fields.count() { + let field = op.extract_field(&mut bx, i); + if !field.layout.is_zst() { + // we found the one non-zero-sized field that is allowed + // now find *its* non-zero-sized field, or stop if it's a + // pointer + op = field; + continue 'descend_newtypes; + } + } + + span_bug!(span, "receiver has no non-zero-sized fields {:?}", op); + } + + // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its + // data pointer and vtable. Look up the method in the vtable, and pass + // the data pointer as the first argument + match op.val { + Pair(data_ptr, meta) => { + llfn = Some(meth::VirtualIndex::from_index(idx).get_fn( + &mut bx, + meta, + op.layout.ty, + &fn_abi, + )); + llargs.push(data_ptr); + continue 'make_args; + } + other => bug!("expected a Pair, got {:?}", other), + } + } else if let Ref(data_ptr, Some(meta), _) = op.val { + // by-value dynamic dispatch + llfn = Some(meth::VirtualIndex::from_index(idx).get_fn( + &mut bx, + meta, + op.layout.ty, + &fn_abi, + )); + llargs.push(data_ptr); + continue; + } else { + span_bug!(span, "can't codegen a virtual call on {:?}", op); + } + } + + // The callee needs to own the argument memory if we pass it + // by-ref, so make a local copy of non-immediate constants. + match (arg, op.val) { + (&mir::Operand::Copy(_), Ref(_, None, _)) + | (&mir::Operand::Constant(_), Ref(_, None, _)) => { + let tmp = PlaceRef::alloca(&mut bx, op.layout); + bx.lifetime_start(tmp.llval, tmp.layout.size); + op.val.store(&mut bx, tmp); + op.val = Ref(tmp.llval, None, tmp.align); + copied_constant_arguments.push(tmp); + } + _ => {} + } + + self.codegen_argument(&mut bx, op, &mut llargs, &fn_abi.args[i]); + } + let num_untupled = untuple.map(|tup| { + self.codegen_arguments_untupled( + &mut bx, + tup, + &mut llargs, + &fn_abi.args[first_args.len()..], + ) + }); + + let needs_location = + instance.map_or(false, |i| i.def.requires_caller_location(self.cx.tcx())); + if needs_location { + let mir_args = if let Some(num_untupled) = num_untupled { + first_args.len() + num_untupled + } else { + args.len() + }; + assert_eq!( + fn_abi.args.len(), + mir_args + 1, + "#[track_caller] fn's must have 1 more argument in their ABI than in their MIR: {:?} {:?} {:?}", + instance, + fn_span, + fn_abi, + ); + let location = + self.get_caller_location(&mut bx, mir::SourceInfo { span: fn_span, ..source_info }); + debug!( + "codegen_call_terminator({:?}): location={:?} (fn_span {:?})", + terminator, location, fn_span + ); + + let last_arg = fn_abi.args.last().unwrap(); + self.codegen_argument(&mut bx, location, &mut llargs, last_arg); + } + + let (is_indirect_call, fn_ptr) = match (llfn, instance) { + (Some(llfn), _) => (true, llfn), + (None, Some(instance)) => (false, bx.get_fn_addr(instance)), + _ => span_bug!(span, "no llfn for call"), + }; + + // For backends that support CFI using type membership (i.e., testing whether a given + // pointer is associated with a type identifier). + if bx.tcx().sess.is_sanitizer_cfi_enabled() && is_indirect_call { + // Emit type metadata and checks. + // FIXME(rcvalle): Add support for generalized identifiers. + // FIXME(rcvalle): Create distinct unnamed MDNodes for internal identifiers. + let typeid = typeid_for_fnabi(bx.tcx(), fn_abi); + let typeid_metadata = self.cx.typeid_metadata(typeid); + + // Test whether the function pointer is associated with the type identifier. + let cond = bx.type_test(fn_ptr, typeid_metadata); + let bb_pass = bx.append_sibling_block("type_test.pass"); + let bb_fail = bx.append_sibling_block("type_test.fail"); + bx.cond_br(cond, bb_pass, bb_fail); + + bx.switch_to_block(bb_pass); + helper.do_call( + self, + &mut bx, + fn_abi, + fn_ptr, + &llargs, + target.as_ref().map(|&target| (ret_dest, target)), + cleanup, + &copied_constant_arguments, + ); + + bx.switch_to_block(bb_fail); + bx.abort(); + bx.unreachable(); + + return; + } + + helper.do_call( + self, + &mut bx, + fn_abi, + fn_ptr, + &llargs, + target.as_ref().map(|&target| (ret_dest, target)), + cleanup, + &copied_constant_arguments, + ); + } + + fn codegen_asm_terminator( + &mut self, + helper: TerminatorCodegenHelper<'tcx>, + mut bx: Bx, + terminator: &mir::Terminator<'tcx>, + template: &[ast::InlineAsmTemplatePiece], + operands: &[mir::InlineAsmOperand<'tcx>], + options: ast::InlineAsmOptions, + line_spans: &[Span], + destination: Option, + cleanup: Option, + instance: Instance<'_>, + ) { + let span = terminator.source_info.span; + + let operands: Vec<_> = operands + .iter() + .map(|op| match *op { + mir::InlineAsmOperand::In { reg, ref value } => { + let value = self.codegen_operand(&mut bx, value); + InlineAsmOperandRef::In { reg, value } + } + mir::InlineAsmOperand::Out { reg, late, ref place } => { + let place = place.map(|place| self.codegen_place(&mut bx, place.as_ref())); + InlineAsmOperandRef::Out { reg, late, place } + } + mir::InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => { + let in_value = self.codegen_operand(&mut bx, in_value); + let out_place = + out_place.map(|out_place| self.codegen_place(&mut bx, out_place.as_ref())); + InlineAsmOperandRef::InOut { reg, late, in_value, out_place } + } + mir::InlineAsmOperand::Const { ref value } => { + let const_value = self + .eval_mir_constant(value) + .unwrap_or_else(|_| span_bug!(span, "asm const cannot be resolved")); + let string = common::asm_const_to_str( + bx.tcx(), + span, + const_value, + bx.layout_of(value.ty()), + ); + InlineAsmOperandRef::Const { string } + } + mir::InlineAsmOperand::SymFn { ref value } => { + let literal = self.monomorphize(value.literal); + if let ty::FnDef(def_id, substs) = *literal.ty().kind() { + let instance = ty::Instance::resolve_for_fn_ptr( + bx.tcx(), + ty::ParamEnv::reveal_all(), + def_id, + substs, + ) + .unwrap(); + InlineAsmOperandRef::SymFn { instance } + } else { + span_bug!(span, "invalid type for asm sym (fn)"); + } + } + mir::InlineAsmOperand::SymStatic { def_id } => { + InlineAsmOperandRef::SymStatic { def_id } + } + }) + .collect(); + + helper.do_inlineasm( + self, + &mut bx, + template, + &operands, + options, + line_spans, + destination, + cleanup, + instance, + ); + } +} + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + pub fn codegen_block(&mut self, bb: mir::BasicBlock) { + let llbb = self.llbb(bb); + let mut bx = Bx::build(self.cx, llbb); + let mir = self.mir; + let data = &mir[bb]; + + debug!("codegen_block({:?}={:?})", bb, data); + + for statement in &data.statements { + bx = self.codegen_statement(bx, statement); + } + + self.codegen_terminator(bx, bb, data.terminator()); + } + + fn codegen_terminator( + &mut self, + mut bx: Bx, + bb: mir::BasicBlock, + terminator: &'tcx mir::Terminator<'tcx>, + ) { + debug!("codegen_terminator: {:?}", terminator); + + // Create the cleanup bundle, if needed. + let funclet_bb = self.cleanup_kinds[bb].funclet_bb(bb); + let helper = TerminatorCodegenHelper { bb, terminator, funclet_bb }; + + self.set_debug_loc(&mut bx, terminator.source_info); + match terminator.kind { + mir::TerminatorKind::Resume => self.codegen_resume_terminator(helper, bx), + + mir::TerminatorKind::Abort => { + self.codegen_abort_terminator(helper, bx, terminator); + } + + mir::TerminatorKind::Goto { target } => { + helper.funclet_br(self, &mut bx, target); + } + + mir::TerminatorKind::SwitchInt { ref discr, switch_ty, ref targets } => { + self.codegen_switchint_terminator(helper, bx, discr, switch_ty, targets); + } + + mir::TerminatorKind::Return => { + self.codegen_return_terminator(bx); + } + + mir::TerminatorKind::Unreachable => { + bx.unreachable(); + } + + mir::TerminatorKind::Drop { place, target, unwind } => { + self.codegen_drop_terminator(helper, bx, place, target, unwind); + } + + mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, cleanup } => { + self.codegen_assert_terminator( + helper, bx, terminator, cond, expected, msg, target, cleanup, + ); + } + + mir::TerminatorKind::DropAndReplace { .. } => { + bug!("undesugared DropAndReplace in codegen: {:?}", terminator); + } + + mir::TerminatorKind::Call { + ref func, + ref args, + destination, + target, + cleanup, + from_hir_call: _, + fn_span, + } => { + self.codegen_call_terminator( + helper, + bx, + terminator, + func, + args, + destination, + target, + cleanup, + fn_span, + ); + } + mir::TerminatorKind::GeneratorDrop | mir::TerminatorKind::Yield { .. } => { + bug!("generator ops in codegen") + } + mir::TerminatorKind::FalseEdge { .. } | mir::TerminatorKind::FalseUnwind { .. } => { + bug!("borrowck false edges in codegen") + } + + mir::TerminatorKind::InlineAsm { + template, + ref operands, + options, + line_spans, + destination, + cleanup, + } => { + self.codegen_asm_terminator( + helper, + bx, + terminator, + template, + operands, + options, + line_spans, + destination, + cleanup, + self.instance, + ); + } + } + } + + fn codegen_argument( + &mut self, + bx: &mut Bx, + op: OperandRef<'tcx, Bx::Value>, + llargs: &mut Vec, + arg: &ArgAbi<'tcx, Ty<'tcx>>, + ) { + // Fill padding with undef value, where applicable. + if let Some(ty) = arg.pad { + llargs.push(bx.const_undef(bx.reg_backend_type(&ty))) + } + + if arg.is_ignore() { + return; + } + + if let PassMode::Pair(..) = arg.mode { + match op.val { + Pair(a, b) => { + llargs.push(a); + llargs.push(b); + return; + } + _ => bug!("codegen_argument: {:?} invalid for pair argument", op), + } + } else if arg.is_unsized_indirect() { + match op.val { + Ref(a, Some(b), _) => { + llargs.push(a); + llargs.push(b); + return; + } + _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op), + } + } + + // Force by-ref if we have to load through a cast pointer. + let (mut llval, align, by_ref) = match op.val { + Immediate(_) | Pair(..) => match arg.mode { + PassMode::Indirect { .. } | PassMode::Cast(_) => { + let scratch = PlaceRef::alloca(bx, arg.layout); + op.val.store(bx, scratch); + (scratch.llval, scratch.align, true) + } + _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi, false), + }, + Ref(llval, _, align) => { + if arg.is_indirect() && align < arg.layout.align.abi { + // `foo(packed.large_field)`. We can't pass the (unaligned) field directly. I + // think that ATM (Rust 1.16) we only pass temporaries, but we shouldn't + // have scary latent bugs around. + + let scratch = PlaceRef::alloca(bx, arg.layout); + base::memcpy_ty( + bx, + scratch.llval, + scratch.align, + llval, + align, + op.layout, + MemFlags::empty(), + ); + (scratch.llval, scratch.align, true) + } else { + (llval, align, true) + } + } + }; + + if by_ref && !arg.is_indirect() { + // Have to load the argument, maybe while casting it. + if let PassMode::Cast(ty) = arg.mode { + let llty = bx.cast_backend_type(&ty); + let addr = bx.pointercast(llval, bx.type_ptr_to(llty)); + llval = bx.load(llty, addr, align.min(arg.layout.align.abi)); + } else { + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = bx.load(bx.backend_type(arg.layout), llval, align); + if let abi::Abi::Scalar(scalar) = arg.layout.abi { + if scalar.is_bool() { + bx.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = bx.to_immediate(llval, arg.layout); + } + } + + llargs.push(llval); + } + + fn codegen_arguments_untupled( + &mut self, + bx: &mut Bx, + operand: &mir::Operand<'tcx>, + llargs: &mut Vec, + args: &[ArgAbi<'tcx, Ty<'tcx>>], + ) -> usize { + let tuple = self.codegen_operand(bx, operand); + + // Handle both by-ref and immediate tuples. + if let Ref(llval, None, align) = tuple.val { + let tuple_ptr = PlaceRef::new_sized_aligned(llval, tuple.layout, align); + for i in 0..tuple.layout.fields.count() { + let field_ptr = tuple_ptr.project_field(bx, i); + let field = bx.load_operand(field_ptr); + self.codegen_argument(bx, field, llargs, &args[i]); + } + } else if let Ref(_, Some(_), _) = tuple.val { + bug!("closure arguments must be sized") + } else { + // If the tuple is immediate, the elements are as well. + for i in 0..tuple.layout.fields.count() { + let op = tuple.extract_field(bx, i); + self.codegen_argument(bx, op, llargs, &args[i]); + } + } + tuple.layout.fields.count() + } + + fn get_caller_location( + &mut self, + bx: &mut Bx, + mut source_info: mir::SourceInfo, + ) -> OperandRef<'tcx, Bx::Value> { + let tcx = bx.tcx(); + + let mut span_to_caller_location = |span: Span| { + let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span); + let caller = tcx.sess.source_map().lookup_char_pos(topmost.lo()); + let const_loc = tcx.const_caller_location(( + Symbol::intern(&caller.file.name.prefer_remapped().to_string_lossy()), + caller.line as u32, + caller.col_display as u32 + 1, + )); + OperandRef::from_const(bx, const_loc, bx.tcx().caller_location_ty()) + }; + + // Walk up the `SourceScope`s, in case some of them are from MIR inlining. + // If so, the starting `source_info.span` is in the innermost inlined + // function, and will be replaced with outer callsite spans as long + // as the inlined functions were `#[track_caller]`. + loop { + let scope_data = &self.mir.source_scopes[source_info.scope]; + + if let Some((callee, callsite_span)) = scope_data.inlined { + // Stop inside the most nested non-`#[track_caller]` function, + // before ever reaching its caller (which is irrelevant). + if !callee.def.requires_caller_location(tcx) { + return span_to_caller_location(source_info.span); + } + source_info.span = callsite_span; + } + + // Skip past all of the parents with `inlined: None`. + match scope_data.inlined_parent_scope { + Some(parent) => source_info.scope = parent, + None => break, + } + } + + // No inlined `SourceScope`s, or all of them were `#[track_caller]`. + self.caller_location.unwrap_or_else(|| span_to_caller_location(source_info.span)) + } + + fn get_personality_slot(&mut self, bx: &mut Bx) -> PlaceRef<'tcx, Bx::Value> { + let cx = bx.cx(); + if let Some(slot) = self.personality_slot { + slot + } else { + let layout = cx.layout_of( + cx.tcx().intern_tup(&[cx.tcx().mk_mut_ptr(cx.tcx().types.u8), cx.tcx().types.i32]), + ); + let slot = PlaceRef::alloca(bx, layout); + self.personality_slot = Some(slot); + slot + } + } + + /// Returns the landing/cleanup pad wrapper around the given basic block. + // FIXME(eddyb) rename this to `eh_pad_for`. + fn landing_pad_for(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock { + if let Some(landing_pad) = self.landing_pads[bb] { + return landing_pad; + } + + let landing_pad = self.landing_pad_for_uncached(bb); + self.landing_pads[bb] = Some(landing_pad); + landing_pad + } + + // FIXME(eddyb) rename this to `eh_pad_for_uncached`. + fn landing_pad_for_uncached(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock { + let llbb = self.llbb(bb); + if base::wants_msvc_seh(self.cx.sess()) { + let funclet; + let ret_llbb; + match self.mir[bb].terminator.as_ref().map(|t| &t.kind) { + // This is a basic block that we're aborting the program for, + // notably in an `extern` function. These basic blocks are inserted + // so that we assert that `extern` functions do indeed not panic, + // and if they do we abort the process. + // + // On MSVC these are tricky though (where we're doing funclets). If + // we were to do a cleanuppad (like below) the normal functions like + // `longjmp` would trigger the abort logic, terminating the + // program. Instead we insert the equivalent of `catch(...)` for C++ + // which magically doesn't trigger when `longjmp` files over this + // frame. + // + // Lots more discussion can be found on #48251 but this codegen is + // modeled after clang's for: + // + // try { + // foo(); + // } catch (...) { + // bar(); + // } + Some(&mir::TerminatorKind::Abort) => { + let cs_bb = + Bx::append_block(self.cx, self.llfn, &format!("cs_funclet{:?}", bb)); + let cp_bb = + Bx::append_block(self.cx, self.llfn, &format!("cp_funclet{:?}", bb)); + ret_llbb = cs_bb; + + let mut cs_bx = Bx::build(self.cx, cs_bb); + let cs = cs_bx.catch_switch(None, None, &[cp_bb]); + + // The "null" here is actually a RTTI type descriptor for the + // C++ personality function, but `catch (...)` has no type so + // it's null. The 64 here is actually a bitfield which + // represents that this is a catch-all block. + let mut cp_bx = Bx::build(self.cx, cp_bb); + let null = cp_bx.const_null( + cp_bx.type_i8p_ext(cp_bx.cx().data_layout().instruction_address_space), + ); + let sixty_four = cp_bx.const_i32(64); + funclet = cp_bx.catch_pad(cs, &[null, sixty_four, null]); + cp_bx.br(llbb); + } + _ => { + let cleanup_bb = + Bx::append_block(self.cx, self.llfn, &format!("funclet_{:?}", bb)); + ret_llbb = cleanup_bb; + let mut cleanup_bx = Bx::build(self.cx, cleanup_bb); + funclet = cleanup_bx.cleanup_pad(None, &[]); + cleanup_bx.br(llbb); + } + } + self.funclets[bb] = Some(funclet); + ret_llbb + } else { + let bb = Bx::append_block(self.cx, self.llfn, "cleanup"); + let mut bx = Bx::build(self.cx, bb); + + let llpersonality = self.cx.eh_personality(); + let llretty = self.landing_pad_type(); + let lp = bx.cleanup_landing_pad(llretty, llpersonality); + + let slot = self.get_personality_slot(&mut bx); + slot.storage_live(&mut bx); + Pair(bx.extract_value(lp, 0), bx.extract_value(lp, 1)).store(&mut bx, slot); + + bx.br(llbb); + bx.llbb() + } + } + + fn landing_pad_type(&self) -> Bx::Type { + let cx = self.cx; + cx.type_struct(&[cx.type_i8p(), cx.type_i32()], false) + } + + fn unreachable_block(&mut self) -> Bx::BasicBlock { + self.unreachable_block.unwrap_or_else(|| { + let llbb = Bx::append_block(self.cx, self.llfn, "unreachable"); + let mut bx = Bx::build(self.cx, llbb); + bx.unreachable(); + self.unreachable_block = Some(llbb); + llbb + }) + } + + fn double_unwind_guard(&mut self) -> Bx::BasicBlock { + self.double_unwind_guard.unwrap_or_else(|| { + assert!(!base::wants_msvc_seh(self.cx.sess())); + + let llbb = Bx::append_block(self.cx, self.llfn, "abort"); + let mut bx = Bx::build(self.cx, llbb); + self.set_debug_loc(&mut bx, mir::SourceInfo::outermost(self.mir.span)); + + let llpersonality = self.cx.eh_personality(); + let llretty = self.landing_pad_type(); + bx.cleanup_landing_pad(llretty, llpersonality); + + let (fn_abi, fn_ptr) = common::build_langcall(&bx, None, LangItem::PanicNoUnwind); + let fn_ty = bx.fn_decl_backend_type(&fn_abi); + + let llret = bx.call(fn_ty, fn_ptr, &[], None); + bx.apply_attrs_callsite(&fn_abi, llret); + bx.do_not_inline(llret); + + bx.unreachable(); + + self.double_unwind_guard = Some(llbb); + llbb + }) + } + + /// Get the backend `BasicBlock` for a MIR `BasicBlock`, either already + /// cached in `self.cached_llbbs`, or created on demand (and cached). + // FIXME(eddyb) rename `llbb` and other `ll`-prefixed things to use a + // more backend-agnostic prefix such as `cg` (i.e. this would be `cgbb`). + pub fn llbb(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock { + self.cached_llbbs[bb].unwrap_or_else(|| { + // FIXME(eddyb) only name the block if `fewer_names` is `false`. + let llbb = Bx::append_block(self.cx, self.llfn, &format!("{:?}", bb)); + self.cached_llbbs[bb] = Some(llbb); + llbb + }) + } + + fn make_return_dest( + &mut self, + bx: &mut Bx, + dest: mir::Place<'tcx>, + fn_ret: &ArgAbi<'tcx, Ty<'tcx>>, + llargs: &mut Vec, + is_intrinsic: bool, + ) -> ReturnDest<'tcx, Bx::Value> { + // If the return is ignored, we can just return a do-nothing `ReturnDest`. + if fn_ret.is_ignore() { + return ReturnDest::Nothing; + } + let dest = if let Some(index) = dest.as_local() { + match self.locals[index] { + LocalRef::Place(dest) => dest, + LocalRef::UnsizedPlace(_) => bug!("return type must be sized"), + LocalRef::Operand(None) => { + // Handle temporary places, specifically `Operand` ones, as + // they don't have `alloca`s. + return if fn_ret.is_indirect() { + // Odd, but possible, case, we have an operand temporary, + // but the calling convention has an indirect return. + let tmp = PlaceRef::alloca(bx, fn_ret.layout); + tmp.storage_live(bx); + llargs.push(tmp.llval); + ReturnDest::IndirectOperand(tmp, index) + } else if is_intrinsic { + // Currently, intrinsics always need a location to store + // the result, so we create a temporary `alloca` for the + // result. + let tmp = PlaceRef::alloca(bx, fn_ret.layout); + tmp.storage_live(bx); + ReturnDest::IndirectOperand(tmp, index) + } else { + ReturnDest::DirectOperand(index) + }; + } + LocalRef::Operand(Some(_)) => { + bug!("place local already assigned to"); + } + } + } else { + self.codegen_place( + bx, + mir::PlaceRef { local: dest.local, projection: &dest.projection }, + ) + }; + if fn_ret.is_indirect() { + if dest.align < dest.layout.align.abi { + // Currently, MIR code generation does not create calls + // that store directly to fields of packed structs (in + // fact, the calls it creates write only to temps). + // + // If someone changes that, please update this code path + // to create a temporary. + span_bug!(self.mir.span, "can't directly store to unaligned value"); + } + llargs.push(dest.llval); + ReturnDest::Nothing + } else { + ReturnDest::Store(dest) + } + } + + fn codegen_transmute(&mut self, bx: &mut Bx, src: &mir::Operand<'tcx>, dst: mir::Place<'tcx>) { + if let Some(index) = dst.as_local() { + match self.locals[index] { + LocalRef::Place(place) => self.codegen_transmute_into(bx, src, place), + LocalRef::UnsizedPlace(_) => bug!("transmute must not involve unsized locals"), + LocalRef::Operand(None) => { + let dst_layout = bx.layout_of(self.monomorphized_place_ty(dst.as_ref())); + assert!(!dst_layout.ty.has_erasable_regions()); + let place = PlaceRef::alloca(bx, dst_layout); + place.storage_live(bx); + self.codegen_transmute_into(bx, src, place); + let op = bx.load_operand(place); + place.storage_dead(bx); + self.locals[index] = LocalRef::Operand(Some(op)); + self.debug_introduce_local(bx, index); + } + LocalRef::Operand(Some(op)) => { + assert!(op.layout.is_zst(), "assigning to initialized SSAtemp"); + } + } + } else { + let dst = self.codegen_place(bx, dst.as_ref()); + self.codegen_transmute_into(bx, src, dst); + } + } + + fn codegen_transmute_into( + &mut self, + bx: &mut Bx, + src: &mir::Operand<'tcx>, + dst: PlaceRef<'tcx, Bx::Value>, + ) { + let src = self.codegen_operand(bx, src); + + // Special-case transmutes between scalars as simple bitcasts. + match (src.layout.abi, dst.layout.abi) { + (abi::Abi::Scalar(src_scalar), abi::Abi::Scalar(dst_scalar)) => { + // HACK(eddyb) LLVM doesn't like `bitcast`s between pointers and non-pointers. + if (src_scalar.primitive() == abi::Pointer) + == (dst_scalar.primitive() == abi::Pointer) + { + assert_eq!(src.layout.size, dst.layout.size); + + // NOTE(eddyb) the `from_immediate` and `to_immediate_scalar` + // conversions allow handling `bool`s the same as `u8`s. + let src = bx.from_immediate(src.immediate()); + let src_as_dst = bx.bitcast(src, bx.backend_type(dst.layout)); + Immediate(bx.to_immediate_scalar(src_as_dst, dst_scalar)).store(bx, dst); + return; + } + } + _ => {} + } + + let llty = bx.backend_type(src.layout); + let cast_ptr = bx.pointercast(dst.llval, bx.type_ptr_to(llty)); + let align = src.layout.align.abi.min(dst.align); + src.val.store(bx, PlaceRef::new_sized_aligned(cast_ptr, src.layout, align)); + } + + // Stores the return value of a function call into it's final location. + fn store_return( + &mut self, + bx: &mut Bx, + dest: ReturnDest<'tcx, Bx::Value>, + ret_abi: &ArgAbi<'tcx, Ty<'tcx>>, + llval: Bx::Value, + ) { + use self::ReturnDest::*; + + match dest { + Nothing => (), + Store(dst) => bx.store_arg(&ret_abi, llval, dst), + IndirectOperand(tmp, index) => { + let op = bx.load_operand(tmp); + tmp.storage_dead(bx); + self.locals[index] = LocalRef::Operand(Some(op)); + self.debug_introduce_local(bx, index); + } + DirectOperand(index) => { + // If there is a cast, we have to store and reload. + let op = if let PassMode::Cast(_) = ret_abi.mode { + let tmp = PlaceRef::alloca(bx, ret_abi.layout); + tmp.storage_live(bx); + bx.store_arg(&ret_abi, llval, tmp); + let op = bx.load_operand(tmp); + tmp.storage_dead(bx); + op + } else { + OperandRef::from_immediate_or_packed_pair(bx, llval, ret_abi.layout) + }; + self.locals[index] = LocalRef::Operand(Some(op)); + self.debug_introduce_local(bx, index); + } + } + } +} + +enum ReturnDest<'tcx, V> { + // Do nothing; the return value is indirect or ignored. + Nothing, + // Store the return value to the pointer. + Store(PlaceRef<'tcx, V>), + // Store an indirect return value to an operand local place. + IndirectOperand(PlaceRef<'tcx, V>, mir::Local), + // Store a direct return value to an operand local place. + DirectOperand(mir::Local), +} diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs new file mode 100644 index 00000000000..9a995fbf65c --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs @@ -0,0 +1,90 @@ +use crate::mir::operand::OperandRef; +use crate::traits::*; +use rustc_middle::mir; +use rustc_middle::mir::interpret::{ConstValue, ErrorHandled}; +use rustc_middle::ty::layout::HasTyCtxt; +use rustc_middle::ty::{self, Ty}; +use rustc_span::source_map::Span; +use rustc_target::abi::Abi; + +use super::FunctionCx; + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + pub fn eval_mir_constant_to_operand( + &self, + bx: &mut Bx, + constant: &mir::Constant<'tcx>, + ) -> Result, ErrorHandled> { + let val = self.eval_mir_constant(constant)?; + let ty = self.monomorphize(constant.ty()); + Ok(OperandRef::from_const(bx, val, ty)) + } + + pub fn eval_mir_constant( + &self, + constant: &mir::Constant<'tcx>, + ) -> Result, ErrorHandled> { + let ct = self.monomorphize(constant.literal); + let ct = match ct { + mir::ConstantKind::Ty(ct) => ct, + mir::ConstantKind::Val(val, _) => return Ok(val), + }; + match ct.kind() { + ty::ConstKind::Unevaluated(ct) => self + .cx + .tcx() + .const_eval_resolve(ty::ParamEnv::reveal_all(), ct, None) + .map_err(|err| { + self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered"); + err + }), + ty::ConstKind::Value(val) => Ok(self.cx.tcx().valtree_to_const_val((ct.ty(), val))), + err => span_bug!( + constant.span, + "encountered bad ConstKind after monomorphizing: {:?}", + err + ), + } + } + + /// process constant containing SIMD shuffle indices + pub fn simd_shuffle_indices( + &mut self, + bx: &Bx, + span: Span, + ty: Ty<'tcx>, + constant: Result, ErrorHandled>, + ) -> (Bx::Value, Ty<'tcx>) { + constant + .map(|val| { + let field_ty = ty.builtin_index().unwrap(); + let c = mir::ConstantKind::from_value(val, ty); + let values: Vec<_> = bx + .tcx() + .destructure_mir_constant(ty::ParamEnv::reveal_all(), c) + .fields + .iter() + .map(|field| { + if let Some(prim) = field.try_to_scalar() { + let layout = bx.layout_of(field_ty); + let Abi::Scalar(scalar) = layout.abi else { + bug!("from_const: invalid ByVal layout: {:#?}", layout); + }; + bx.scalar_to_backend(prim, scalar, bx.immediate_backend_type(layout)) + } else { + bug!("simd shuffle field {:?}", field) + } + }) + .collect(); + let llval = bx.const_struct(&values, false); + (llval, c.ty()) + }) + .unwrap_or_else(|_| { + bx.tcx().sess.span_err(span, "could not evaluate shuffle_indices at compile time"); + // We've errored, so we don't have to produce working code. + let ty = self.monomorphize(ty); + let llty = bx.backend_type(bx.layout_of(ty)); + (bx.const_undef(llty), ty) + }) + } +} diff --git a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs new file mode 100644 index 00000000000..f1fe495282a --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs @@ -0,0 +1,55 @@ +use crate::traits::*; + +use rustc_middle::mir::coverage::*; +use rustc_middle::mir::Coverage; +use rustc_middle::mir::SourceScope; + +use super::FunctionCx; + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage, scope: SourceScope) { + // Determine the instance that coverage data was originally generated for. + let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) { + self.monomorphize(inlined) + } else { + self.instance + }; + + let Coverage { kind, code_region } = coverage; + match kind { + CoverageKind::Counter { function_source_hash, id } => { + if bx.set_function_source_hash(instance, function_source_hash) { + // If `set_function_source_hash()` returned true, the coverage map is enabled, + // so continue adding the counter. + if let Some(code_region) = code_region { + // Note: Some counters do not have code regions, but may still be referenced + // from expressions. In that case, don't add the counter to the coverage map, + // but do inject the counter intrinsic. + bx.add_coverage_counter(instance, id, code_region); + } + + let coverageinfo = bx.tcx().coverageinfo(instance.def); + + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(function_source_hash); + let num_counters = bx.const_u32(coverageinfo.num_counters); + let index = bx.const_u32(id.zero_based_index()); + debug!( + "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", + fn_name, hash, num_counters, index, + ); + bx.instrprof_increment(fn_name, hash, num_counters, index); + } + } + CoverageKind::Expression { id, lhs, op, rhs } => { + bx.add_coverage_counter_expression(instance, id, lhs, op, rhs, code_region); + } + CoverageKind::Unreachable => { + bx.add_coverage_unreachable( + instance, + code_region.expect("unreachable regions always have code regions"), + ); + } + } + } +} diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs new file mode 100644 index 00000000000..8c3186efc63 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -0,0 +1,418 @@ +use crate::traits::*; +use rustc_index::vec::IndexVec; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mir; +use rustc_middle::ty; +use rustc_middle::ty::layout::LayoutOf; +use rustc_session::config::DebugInfo; +use rustc_span::symbol::{kw, Symbol}; +use rustc_span::{BytePos, Span}; +use rustc_target::abi::Abi; +use rustc_target::abi::Size; + +use super::operand::{OperandRef, OperandValue}; +use super::place::PlaceRef; +use super::{FunctionCx, LocalRef}; + +pub struct FunctionDebugContext { + pub scopes: IndexVec>, +} + +#[derive(Copy, Clone)] +pub enum VariableKind { + ArgumentVariable(usize /*index*/), + LocalVariable, +} + +/// Like `mir::VarDebugInfo`, but within a `mir::Local`. +#[derive(Copy, Clone)] +pub struct PerLocalVarDebugInfo<'tcx, D> { + pub name: Symbol, + pub source_info: mir::SourceInfo, + + /// `DIVariable` returned by `create_dbg_var`. + pub dbg_var: Option, + + /// `.place.projection` from `mir::VarDebugInfo`. + pub projection: &'tcx ty::List>, +} + +#[derive(Clone, Copy, Debug)] +pub struct DebugScope { + pub dbg_scope: S, + + /// Call site location, if this scope was inlined from another function. + pub inlined_at: Option, + + // Start and end offsets of the file to which this DIScope belongs. + // These are used to quickly determine whether some span refers to the same file. + pub file_start_pos: BytePos, + pub file_end_pos: BytePos, +} + +impl<'tcx, S: Copy, L: Copy> DebugScope { + /// DILocations inherit source file name from the parent DIScope. Due to macro expansions + /// it may so happen that the current span belongs to a different file than the DIScope + /// corresponding to span's containing source scope. If so, we need to create a DIScope + /// "extension" into that file. + pub fn adjust_dbg_scope_for_span>( + &self, + cx: &Cx, + span: Span, + ) -> S { + let pos = span.lo(); + if pos < self.file_start_pos || pos >= self.file_end_pos { + let sm = cx.sess().source_map(); + cx.extend_scope_to_file(self.dbg_scope, &sm.lookup_char_pos(pos).file) + } else { + self.dbg_scope + } + } +} + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) { + bx.set_span(source_info.span); + if let Some(dbg_loc) = self.dbg_loc(source_info) { + bx.set_dbg_loc(dbg_loc); + } + } + + fn dbg_loc(&self, source_info: mir::SourceInfo) -> Option { + let (dbg_scope, inlined_at, span) = self.adjusted_span_and_dbg_scope(source_info)?; + Some(self.cx.dbg_loc(dbg_scope, inlined_at, span)) + } + + fn adjusted_span_and_dbg_scope( + &self, + source_info: mir::SourceInfo, + ) -> Option<(Bx::DIScope, Option, Span)> { + let span = self.adjust_span_for_debugging(source_info.span); + let scope = &self.debug_context.as_ref()?.scopes[source_info.scope]; + Some((scope.adjust_dbg_scope_for_span(self.cx, span), scope.inlined_at, span)) + } + + /// In order to have a good line stepping behavior in debugger, we overwrite debug + /// locations of macro expansions with that of the outermost expansion site + /// (unless the crate is being compiled with `-Z debug-macros`). + fn adjust_span_for_debugging(&self, mut span: Span) -> Span { + // Bail out if debug info emission is not enabled. + if self.debug_context.is_none() { + return span; + } + + if span.from_expansion() && !self.cx.sess().opts.unstable_opts.debug_macros { + // Walk up the macro expansion chain until we reach a non-expanded span. + // We also stop at the function body level because no line stepping can occur + // at the level above that. + // Use span of the outermost expansion site, while keeping the original lexical scope. + span = rustc_span::hygiene::walk_chain(span, self.mir.span.ctxt()); + } + + span + } + + fn spill_operand_to_stack( + operand: &OperandRef<'tcx, Bx::Value>, + name: Option, + bx: &mut Bx, + ) -> PlaceRef<'tcx, Bx::Value> { + // "Spill" the value onto the stack, for debuginfo, + // without forcing non-debuginfo uses of the local + // to also load from the stack every single time. + // FIXME(#68817) use `llvm.dbg.value` instead, + // at least for the cases which LLVM handles correctly. + let spill_slot = PlaceRef::alloca(bx, operand.layout); + if let Some(name) = name { + bx.set_var_name(spill_slot.llval, &(name + ".dbg.spill")); + } + operand.val.store(bx, spill_slot); + spill_slot + } + + /// Apply debuginfo and/or name, after creating the `alloca` for a local, + /// or initializing the local with an operand (whichever applies). + pub fn debug_introduce_local(&self, bx: &mut Bx, local: mir::Local) { + let full_debug_info = bx.sess().opts.debuginfo == DebugInfo::Full; + + // FIXME(eddyb) maybe name the return place as `_0` or `return`? + if local == mir::RETURN_PLACE && !self.mir.local_decls[mir::RETURN_PLACE].is_user_variable() + { + return; + } + + let vars = match &self.per_local_var_debug_info { + Some(per_local) => &per_local[local], + None => return, + }; + let whole_local_var = vars.iter().find(|var| var.projection.is_empty()).copied(); + let has_proj = || vars.iter().any(|var| !var.projection.is_empty()); + + let fallback_var = if self.mir.local_kind(local) == mir::LocalKind::Arg { + let arg_index = local.index() - 1; + + // Add debuginfo even to unnamed arguments. + // FIXME(eddyb) is this really needed? + if arg_index == 0 && has_proj() { + // Hide closure environments from debuginfo. + // FIXME(eddyb) shouldn't `ArgumentVariable` indices + // be offset to account for the hidden environment? + None + } else if whole_local_var.is_some() { + // No need to make up anything, there is a `mir::VarDebugInfo` + // covering the whole local. + // FIXME(eddyb) take `whole_local_var.source_info.scope` into + // account, just in case it doesn't use `ArgumentVariable` + // (after #67586 gets fixed). + None + } else { + let name = kw::Empty; + let decl = &self.mir.local_decls[local]; + let dbg_var = if full_debug_info { + self.adjusted_span_and_dbg_scope(decl.source_info).map( + |(dbg_scope, _, span)| { + // FIXME(eddyb) is this `+ 1` needed at all? + let kind = VariableKind::ArgumentVariable(arg_index + 1); + + let arg_ty = self.monomorphize(decl.ty); + + self.cx.create_dbg_var(name, arg_ty, dbg_scope, kind, span) + }, + ) + } else { + None + }; + + Some(PerLocalVarDebugInfo { + name, + source_info: decl.source_info, + dbg_var, + projection: ty::List::empty(), + }) + } + } else { + None + }; + + let local_ref = &self.locals[local]; + + let name = if bx.sess().fewer_names() { + None + } else { + Some(match whole_local_var.or(fallback_var) { + Some(var) if var.name != kw::Empty => var.name.to_string(), + _ => format!("{:?}", local), + }) + }; + + if let Some(name) = &name { + match local_ref { + LocalRef::Place(place) | LocalRef::UnsizedPlace(place) => { + bx.set_var_name(place.llval, name); + } + LocalRef::Operand(Some(operand)) => match operand.val { + OperandValue::Ref(x, ..) | OperandValue::Immediate(x) => { + bx.set_var_name(x, name); + } + OperandValue::Pair(a, b) => { + // FIXME(eddyb) these are scalar components, + // maybe extract the high-level fields? + bx.set_var_name(a, &(name.clone() + ".0")); + bx.set_var_name(b, &(name.clone() + ".1")); + } + }, + LocalRef::Operand(None) => {} + } + } + + if !full_debug_info || vars.is_empty() && fallback_var.is_none() { + return; + } + + let base = match local_ref { + LocalRef::Operand(None) => return, + + LocalRef::Operand(Some(operand)) => { + // Don't spill operands onto the stack in naked functions. + // See: https://github.com/rust-lang/rust/issues/42779 + let attrs = bx.tcx().codegen_fn_attrs(self.instance.def_id()); + if attrs.flags.contains(CodegenFnAttrFlags::NAKED) { + return; + } + + Self::spill_operand_to_stack(operand, name, bx) + } + + LocalRef::Place(place) => *place, + + // FIXME(eddyb) add debuginfo for unsized places too. + LocalRef::UnsizedPlace(_) => return, + }; + + let vars = vars.iter().copied().chain(fallback_var); + + for var in vars { + let Some(dbg_var) = var.dbg_var else { continue }; + let Some(dbg_loc) = self.dbg_loc(var.source_info) else { continue }; + + let mut direct_offset = Size::ZERO; + // FIXME(eddyb) use smallvec here. + let mut indirect_offsets = vec![]; + let mut place = base; + + for elem in &var.projection[..] { + match *elem { + mir::ProjectionElem::Deref => { + indirect_offsets.push(Size::ZERO); + place = bx.load_operand(place).deref(bx.cx()); + } + mir::ProjectionElem::Field(field, _) => { + let i = field.index(); + let offset = indirect_offsets.last_mut().unwrap_or(&mut direct_offset); + *offset += place.layout.fields.offset(i); + place = place.project_field(bx, i); + } + mir::ProjectionElem::Downcast(_, variant) => { + place = place.project_downcast(bx, variant); + } + _ => span_bug!( + var.source_info.span, + "unsupported var debuginfo place `{:?}`", + mir::Place { local, projection: var.projection }, + ), + } + } + + // When targeting MSVC, create extra allocas for arguments instead of pointing multiple + // dbg_var_addr() calls into the same alloca with offsets. MSVC uses CodeView records + // not DWARF and LLVM doesn't support translating the resulting + // [DW_OP_deref, DW_OP_plus_uconst, offset, DW_OP_deref] debug info to CodeView. + // Creating extra allocas on the stack makes the resulting debug info simple enough + // that LLVM can generate correct CodeView records and thus the values appear in the + // debugger. (#83709) + let should_create_individual_allocas = bx.cx().sess().target.is_like_msvc + && self.mir.local_kind(local) == mir::LocalKind::Arg + // LLVM can handle simple things but anything more complex than just a direct + // offset or one indirect offset of 0 is too complex for it to generate CV records + // correctly. + && (direct_offset != Size::ZERO + || !matches!(&indirect_offsets[..], [Size::ZERO] | [])); + + if should_create_individual_allocas { + // Create a variable which will be a pointer to the actual value + let ptr_ty = bx.tcx().mk_ty(ty::RawPtr(ty::TypeAndMut { + mutbl: mir::Mutability::Mut, + ty: place.layout.ty, + })); + let ptr_layout = bx.layout_of(ptr_ty); + let alloca = PlaceRef::alloca(bx, ptr_layout); + bx.set_var_name(alloca.llval, &(var.name.to_string() + ".dbg.spill")); + + // Write the pointer to the variable + bx.store(place.llval, alloca.llval, alloca.align); + + // Point the debug info to `*alloca` for the current variable + bx.dbg_var_addr(dbg_var, dbg_loc, alloca.llval, Size::ZERO, &[Size::ZERO]); + } else { + bx.dbg_var_addr(dbg_var, dbg_loc, base.llval, direct_offset, &indirect_offsets); + } + } + } + + pub fn debug_introduce_locals(&self, bx: &mut Bx) { + if bx.sess().opts.debuginfo == DebugInfo::Full || !bx.sess().fewer_names() { + for local in self.locals.indices() { + self.debug_introduce_local(bx, local); + } + } + } + + /// Partition all `VarDebugInfo` in `self.mir`, by their base `Local`. + pub fn compute_per_local_var_debug_info( + &self, + bx: &mut Bx, + ) -> Option>>> { + let full_debug_info = self.cx.sess().opts.debuginfo == DebugInfo::Full; + + let target_is_msvc = self.cx.sess().target.is_like_msvc; + + if !full_debug_info && self.cx.sess().fewer_names() { + return None; + } + + let mut per_local = IndexVec::from_elem(vec![], &self.mir.local_decls); + for var in &self.mir.var_debug_info { + let dbg_scope_and_span = if full_debug_info { + self.adjusted_span_and_dbg_scope(var.source_info) + } else { + None + }; + + let dbg_var = dbg_scope_and_span.map(|(dbg_scope, _, span)| { + let (var_ty, var_kind) = match var.value { + mir::VarDebugInfoContents::Place(place) => { + let var_ty = self.monomorphized_place_ty(place.as_ref()); + let var_kind = if self.mir.local_kind(place.local) == mir::LocalKind::Arg + && place.projection.is_empty() + && var.source_info.scope == mir::OUTERMOST_SOURCE_SCOPE + { + let arg_index = place.local.index() - 1; + if target_is_msvc { + // ScalarPair parameters are spilled to the stack so they need to + // be marked as a `LocalVariable` for MSVC debuggers to visualize + // their data correctly. (See #81894 & #88625) + let var_ty_layout = self.cx.layout_of(var_ty); + if let Abi::ScalarPair(_, _) = var_ty_layout.abi { + VariableKind::LocalVariable + } else { + VariableKind::ArgumentVariable(arg_index + 1) + } + } else { + // FIXME(eddyb) shouldn't `ArgumentVariable` indices be + // offset in closures to account for the hidden environment? + // Also, is this `+ 1` needed at all? + VariableKind::ArgumentVariable(arg_index + 1) + } + } else { + VariableKind::LocalVariable + }; + (var_ty, var_kind) + } + mir::VarDebugInfoContents::Const(c) => { + let ty = self.monomorphize(c.ty()); + (ty, VariableKind::LocalVariable) + } + }; + + self.cx.create_dbg_var(var.name, var_ty, dbg_scope, var_kind, span) + }); + + match var.value { + mir::VarDebugInfoContents::Place(place) => { + per_local[place.local].push(PerLocalVarDebugInfo { + name: var.name, + source_info: var.source_info, + dbg_var, + projection: place.projection, + }); + } + mir::VarDebugInfoContents::Const(c) => { + if let Some(dbg_var) = dbg_var { + let Some(dbg_loc) = self.dbg_loc(var.source_info) else { continue }; + + if let Ok(operand) = self.eval_mir_constant_to_operand(bx, &c) { + let base = Self::spill_operand_to_stack( + &operand, + Some(var.name.to_string()), + bx, + ); + + bx.dbg_var_addr(dbg_var, dbg_loc, base.llval, Size::ZERO, &[]); + } + } + } + } + } + Some(per_local) + } +} diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs new file mode 100644 index 00000000000..94ac71a4dd2 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -0,0 +1,636 @@ +use super::operand::{OperandRef, OperandValue}; +use super::place::PlaceRef; +use super::FunctionCx; +use crate::common::{span_invalid_monomorphization_error, IntPredicate}; +use crate::glue; +use crate::meth; +use crate::traits::*; +use crate::MemFlags; + +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_span::{sym, Span}; +use rustc_target::abi::{ + call::{FnAbi, PassMode}, + WrappingRange, +}; + +fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + allow_overlap: bool, + volatile: bool, + ty: Ty<'tcx>, + dst: Bx::Value, + src: Bx::Value, + count: Bx::Value, +) { + let layout = bx.layout_of(ty); + let size = layout.size; + let align = layout.align.abi; + let size = bx.mul(bx.const_usize(size.bytes()), count); + let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() }; + if allow_overlap { + bx.memmove(dst, align, src, align, size, flags); + } else { + bx.memcpy(dst, align, src, align, size, flags); + } +} + +fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + volatile: bool, + ty: Ty<'tcx>, + dst: Bx::Value, + val: Bx::Value, + count: Bx::Value, +) { + let layout = bx.layout_of(ty); + let size = layout.size; + let align = layout.align.abi; + let size = bx.mul(bx.const_usize(size.bytes()), count); + let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() }; + bx.memset(dst, val, size, align, flags); +} + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + pub fn codegen_intrinsic_call( + bx: &mut Bx, + instance: ty::Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + args: &[OperandRef<'tcx, Bx::Value>], + llresult: Bx::Value, + span: Span, + ) { + let callee_ty = instance.ty(bx.tcx(), ty::ParamEnv::reveal_all()); + + let ty::FnDef(def_id, substs) = *callee_ty.kind() else { + bug!("expected fn item type, found {}", callee_ty); + }; + + let sig = callee_ty.fn_sig(bx.tcx()); + let sig = bx.tcx().normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig); + let arg_tys = sig.inputs(); + let ret_ty = sig.output(); + let name = bx.tcx().item_name(def_id); + let name_str = name.as_str(); + + let llret_ty = bx.backend_type(bx.layout_of(ret_ty)); + let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); + + let llval = match name { + sym::assume => { + bx.assume(args[0].immediate()); + return; + } + sym::abort => { + bx.abort(); + return; + } + + sym::va_start => bx.va_start(args[0].immediate()), + sym::va_end => bx.va_end(args[0].immediate()), + sym::size_of_val => { + let tp_ty = substs.type_at(0); + if let OperandValue::Pair(_, meta) = args[0].val { + let (llsize, _) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta)); + llsize + } else { + bx.const_usize(bx.layout_of(tp_ty).size.bytes()) + } + } + sym::min_align_of_val => { + let tp_ty = substs.type_at(0); + if let OperandValue::Pair(_, meta) = args[0].val { + let (_, llalign) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta)); + llalign + } else { + bx.const_usize(bx.layout_of(tp_ty).align.abi.bytes()) + } + } + sym::vtable_size | sym::vtable_align => { + let vtable = args[0].immediate(); + let idx = match name { + sym::vtable_size => ty::COMMON_VTABLE_ENTRIES_SIZE, + sym::vtable_align => ty::COMMON_VTABLE_ENTRIES_ALIGN, + _ => bug!(), + }; + let value = meth::VirtualIndex::from_index(idx).get_usize(bx, vtable); + if name == sym::vtable_align { + // Alignment is always nonzero. + bx.range_metadata(value, WrappingRange { start: 1, end: !0 }); + }; + value + } + sym::pref_align_of + | sym::needs_drop + | sym::type_id + | sym::type_name + | sym::variant_count => { + let value = bx + .tcx() + .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None) + .unwrap(); + OperandRef::from_const(bx, value, ret_ty).immediate_or_packed_pair(bx) + } + sym::offset => { + let ty = substs.type_at(0); + let layout = bx.layout_of(ty); + let ptr = args[0].immediate(); + let offset = args[1].immediate(); + bx.inbounds_gep(bx.backend_type(layout), ptr, &[offset]) + } + sym::arith_offset => { + let ty = substs.type_at(0); + let layout = bx.layout_of(ty); + let ptr = args[0].immediate(); + let offset = args[1].immediate(); + bx.gep(bx.backend_type(layout), ptr, &[offset]) + } + sym::copy => { + copy_intrinsic( + bx, + true, + false, + substs.type_at(0), + args[1].immediate(), + args[0].immediate(), + args[2].immediate(), + ); + return; + } + sym::write_bytes => { + memset_intrinsic( + bx, + false, + substs.type_at(0), + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + ); + return; + } + + sym::volatile_copy_nonoverlapping_memory => { + copy_intrinsic( + bx, + false, + true, + substs.type_at(0), + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + ); + return; + } + sym::volatile_copy_memory => { + copy_intrinsic( + bx, + true, + true, + substs.type_at(0), + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + ); + return; + } + sym::volatile_set_memory => { + memset_intrinsic( + bx, + true, + substs.type_at(0), + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + ); + return; + } + sym::volatile_store => { + let dst = args[0].deref(bx.cx()); + args[1].val.volatile_store(bx, dst); + return; + } + sym::unaligned_volatile_store => { + let dst = args[0].deref(bx.cx()); + args[1].val.unaligned_volatile_store(bx, dst); + return; + } + sym::add_with_overflow + | sym::sub_with_overflow + | sym::mul_with_overflow + | sym::unchecked_div + | sym::unchecked_rem + | sym::unchecked_shl + | sym::unchecked_shr + | sym::unchecked_add + | sym::unchecked_sub + | sym::unchecked_mul + | sym::exact_div => { + let ty = arg_tys[0]; + match int_type_width_signed(ty, bx.tcx()) { + Some((_width, signed)) => match name { + sym::add_with_overflow + | sym::sub_with_overflow + | sym::mul_with_overflow => { + let op = match name { + sym::add_with_overflow => OverflowOp::Add, + sym::sub_with_overflow => OverflowOp::Sub, + sym::mul_with_overflow => OverflowOp::Mul, + _ => bug!(), + }; + let (val, overflow) = + bx.checked_binop(op, ty, args[0].immediate(), args[1].immediate()); + // Convert `i1` to a `bool`, and write it to the out parameter + let val = bx.from_immediate(val); + let overflow = bx.from_immediate(overflow); + + let dest = result.project_field(bx, 0); + bx.store(val, dest.llval, dest.align); + let dest = result.project_field(bx, 1); + bx.store(overflow, dest.llval, dest.align); + + return; + } + sym::exact_div => { + if signed { + bx.exactsdiv(args[0].immediate(), args[1].immediate()) + } else { + bx.exactudiv(args[0].immediate(), args[1].immediate()) + } + } + sym::unchecked_div => { + if signed { + bx.sdiv(args[0].immediate(), args[1].immediate()) + } else { + bx.udiv(args[0].immediate(), args[1].immediate()) + } + } + sym::unchecked_rem => { + if signed { + bx.srem(args[0].immediate(), args[1].immediate()) + } else { + bx.urem(args[0].immediate(), args[1].immediate()) + } + } + sym::unchecked_shl => bx.shl(args[0].immediate(), args[1].immediate()), + sym::unchecked_shr => { + if signed { + bx.ashr(args[0].immediate(), args[1].immediate()) + } else { + bx.lshr(args[0].immediate(), args[1].immediate()) + } + } + sym::unchecked_add => { + if signed { + bx.unchecked_sadd(args[0].immediate(), args[1].immediate()) + } else { + bx.unchecked_uadd(args[0].immediate(), args[1].immediate()) + } + } + sym::unchecked_sub => { + if signed { + bx.unchecked_ssub(args[0].immediate(), args[1].immediate()) + } else { + bx.unchecked_usub(args[0].immediate(), args[1].immediate()) + } + } + sym::unchecked_mul => { + if signed { + bx.unchecked_smul(args[0].immediate(), args[1].immediate()) + } else { + bx.unchecked_umul(args[0].immediate(), args[1].immediate()) + } + } + _ => bug!(), + }, + None => { + span_invalid_monomorphization_error( + bx.tcx().sess, + span, + &format!( + "invalid monomorphization of `{}` intrinsic: \ + expected basic integer type, found `{}`", + name, ty + ), + ); + return; + } + } + } + sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => { + match float_type_width(arg_tys[0]) { + Some(_width) => match name { + sym::fadd_fast => bx.fadd_fast(args[0].immediate(), args[1].immediate()), + sym::fsub_fast => bx.fsub_fast(args[0].immediate(), args[1].immediate()), + sym::fmul_fast => bx.fmul_fast(args[0].immediate(), args[1].immediate()), + sym::fdiv_fast => bx.fdiv_fast(args[0].immediate(), args[1].immediate()), + sym::frem_fast => bx.frem_fast(args[0].immediate(), args[1].immediate()), + _ => bug!(), + }, + None => { + span_invalid_monomorphization_error( + bx.tcx().sess, + span, + &format!( + "invalid monomorphization of `{}` intrinsic: \ + expected basic float type, found `{}`", + name, arg_tys[0] + ), + ); + return; + } + } + } + + sym::float_to_int_unchecked => { + if float_type_width(arg_tys[0]).is_none() { + span_invalid_monomorphization_error( + bx.tcx().sess, + span, + &format!( + "invalid monomorphization of `float_to_int_unchecked` \ + intrinsic: expected basic float type, \ + found `{}`", + arg_tys[0] + ), + ); + return; + } + let Some((_width, signed)) = int_type_width_signed(ret_ty, bx.tcx()) else { + span_invalid_monomorphization_error( + bx.tcx().sess, + span, + &format!( + "invalid monomorphization of `float_to_int_unchecked` \ + intrinsic: expected basic integer type, \ + found `{}`", + ret_ty + ), + ); + return; + }; + if signed { + bx.fptosi(args[0].immediate(), llret_ty) + } else { + bx.fptoui(args[0].immediate(), llret_ty) + } + } + + sym::discriminant_value => { + if ret_ty.is_integral() { + args[0].deref(bx.cx()).codegen_get_discr(bx, ret_ty) + } else { + span_bug!(span, "Invalid discriminant type for `{:?}`", arg_tys[0]) + } + } + + sym::const_allocate => { + // returns a null pointer at runtime. + bx.const_null(bx.type_i8p()) + } + + sym::const_deallocate => { + // nop at runtime. + return; + } + + // This requires that atomic intrinsics follow a specific naming pattern: + // "atomic_[_]" + name if let Some(atomic) = name_str.strip_prefix("atomic_") => { + use crate::common::AtomicOrdering::*; + use crate::common::{AtomicRmwBinOp, SynchronizationScope}; + + let Some((instruction, ordering)) = atomic.split_once('_') else { + bx.sess().fatal("Atomic intrinsic missing memory ordering"); + }; + + let parse_ordering = |bx: &Bx, s| match s { + "unordered" => Unordered, + "relaxed" => Relaxed, + "acquire" => Acquire, + "release" => Release, + "acqrel" => AcquireRelease, + "seqcst" => SequentiallyConsistent, + _ => bx.sess().fatal("unknown ordering in atomic intrinsic"), + }; + + let invalid_monomorphization = |ty| { + span_invalid_monomorphization_error( + bx.tcx().sess, + span, + &format!( + "invalid monomorphization of `{}` intrinsic: \ + expected basic integer type, found `{}`", + name, ty + ), + ); + }; + + match instruction { + "cxchg" | "cxchgweak" => { + let Some((success, failure)) = ordering.split_once('_') else { + bx.sess().fatal("Atomic compare-exchange intrinsic missing failure memory ordering"); + }; + let ty = substs.type_at(0); + if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { + let weak = instruction == "cxchgweak"; + let mut dst = args[0].immediate(); + let mut cmp = args[1].immediate(); + let mut src = args[2].immediate(); + if ty.is_unsafe_ptr() { + // Some platforms do not support atomic operations on pointers, + // so we cast to integer first. + let ptr_llty = bx.type_ptr_to(bx.type_isize()); + dst = bx.pointercast(dst, ptr_llty); + cmp = bx.ptrtoint(cmp, bx.type_isize()); + src = bx.ptrtoint(src, bx.type_isize()); + } + let pair = bx.atomic_cmpxchg(dst, cmp, src, parse_ordering(bx, success), parse_ordering(bx, failure), weak); + let val = bx.extract_value(pair, 0); + let success = bx.extract_value(pair, 1); + let val = bx.from_immediate(val); + let success = bx.from_immediate(success); + + let dest = result.project_field(bx, 0); + bx.store(val, dest.llval, dest.align); + let dest = result.project_field(bx, 1); + bx.store(success, dest.llval, dest.align); + return; + } else { + return invalid_monomorphization(ty); + } + } + + "load" => { + let ty = substs.type_at(0); + if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { + let layout = bx.layout_of(ty); + let size = layout.size; + let mut source = args[0].immediate(); + if ty.is_unsafe_ptr() { + // Some platforms do not support atomic operations on pointers, + // so we cast to integer first... + let llty = bx.type_isize(); + let ptr_llty = bx.type_ptr_to(llty); + source = bx.pointercast(source, ptr_llty); + let result = bx.atomic_load(llty, source, parse_ordering(bx, ordering), size); + // ... and then cast the result back to a pointer + bx.inttoptr(result, bx.backend_type(layout)) + } else { + bx.atomic_load(bx.backend_type(layout), source, parse_ordering(bx, ordering), size) + } + } else { + return invalid_monomorphization(ty); + } + } + + "store" => { + let ty = substs.type_at(0); + if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { + let size = bx.layout_of(ty).size; + let mut val = args[1].immediate(); + let mut ptr = args[0].immediate(); + if ty.is_unsafe_ptr() { + // Some platforms do not support atomic operations on pointers, + // so we cast to integer first. + let ptr_llty = bx.type_ptr_to(bx.type_isize()); + ptr = bx.pointercast(ptr, ptr_llty); + val = bx.ptrtoint(val, bx.type_isize()); + } + bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size); + return; + } else { + return invalid_monomorphization(ty); + } + } + + "fence" => { + bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::CrossThread); + return; + } + + "singlethreadfence" => { + bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::SingleThread); + return; + } + + // These are all AtomicRMW ops + op => { + let atom_op = match op { + "xchg" => AtomicRmwBinOp::AtomicXchg, + "xadd" => AtomicRmwBinOp::AtomicAdd, + "xsub" => AtomicRmwBinOp::AtomicSub, + "and" => AtomicRmwBinOp::AtomicAnd, + "nand" => AtomicRmwBinOp::AtomicNand, + "or" => AtomicRmwBinOp::AtomicOr, + "xor" => AtomicRmwBinOp::AtomicXor, + "max" => AtomicRmwBinOp::AtomicMax, + "min" => AtomicRmwBinOp::AtomicMin, + "umax" => AtomicRmwBinOp::AtomicUMax, + "umin" => AtomicRmwBinOp::AtomicUMin, + _ => bx.sess().fatal("unknown atomic operation"), + }; + + let ty = substs.type_at(0); + if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { + let mut ptr = args[0].immediate(); + let mut val = args[1].immediate(); + if ty.is_unsafe_ptr() { + // Some platforms do not support atomic operations on pointers, + // so we cast to integer first. + let ptr_llty = bx.type_ptr_to(bx.type_isize()); + ptr = bx.pointercast(ptr, ptr_llty); + val = bx.ptrtoint(val, bx.type_isize()); + } + bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering)) + } else { + return invalid_monomorphization(ty); + } + } + } + } + + sym::nontemporal_store => { + let dst = args[0].deref(bx.cx()); + args[1].val.nontemporal_store(bx, dst); + return; + } + + sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => { + let a = args[0].immediate(); + let b = args[1].immediate(); + if name == sym::ptr_guaranteed_eq { + bx.icmp(IntPredicate::IntEQ, a, b) + } else { + bx.icmp(IntPredicate::IntNE, a, b) + } + } + + sym::ptr_offset_from | sym::ptr_offset_from_unsigned => { + let ty = substs.type_at(0); + let pointee_size = bx.layout_of(ty).size; + + let a = args[0].immediate(); + let b = args[1].immediate(); + let a = bx.ptrtoint(a, bx.type_isize()); + let b = bx.ptrtoint(b, bx.type_isize()); + let pointee_size = bx.const_usize(pointee_size.bytes()); + if name == sym::ptr_offset_from { + // This is the same sequence that Clang emits for pointer subtraction. + // It can be neither `nsw` nor `nuw` because the input is treated as + // unsigned but then the output is treated as signed, so neither works. + let d = bx.sub(a, b); + // this is where the signed magic happens (notice the `s` in `exactsdiv`) + bx.exactsdiv(d, pointee_size) + } else { + // The `_unsigned` version knows the relative ordering of the pointers, + // so can use `sub nuw` and `udiv exact` instead of dealing in signed. + let d = bx.unchecked_usub(a, b); + bx.exactudiv(d, pointee_size) + } + } + + _ => { + // Need to use backend-specific things in the implementation. + bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span); + return; + } + }; + + if !fn_abi.ret.is_ignore() { + if let PassMode::Cast(ty) = fn_abi.ret.mode { + let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(&ty)); + let ptr = bx.pointercast(result.llval, ptr_llty); + bx.store(llval, ptr, result.align); + } else { + OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout) + .val + .store(bx, result); + } + } + } +} + +// Returns the width of an int Ty, and if it's signed or not +// Returns None if the type is not an integer +// FIXME: there’s multiple of this functions, investigate using some of the already existing +// stuffs. +fn int_type_width_signed(ty: Ty<'_>, tcx: TyCtxt<'_>) -> Option<(u64, bool)> { + match ty.kind() { + ty::Int(t) => { + Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), true)) + } + ty::Uint(t) => { + Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), false)) + } + _ => None, + } +} + +// Returns the width of a float Ty +// Returns None if the type is not a float +fn float_type_width(ty: Ty<'_>) -> Option { + match ty.kind() { + ty::Float(t) => Some(t.bit_width()), + _ => None, + } +} diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs new file mode 100644 index 00000000000..8ee375fa9e3 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -0,0 +1,410 @@ +use crate::traits::*; +use rustc_middle::mir; +use rustc_middle::mir::interpret::ErrorHandled; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout}; +use rustc_middle::ty::{self, Instance, Ty, TypeFoldable, TypeVisitable}; +use rustc_target::abi::call::{FnAbi, PassMode}; + +use std::iter; + +use rustc_index::bit_set::BitSet; +use rustc_index::vec::IndexVec; + +use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo}; +use self::place::PlaceRef; +use rustc_middle::mir::traversal; + +use self::operand::{OperandRef, OperandValue}; + +/// Master context for codegenning from MIR. +pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { + instance: Instance<'tcx>, + + mir: &'tcx mir::Body<'tcx>, + + debug_context: Option>, + + llfn: Bx::Function, + + cx: &'a Bx::CodegenCx, + + fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, + + /// When unwinding is initiated, we have to store this personality + /// value somewhere so that we can load it and re-use it in the + /// resume instruction. The personality is (afaik) some kind of + /// value used for C++ unwinding, which must filter by type: we + /// don't really care about it very much. Anyway, this value + /// contains an alloca into which the personality is stored and + /// then later loaded when generating the DIVERGE_BLOCK. + personality_slot: Option>, + + /// A backend `BasicBlock` for each MIR `BasicBlock`, created lazily + /// as-needed (e.g. RPO reaching it or another block branching to it). + // FIXME(eddyb) rename `llbbs` and other `ll`-prefixed things to use a + // more backend-agnostic prefix such as `cg` (i.e. this would be `cgbbs`). + cached_llbbs: IndexVec>, + + /// The funclet status of each basic block + cleanup_kinds: IndexVec, + + /// When targeting MSVC, this stores the cleanup info for each funclet BB. + /// This is initialized at the same time as the `landing_pads` entry for the + /// funclets' head block, i.e. when needed by an unwind / `cleanup_ret` edge. + funclets: IndexVec>, + + /// This stores the cached landing/cleanup pad block for a given BB. + // FIXME(eddyb) rename this to `eh_pads`. + landing_pads: IndexVec>, + + /// Cached unreachable block + unreachable_block: Option, + + /// Cached double unwind guarding block + double_unwind_guard: Option, + + /// The location where each MIR arg/var/tmp/ret is stored. This is + /// usually an `PlaceRef` representing an alloca, but not always: + /// sometimes we can skip the alloca and just store the value + /// directly using an `OperandRef`, which makes for tighter LLVM + /// IR. The conditions for using an `OperandRef` are as follows: + /// + /// - the type of the local must be judged "immediate" by `is_llvm_immediate` + /// - the operand must never be referenced indirectly + /// - we should not take its address using the `&` operator + /// - nor should it appear in a place path like `tmp.a` + /// - the operand must be defined by an rvalue that can generate immediate + /// values + /// + /// Avoiding allocs can also be important for certain intrinsics, + /// notably `expect`. + locals: IndexVec>, + + /// All `VarDebugInfo` from the MIR body, partitioned by `Local`. + /// This is `None` if no var`#[non_exhaustive]`iable debuginfo/names are needed. + per_local_var_debug_info: + Option>>>, + + /// Caller location propagated if this function has `#[track_caller]`. + caller_location: Option>, +} + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + pub fn monomorphize(&self, value: T) -> T + where + T: Copy + TypeFoldable<'tcx>, + { + debug!("monomorphize: self.instance={:?}", self.instance); + self.instance.subst_mir_and_normalize_erasing_regions( + self.cx.tcx(), + ty::ParamEnv::reveal_all(), + value, + ) + } +} + +enum LocalRef<'tcx, V> { + Place(PlaceRef<'tcx, V>), + /// `UnsizedPlace(p)`: `p` itself is a thin pointer (indirect place). + /// `*p` is the fat pointer that references the actual unsized place. + /// Every time it is initialized, we have to reallocate the place + /// and update the fat pointer. That's the reason why it is indirect. + UnsizedPlace(PlaceRef<'tcx, V>), + Operand(Option>), +} + +impl<'a, 'tcx, V: CodegenObject> LocalRef<'tcx, V> { + fn new_operand>( + bx: &mut Bx, + layout: TyAndLayout<'tcx>, + ) -> LocalRef<'tcx, V> { + if layout.is_zst() { + // Zero-size temporaries aren't always initialized, which + // doesn't matter because they don't contain data, but + // we need something in the operand. + LocalRef::Operand(Some(OperandRef::new_zst(bx, layout))) + } else { + LocalRef::Operand(None) + } + } +} + +/////////////////////////////////////////////////////////////////////////// + +#[instrument(level = "debug", skip(cx))] +pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + cx: &'a Bx::CodegenCx, + instance: Instance<'tcx>, +) { + assert!(!instance.substs.needs_infer()); + + let llfn = cx.get_fn(instance); + + let mir = cx.tcx().instance_mir(instance.def); + + let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); + debug!("fn_abi: {:?}", fn_abi); + + let debug_context = cx.create_function_debug_context(instance, &fn_abi, llfn, &mir); + + let start_llbb = Bx::append_block(cx, llfn, "start"); + let mut bx = Bx::build(cx, start_llbb); + + if mir.basic_blocks().iter().any(|bb| bb.is_cleanup) { + bx.set_personality_fn(cx.eh_personality()); + } + + let cleanup_kinds = analyze::cleanup_kinds(&mir); + let cached_llbbs: IndexVec> = mir + .basic_blocks() + .indices() + .map(|bb| if bb == mir::START_BLOCK { Some(start_llbb) } else { None }) + .collect(); + + let mut fx = FunctionCx { + instance, + mir, + llfn, + fn_abi, + cx, + personality_slot: None, + cached_llbbs, + unreachable_block: None, + double_unwind_guard: None, + cleanup_kinds, + landing_pads: IndexVec::from_elem(None, mir.basic_blocks()), + funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks().len()), + locals: IndexVec::new(), + debug_context, + per_local_var_debug_info: None, + caller_location: None, + }; + + fx.per_local_var_debug_info = fx.compute_per_local_var_debug_info(&mut bx); + + // Evaluate all required consts; codegen later assumes that CTFE will never fail. + let mut all_consts_ok = true; + for const_ in &mir.required_consts { + if let Err(err) = fx.eval_mir_constant(const_) { + all_consts_ok = false; + match err { + // errored or at least linted + ErrorHandled::Reported(_) | ErrorHandled::Linted => {} + ErrorHandled::TooGeneric => { + span_bug!(const_.span, "codgen encountered polymorphic constant: {:?}", err) + } + } + } + } + if !all_consts_ok { + // We leave the IR in some half-built state here, and rely on this code not even being + // submitted to LLVM once an error was raised. + return; + } + + let memory_locals = analyze::non_ssa_locals(&fx); + + // Allocate variable and temp allocas + fx.locals = { + let args = arg_local_refs(&mut bx, &mut fx, &memory_locals); + + let mut allocate_local = |local| { + let decl = &mir.local_decls[local]; + let layout = bx.layout_of(fx.monomorphize(decl.ty)); + assert!(!layout.ty.has_erasable_regions()); + + if local == mir::RETURN_PLACE && fx.fn_abi.ret.is_indirect() { + debug!("alloc: {:?} (return place) -> place", local); + let llretptr = bx.get_param(0); + return LocalRef::Place(PlaceRef::new_sized(llretptr, layout)); + } + + if memory_locals.contains(local) { + debug!("alloc: {:?} -> place", local); + if layout.is_unsized() { + LocalRef::UnsizedPlace(PlaceRef::alloca_unsized_indirect(&mut bx, layout)) + } else { + LocalRef::Place(PlaceRef::alloca(&mut bx, layout)) + } + } else { + debug!("alloc: {:?} -> operand", local); + LocalRef::new_operand(&mut bx, layout) + } + }; + + let retptr = allocate_local(mir::RETURN_PLACE); + iter::once(retptr) + .chain(args.into_iter()) + .chain(mir.vars_and_temps_iter().map(allocate_local)) + .collect() + }; + + // Apply debuginfo to the newly allocated locals. + fx.debug_introduce_locals(&mut bx); + + // Codegen the body of each block using reverse postorder + for (bb, _) in traversal::reverse_postorder(&mir) { + fx.codegen_block(bb); + } +} + +/// Produces, for each argument, a `Value` pointing at the +/// argument's value. As arguments are places, these are always +/// indirect. +fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + fx: &mut FunctionCx<'a, 'tcx, Bx>, + memory_locals: &BitSet, +) -> Vec> { + let mir = fx.mir; + let mut idx = 0; + let mut llarg_idx = fx.fn_abi.ret.is_indirect() as usize; + + let mut num_untupled = None; + + let args = mir + .args_iter() + .enumerate() + .map(|(arg_index, local)| { + let arg_decl = &mir.local_decls[local]; + + if Some(local) == mir.spread_arg { + // This argument (e.g., the last argument in the "rust-call" ABI) + // is a tuple that was spread at the ABI level and now we have + // to reconstruct it into a tuple local variable, from multiple + // individual LLVM function arguments. + + let arg_ty = fx.monomorphize(arg_decl.ty); + let ty::Tuple(tupled_arg_tys) = arg_ty.kind() else { + bug!("spread argument isn't a tuple?!"); + }; + + let place = PlaceRef::alloca(bx, bx.layout_of(arg_ty)); + for i in 0..tupled_arg_tys.len() { + let arg = &fx.fn_abi.args[idx]; + idx += 1; + if arg.pad.is_some() { + llarg_idx += 1; + } + let pr_field = place.project_field(bx, i); + bx.store_fn_arg(arg, &mut llarg_idx, pr_field); + } + assert_eq!( + None, + num_untupled.replace(tupled_arg_tys.len()), + "Replaced existing num_tupled" + ); + + return LocalRef::Place(place); + } + + if fx.fn_abi.c_variadic && arg_index == fx.fn_abi.args.len() { + let arg_ty = fx.monomorphize(arg_decl.ty); + + let va_list = PlaceRef::alloca(bx, bx.layout_of(arg_ty)); + bx.va_start(va_list.llval); + + return LocalRef::Place(va_list); + } + + let arg = &fx.fn_abi.args[idx]; + idx += 1; + if arg.pad.is_some() { + llarg_idx += 1; + } + + if !memory_locals.contains(local) { + // We don't have to cast or keep the argument in the alloca. + // FIXME(eddyb): We should figure out how to use llvm.dbg.value instead + // of putting everything in allocas just so we can use llvm.dbg.declare. + let local = |op| LocalRef::Operand(Some(op)); + match arg.mode { + PassMode::Ignore => { + return local(OperandRef::new_zst(bx, arg.layout)); + } + PassMode::Direct(_) => { + let llarg = bx.get_param(llarg_idx); + llarg_idx += 1; + return local(OperandRef::from_immediate_or_packed_pair( + bx, llarg, arg.layout, + )); + } + PassMode::Pair(..) => { + let (a, b) = (bx.get_param(llarg_idx), bx.get_param(llarg_idx + 1)); + llarg_idx += 2; + + return local(OperandRef { + val: OperandValue::Pair(a, b), + layout: arg.layout, + }); + } + _ => {} + } + } + + if arg.is_sized_indirect() { + // Don't copy an indirect argument to an alloca, the caller + // already put it in a temporary alloca and gave it up. + // FIXME: lifetimes + let llarg = bx.get_param(llarg_idx); + llarg_idx += 1; + LocalRef::Place(PlaceRef::new_sized(llarg, arg.layout)) + } else if arg.is_unsized_indirect() { + // As the storage for the indirect argument lives during + // the whole function call, we just copy the fat pointer. + let llarg = bx.get_param(llarg_idx); + llarg_idx += 1; + let llextra = bx.get_param(llarg_idx); + llarg_idx += 1; + let indirect_operand = OperandValue::Pair(llarg, llextra); + + let tmp = PlaceRef::alloca_unsized_indirect(bx, arg.layout); + indirect_operand.store(bx, tmp); + LocalRef::UnsizedPlace(tmp) + } else { + let tmp = PlaceRef::alloca(bx, arg.layout); + bx.store_fn_arg(arg, &mut llarg_idx, tmp); + LocalRef::Place(tmp) + } + }) + .collect::>(); + + if fx.instance.def.requires_caller_location(bx.tcx()) { + let mir_args = if let Some(num_untupled) = num_untupled { + // Subtract off the tupled argument that gets 'expanded' + args.len() - 1 + num_untupled + } else { + args.len() + }; + assert_eq!( + fx.fn_abi.args.len(), + mir_args + 1, + "#[track_caller] instance {:?} must have 1 more argument in their ABI than in their MIR", + fx.instance + ); + + let arg = fx.fn_abi.args.last().unwrap(); + match arg.mode { + PassMode::Direct(_) => (), + _ => bug!("caller location must be PassMode::Direct, found {:?}", arg.mode), + } + + fx.caller_location = Some(OperandRef { + val: OperandValue::Immediate(bx.get_param(llarg_idx)), + layout: arg.layout, + }); + } + + args +} + +mod analyze; +mod block; +pub mod constant; +pub mod coverageinfo; +pub mod debuginfo; +mod intrinsic; +pub mod operand; +pub mod place; +mod rvalue; +mod statement; diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs new file mode 100644 index 00000000000..c612634fce2 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -0,0 +1,461 @@ +use super::place::PlaceRef; +use super::{FunctionCx, LocalRef}; + +use crate::base; +use crate::glue; +use crate::traits::*; +use crate::MemFlags; + +use rustc_middle::mir; +use rustc_middle::mir::interpret::{ConstValue, Pointer, Scalar}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::Ty; +use rustc_target::abi::{Abi, Align, Size}; + +use std::fmt; + +/// The representation of a Rust value. The enum variant is in fact +/// uniquely determined by the value's type, but is kept as a +/// safety check. +#[derive(Copy, Clone, Debug)] +pub enum OperandValue { + /// A reference to the actual operand. The data is guaranteed + /// to be valid for the operand's lifetime. + /// The second value, if any, is the extra data (vtable or length) + /// which indicates that it refers to an unsized rvalue. + Ref(V, Option, Align), + /// A single LLVM value. + Immediate(V), + /// A pair of immediate LLVM values. Used by fat pointers too. + Pair(V, V), +} + +/// An `OperandRef` is an "SSA" reference to a Rust value, along with +/// its type. +/// +/// NOTE: unless you know a value's type exactly, you should not +/// generate LLVM opcodes acting on it and instead act via methods, +/// to avoid nasty edge cases. In particular, using `Builder::store` +/// directly is sure to cause problems -- use `OperandRef::store` +/// instead. +#[derive(Copy, Clone)] +pub struct OperandRef<'tcx, V> { + // The value. + pub val: OperandValue, + + // The layout of value, based on its Rust type. + pub layout: TyAndLayout<'tcx>, +} + +impl fmt::Debug for OperandRef<'_, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout) + } +} + +impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { + pub fn new_zst>( + bx: &mut Bx, + layout: TyAndLayout<'tcx>, + ) -> OperandRef<'tcx, V> { + assert!(layout.is_zst()); + OperandRef { + val: OperandValue::Immediate(bx.const_undef(bx.immediate_backend_type(layout))), + layout, + } + } + + pub fn from_const>( + bx: &mut Bx, + val: ConstValue<'tcx>, + ty: Ty<'tcx>, + ) -> Self { + let layout = bx.layout_of(ty); + + if layout.is_zst() { + return OperandRef::new_zst(bx, layout); + } + + let val = match val { + ConstValue::Scalar(x) => { + let Abi::Scalar(scalar) = layout.abi else { + bug!("from_const: invalid ByVal layout: {:#?}", layout); + }; + let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout)); + OperandValue::Immediate(llval) + } + ConstValue::ZeroSized => { + let llval = bx.zst_to_backend(bx.immediate_backend_type(layout)); + OperandValue::Immediate(llval) + } + ConstValue::Slice { data, start, end } => { + let Abi::ScalarPair(a_scalar, _) = layout.abi else { + bug!("from_const: invalid ScalarPair layout: {:#?}", layout); + }; + let a = Scalar::from_pointer( + Pointer::new(bx.tcx().create_memory_alloc(data), Size::from_bytes(start)), + &bx.tcx(), + ); + let a_llval = bx.scalar_to_backend( + a, + a_scalar, + bx.scalar_pair_element_backend_type(layout, 0, true), + ); + let b_llval = bx.const_usize((end - start) as u64); + OperandValue::Pair(a_llval, b_llval) + } + ConstValue::ByRef { alloc, offset } => { + return bx.load_operand(bx.from_const_alloc(layout, alloc, offset)); + } + }; + + OperandRef { val, layout } + } + + /// Asserts that this operand refers to a scalar and returns + /// a reference to its value. + pub fn immediate(self) -> V { + match self.val { + OperandValue::Immediate(s) => s, + _ => bug!("not immediate: {:?}", self), + } + } + + pub fn deref>(self, cx: &Cx) -> PlaceRef<'tcx, V> { + if self.layout.ty.is_box() { + bug!("dereferencing {:?} in codegen", self.layout.ty); + } + + let projected_ty = self + .layout + .ty + .builtin_deref(true) + .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self)) + .ty; + + let (llptr, llextra) = match self.val { + OperandValue::Immediate(llptr) => (llptr, None), + OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)), + OperandValue::Ref(..) => bug!("Deref of by-Ref operand {:?}", self), + }; + let layout = cx.layout_of(projected_ty); + PlaceRef { llval: llptr, llextra, layout, align: layout.align.abi } + } + + /// If this operand is a `Pair`, we return an aggregate with the two values. + /// For other cases, see `immediate`. + pub fn immediate_or_packed_pair>( + self, + bx: &mut Bx, + ) -> V { + if let OperandValue::Pair(a, b) = self.val { + let llty = bx.cx().backend_type(self.layout); + debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty); + // Reconstruct the immediate aggregate. + let mut llpair = bx.cx().const_undef(llty); + let imm_a = bx.from_immediate(a); + let imm_b = bx.from_immediate(b); + llpair = bx.insert_value(llpair, imm_a, 0); + llpair = bx.insert_value(llpair, imm_b, 1); + llpair + } else { + self.immediate() + } + } + + /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`. + pub fn from_immediate_or_packed_pair>( + bx: &mut Bx, + llval: V, + layout: TyAndLayout<'tcx>, + ) -> Self { + let val = if let Abi::ScalarPair(a, b) = layout.abi { + debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval, layout); + + // Deconstruct the immediate aggregate. + let a_llval = bx.extract_value(llval, 0); + let a_llval = bx.to_immediate_scalar(a_llval, a); + let b_llval = bx.extract_value(llval, 1); + let b_llval = bx.to_immediate_scalar(b_llval, b); + OperandValue::Pair(a_llval, b_llval) + } else { + OperandValue::Immediate(llval) + }; + OperandRef { val, layout } + } + + pub fn extract_field>( + &self, + bx: &mut Bx, + i: usize, + ) -> Self { + let field = self.layout.field(bx.cx(), i); + let offset = self.layout.fields.offset(i); + + let mut val = match (self.val, self.layout.abi) { + // If the field is ZST, it has no data. + _ if field.is_zst() => { + return OperandRef::new_zst(bx, field); + } + + // Newtype of a scalar, scalar pair or vector. + (OperandValue::Immediate(_) | OperandValue::Pair(..), _) + if field.size == self.layout.size => + { + assert_eq!(offset.bytes(), 0); + self.val + } + + // Extract a scalar component from a pair. + (OperandValue::Pair(a_llval, b_llval), Abi::ScalarPair(a, b)) => { + if offset.bytes() == 0 { + assert_eq!(field.size, a.size(bx.cx())); + OperandValue::Immediate(a_llval) + } else { + assert_eq!(offset, a.size(bx.cx()).align_to(b.align(bx.cx()).abi)); + assert_eq!(field.size, b.size(bx.cx())); + OperandValue::Immediate(b_llval) + } + } + + // `#[repr(simd)]` types are also immediate. + (OperandValue::Immediate(llval), Abi::Vector { .. }) => { + OperandValue::Immediate(bx.extract_element(llval, bx.cx().const_usize(i as u64))) + } + + _ => bug!("OperandRef::extract_field({:?}): not applicable", self), + }; + + match (&mut val, field.abi) { + (OperandValue::Immediate(llval), _) => { + // Bools in union fields needs to be truncated. + *llval = bx.to_immediate(*llval, field); + // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. + *llval = bx.bitcast(*llval, bx.cx().immediate_backend_type(field)); + } + (OperandValue::Pair(a, b), Abi::ScalarPair(a_abi, b_abi)) => { + // Bools in union fields needs to be truncated. + *a = bx.to_immediate_scalar(*a, a_abi); + *b = bx.to_immediate_scalar(*b, b_abi); + // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. + *a = bx.bitcast(*a, bx.cx().scalar_pair_element_backend_type(field, 0, true)); + *b = bx.bitcast(*b, bx.cx().scalar_pair_element_backend_type(field, 1, true)); + } + (OperandValue::Pair(..), _) => bug!(), + (OperandValue::Ref(..), _) => bug!(), + } + + OperandRef { val, layout: field } + } +} + +impl<'a, 'tcx, V: CodegenObject> OperandValue { + pub fn store>( + self, + bx: &mut Bx, + dest: PlaceRef<'tcx, V>, + ) { + self.store_with_flags(bx, dest, MemFlags::empty()); + } + + pub fn volatile_store>( + self, + bx: &mut Bx, + dest: PlaceRef<'tcx, V>, + ) { + self.store_with_flags(bx, dest, MemFlags::VOLATILE); + } + + pub fn unaligned_volatile_store>( + self, + bx: &mut Bx, + dest: PlaceRef<'tcx, V>, + ) { + self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED); + } + + pub fn nontemporal_store>( + self, + bx: &mut Bx, + dest: PlaceRef<'tcx, V>, + ) { + self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL); + } + + fn store_with_flags>( + self, + bx: &mut Bx, + dest: PlaceRef<'tcx, V>, + flags: MemFlags, + ) { + debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest); + // Avoid generating stores of zero-sized values, because the only way to have a zero-sized + // value is through `undef`, and store itself is useless. + if dest.layout.is_zst() { + return; + } + match self { + OperandValue::Ref(r, None, source_align) => { + if flags.contains(MemFlags::NONTEMPORAL) { + // HACK(nox): This is inefficient but there is no nontemporal memcpy. + let ty = bx.backend_type(dest.layout); + let ptr = bx.pointercast(r, bx.type_ptr_to(ty)); + let val = bx.load(ty, ptr, source_align); + bx.store_with_flags(val, dest.llval, dest.align, flags); + return; + } + base::memcpy_ty(bx, dest.llval, dest.align, r, source_align, dest.layout, flags) + } + OperandValue::Ref(_, Some(_), _) => { + bug!("cannot directly store unsized values"); + } + OperandValue::Immediate(s) => { + let val = bx.from_immediate(s); + bx.store_with_flags(val, dest.llval, dest.align, flags); + } + OperandValue::Pair(a, b) => { + let Abi::ScalarPair(a_scalar, b_scalar) = dest.layout.abi else { + bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout); + }; + let ty = bx.backend_type(dest.layout); + let b_offset = a_scalar.size(bx).align_to(b_scalar.align(bx).abi); + + let llptr = bx.struct_gep(ty, dest.llval, 0); + let val = bx.from_immediate(a); + let align = dest.align; + bx.store_with_flags(val, llptr, align, flags); + + let llptr = bx.struct_gep(ty, dest.llval, 1); + let val = bx.from_immediate(b); + let align = dest.align.restrict_for_offset(b_offset); + bx.store_with_flags(val, llptr, align, flags); + } + } + } + + pub fn store_unsized>( + self, + bx: &mut Bx, + indirect_dest: PlaceRef<'tcx, V>, + ) { + debug!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest); + let flags = MemFlags::empty(); + + // `indirect_dest` must have `*mut T` type. We extract `T` out of it. + let unsized_ty = indirect_dest + .layout + .ty + .builtin_deref(true) + .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest)) + .ty; + + let OperandValue::Ref(llptr, Some(llextra), _) = self else { + bug!("store_unsized called with a sized value") + }; + + // FIXME: choose an appropriate alignment, or use dynamic align somehow + let max_align = Align::from_bits(128).unwrap(); + let min_align = Align::from_bits(8).unwrap(); + + // Allocate an appropriate region on the stack, and copy the value into it + let (llsize, _) = glue::size_and_align_of_dst(bx, unsized_ty, Some(llextra)); + let lldst = bx.array_alloca(bx.cx().type_i8(), llsize, max_align); + bx.memcpy(lldst, max_align, llptr, min_align, llsize, flags); + + // Store the allocated region and the extra to the indirect place. + let indirect_operand = OperandValue::Pair(lldst, llextra); + indirect_operand.store(bx, indirect_dest); + } +} + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + fn maybe_codegen_consume_direct( + &mut self, + bx: &mut Bx, + place_ref: mir::PlaceRef<'tcx>, + ) -> Option> { + debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref); + + match self.locals[place_ref.local] { + LocalRef::Operand(Some(mut o)) => { + // Moves out of scalar and scalar pair fields are trivial. + for elem in place_ref.projection.iter() { + match elem { + mir::ProjectionElem::Field(ref f, _) => { + o = o.extract_field(bx, f.index()); + } + mir::ProjectionElem::Index(_) + | mir::ProjectionElem::ConstantIndex { .. } => { + // ZSTs don't require any actual memory access. + // FIXME(eddyb) deduplicate this with the identical + // checks in `codegen_consume` and `extract_field`. + let elem = o.layout.field(bx.cx(), 0); + if elem.is_zst() { + o = OperandRef::new_zst(bx, elem); + } else { + return None; + } + } + _ => return None, + } + } + + Some(o) + } + LocalRef::Operand(None) => { + bug!("use of {:?} before def", place_ref); + } + LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => { + // watch out for locals that do not have an + // alloca; they are handled somewhat differently + None + } + } + } + + pub fn codegen_consume( + &mut self, + bx: &mut Bx, + place_ref: mir::PlaceRef<'tcx>, + ) -> OperandRef<'tcx, Bx::Value> { + debug!("codegen_consume(place_ref={:?})", place_ref); + + let ty = self.monomorphized_place_ty(place_ref); + let layout = bx.cx().layout_of(ty); + + // ZSTs don't require any actual memory access. + if layout.is_zst() { + return OperandRef::new_zst(bx, layout); + } + + if let Some(o) = self.maybe_codegen_consume_direct(bx, place_ref) { + return o; + } + + // for most places, to consume them we just load them + // out from their home + let place = self.codegen_place(bx, place_ref); + bx.load_operand(place) + } + + pub fn codegen_operand( + &mut self, + bx: &mut Bx, + operand: &mir::Operand<'tcx>, + ) -> OperandRef<'tcx, Bx::Value> { + debug!("codegen_operand(operand={:?})", operand); + + match *operand { + mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => { + self.codegen_consume(bx, place.as_ref()) + } + + mir::Operand::Constant(ref constant) => { + // This cannot fail because we checked all required_consts in advance. + self.eval_mir_constant_to_operand(bx, constant).unwrap_or_else(|_err| { + span_bug!(constant.span, "erroneous constant not captured by required_consts") + }) + } + } + } +} diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs new file mode 100644 index 00000000000..268c4d76503 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -0,0 +1,549 @@ +use super::operand::OperandValue; +use super::{FunctionCx, LocalRef}; + +use crate::common::IntPredicate; +use crate::glue; +use crate::traits::*; +use crate::MemFlags; + +use rustc_middle::mir; +use rustc_middle::mir::tcx::PlaceTy; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; +use rustc_middle::ty::{self, Ty}; +use rustc_target::abi::{Abi, Align, FieldsShape, Int, TagEncoding}; +use rustc_target::abi::{VariantIdx, Variants}; + +#[derive(Copy, Clone, Debug)] +pub struct PlaceRef<'tcx, V> { + /// A pointer to the contents of the place. + pub llval: V, + + /// This place's extra data if it is unsized, or `None` if null. + pub llextra: Option, + + /// The monomorphized type of this place, including variant information. + pub layout: TyAndLayout<'tcx>, + + /// The alignment we know for this place. + pub align: Align, +} + +impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { + pub fn new_sized(llval: V, layout: TyAndLayout<'tcx>) -> PlaceRef<'tcx, V> { + assert!(!layout.is_unsized()); + PlaceRef { llval, llextra: None, layout, align: layout.align.abi } + } + + pub fn new_sized_aligned( + llval: V, + layout: TyAndLayout<'tcx>, + align: Align, + ) -> PlaceRef<'tcx, V> { + assert!(!layout.is_unsized()); + PlaceRef { llval, llextra: None, layout, align } + } + + // FIXME(eddyb) pass something else for the name so no work is done + // unless LLVM IR names are turned on (e.g. for `--emit=llvm-ir`). + pub fn alloca>( + bx: &mut Bx, + layout: TyAndLayout<'tcx>, + ) -> Self { + assert!(!layout.is_unsized(), "tried to statically allocate unsized place"); + let tmp = bx.alloca(bx.cx().backend_type(layout), layout.align.abi); + Self::new_sized(tmp, layout) + } + + /// Returns a place for an indirect reference to an unsized place. + // FIXME(eddyb) pass something else for the name so no work is done + // unless LLVM IR names are turned on (e.g. for `--emit=llvm-ir`). + pub fn alloca_unsized_indirect>( + bx: &mut Bx, + layout: TyAndLayout<'tcx>, + ) -> Self { + assert!(layout.is_unsized(), "tried to allocate indirect place for sized values"); + let ptr_ty = bx.cx().tcx().mk_mut_ptr(layout.ty); + let ptr_layout = bx.cx().layout_of(ptr_ty); + Self::alloca(bx, ptr_layout) + } + + pub fn len>(&self, cx: &Cx) -> V { + if let FieldsShape::Array { count, .. } = self.layout.fields { + if self.layout.is_unsized() { + assert_eq!(count, 0); + self.llextra.unwrap() + } else { + cx.const_usize(count) + } + } else { + bug!("unexpected layout `{:#?}` in PlaceRef::len", self.layout) + } + } +} + +impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { + /// Access a field, at a point when the value's case is known. + pub fn project_field>( + self, + bx: &mut Bx, + ix: usize, + ) -> Self { + let field = self.layout.field(bx.cx(), ix); + let offset = self.layout.fields.offset(ix); + let effective_field_align = self.align.restrict_for_offset(offset); + + let mut simple = || { + let llval = match self.layout.abi { + _ if offset.bytes() == 0 => { + // Unions and newtypes only use an offset of 0. + // Also handles the first field of Scalar, ScalarPair, and Vector layouts. + self.llval + } + Abi::ScalarPair(a, b) + if offset == a.size(bx.cx()).align_to(b.align(bx.cx()).abi) => + { + // Offset matches second field. + let ty = bx.backend_type(self.layout); + bx.struct_gep(ty, self.llval, 1) + } + Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } if field.is_zst() => { + // ZST fields are not included in Scalar, ScalarPair, and Vector layouts, so manually offset the pointer. + let byte_ptr = bx.pointercast(self.llval, bx.cx().type_i8p()); + bx.gep(bx.cx().type_i8(), byte_ptr, &[bx.const_usize(offset.bytes())]) + } + Abi::Scalar(_) | Abi::ScalarPair(..) => { + // All fields of Scalar and ScalarPair layouts must have been handled by this point. + // Vector layouts have additional fields for each element of the vector, so don't panic in that case. + bug!( + "offset of non-ZST field `{:?}` does not match layout `{:#?}`", + field, + self.layout + ); + } + _ => { + let ty = bx.backend_type(self.layout); + bx.struct_gep(ty, self.llval, bx.cx().backend_field_index(self.layout, ix)) + } + }; + PlaceRef { + // HACK(eddyb): have to bitcast pointers until LLVM removes pointee types. + llval: bx.pointercast(llval, bx.cx().type_ptr_to(bx.cx().backend_type(field))), + llextra: if bx.cx().type_has_metadata(field.ty) { self.llextra } else { None }, + layout: field, + align: effective_field_align, + } + }; + + // Simple cases, which don't need DST adjustment: + // * no metadata available - just log the case + // * known alignment - sized types, `[T]`, `str` or a foreign type + // * packed struct - there is no alignment padding + match field.ty.kind() { + _ if self.llextra.is_none() => { + debug!( + "unsized field `{}`, of `{:?}` has no metadata for adjustment", + ix, self.llval + ); + return simple(); + } + _ if !field.is_unsized() => return simple(), + ty::Slice(..) | ty::Str | ty::Foreign(..) => return simple(), + ty::Adt(def, _) => { + if def.repr().packed() { + // FIXME(eddyb) generalize the adjustment when we + // start supporting packing to larger alignments. + assert_eq!(self.layout.align.abi.bytes(), 1); + return simple(); + } + } + _ => {} + } + + // We need to get the pointer manually now. + // We do this by casting to a `*i8`, then offsetting it by the appropriate amount. + // We do this instead of, say, simply adjusting the pointer from the result of a GEP + // because the field may have an arbitrary alignment in the LLVM representation + // anyway. + // + // To demonstrate: + // + // struct Foo { + // x: u16, + // y: T + // } + // + // The type `Foo>` is represented in LLVM as `{ u16, { u16, u8 }}`, meaning that + // the `y` field has 16-bit alignment. + + let meta = self.llextra; + + let unaligned_offset = bx.cx().const_usize(offset.bytes()); + + // Get the alignment of the field + let (_, unsized_align) = glue::size_and_align_of_dst(bx, field.ty, meta); + + // Bump the unaligned offset up to the appropriate alignment + let offset = round_up_const_value_to_alignment(bx, unaligned_offset, unsized_align); + + debug!("struct_field_ptr: DST field offset: {:?}", offset); + + // Cast and adjust pointer. + let byte_ptr = bx.pointercast(self.llval, bx.cx().type_i8p()); + let byte_ptr = bx.gep(bx.cx().type_i8(), byte_ptr, &[offset]); + + // Finally, cast back to the type expected. + let ll_fty = bx.cx().backend_type(field); + debug!("struct_field_ptr: Field type is {:?}", ll_fty); + + PlaceRef { + llval: bx.pointercast(byte_ptr, bx.cx().type_ptr_to(ll_fty)), + llextra: self.llextra, + layout: field, + align: effective_field_align, + } + } + + /// Obtain the actual discriminant of a value. + #[instrument(level = "trace", skip(bx))] + pub fn codegen_get_discr>( + self, + bx: &mut Bx, + cast_to: Ty<'tcx>, + ) -> V { + let cast_to = bx.cx().immediate_backend_type(bx.cx().layout_of(cast_to)); + if self.layout.abi.is_uninhabited() { + return bx.cx().const_undef(cast_to); + } + let (tag_scalar, tag_encoding, tag_field) = match self.layout.variants { + Variants::Single { index } => { + let discr_val = self + .layout + .ty + .discriminant_for_variant(bx.cx().tcx(), index) + .map_or(index.as_u32() as u128, |discr| discr.val); + return bx.cx().const_uint_big(cast_to, discr_val); + } + Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => { + (tag, tag_encoding, tag_field) + } + }; + + // Read the tag/niche-encoded discriminant from memory. + let tag = self.project_field(bx, tag_field); + let tag = bx.load_operand(tag); + + // Decode the discriminant (specifically if it's niche-encoded). + match *tag_encoding { + TagEncoding::Direct => { + let signed = match tag_scalar.primitive() { + // We use `i1` for bytes that are always `0` or `1`, + // e.g., `#[repr(i8)] enum E { A, B }`, but we can't + // let LLVM interpret the `i1` as signed, because + // then `i1 1` (i.e., `E::B`) is effectively `i8 -1`. + Int(_, signed) => !tag_scalar.is_bool() && signed, + _ => false, + }; + bx.intcast(tag.immediate(), cast_to, signed) + } + TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => { + // Rebase from niche values to discriminants, and check + // whether the result is in range for the niche variants. + let niche_llty = bx.cx().immediate_backend_type(tag.layout); + let tag = tag.immediate(); + + // We first compute the "relative discriminant" (wrt `niche_variants`), + // that is, if `n = niche_variants.end() - niche_variants.start()`, + // we remap `niche_start..=niche_start + n` (which may wrap around) + // to (non-wrap-around) `0..=n`, to be able to check whether the + // discriminant corresponds to a niche variant with one comparison. + // We also can't go directly to the (variant index) discriminant + // and check that it is in the range `niche_variants`, because + // that might not fit in the same type, on top of needing an extra + // comparison (see also the comment on `let niche_discr`). + let relative_discr = if niche_start == 0 { + // Avoid subtracting `0`, which wouldn't work for pointers. + // FIXME(eddyb) check the actual primitive type here. + tag + } else { + bx.sub(tag, bx.cx().const_uint_big(niche_llty, niche_start)) + }; + let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32(); + let is_niche = if relative_max == 0 { + // Avoid calling `const_uint`, which wouldn't work for pointers. + // Also use canonical == 0 instead of non-canonical u<= 0. + // FIXME(eddyb) check the actual primitive type here. + bx.icmp(IntPredicate::IntEQ, relative_discr, bx.cx().const_null(niche_llty)) + } else { + let relative_max = bx.cx().const_uint(niche_llty, relative_max as u64); + bx.icmp(IntPredicate::IntULE, relative_discr, relative_max) + }; + + // NOTE(eddyb) this addition needs to be performed on the final + // type, in case the niche itself can't represent all variant + // indices (e.g. `u8` niche with more than `256` variants, + // but enough uninhabited variants so that the remaining variants + // fit in the niche). + // In other words, `niche_variants.end - niche_variants.start` + // is representable in the niche, but `niche_variants.end` + // might not be, in extreme cases. + let niche_discr = { + let relative_discr = if relative_max == 0 { + // HACK(eddyb) since we have only one niche, we know which + // one it is, and we can avoid having a dynamic value here. + bx.cx().const_uint(cast_to, 0) + } else { + bx.intcast(relative_discr, cast_to, false) + }; + bx.add( + relative_discr, + bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64), + ) + }; + + bx.select( + is_niche, + niche_discr, + bx.cx().const_uint(cast_to, dataful_variant.as_u32() as u64), + ) + } + } + } + + /// Sets the discriminant for a new value of the given case of the given + /// representation. + pub fn codegen_set_discr>( + &self, + bx: &mut Bx, + variant_index: VariantIdx, + ) { + if self.layout.for_variant(bx.cx(), variant_index).abi.is_uninhabited() { + // We play it safe by using a well-defined `abort`, but we could go for immediate UB + // if that turns out to be helpful. + bx.abort(); + return; + } + match self.layout.variants { + Variants::Single { index } => { + assert_eq!(index, variant_index); + } + Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } => { + let ptr = self.project_field(bx, tag_field); + let to = + self.layout.ty.discriminant_for_variant(bx.tcx(), variant_index).unwrap().val; + bx.store( + bx.cx().const_uint_big(bx.cx().backend_type(ptr.layout), to), + ptr.llval, + ptr.align, + ); + } + Variants::Multiple { + tag_encoding: + TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start }, + tag_field, + .. + } => { + if variant_index != dataful_variant { + if bx.cx().sess().target.arch == "arm" + || bx.cx().sess().target.arch == "aarch64" + { + // FIXME(#34427): as workaround for LLVM bug on ARM, + // use memset of 0 before assigning niche value. + let fill_byte = bx.cx().const_u8(0); + let size = bx.cx().const_usize(self.layout.size.bytes()); + bx.memset(self.llval, fill_byte, size, self.align, MemFlags::empty()); + } + + let niche = self.project_field(bx, tag_field); + let niche_llty = bx.cx().immediate_backend_type(niche.layout); + let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); + let niche_value = (niche_value as u128).wrapping_add(niche_start); + // FIXME(eddyb): check the actual primitive type here. + let niche_llval = if niche_value == 0 { + // HACK(eddyb): using `c_null` as it works on all types. + bx.cx().const_null(niche_llty) + } else { + bx.cx().const_uint_big(niche_llty, niche_value) + }; + OperandValue::Immediate(niche_llval).store(bx, niche); + } + } + } + } + + pub fn project_index>( + &self, + bx: &mut Bx, + llindex: V, + ) -> Self { + // Statically compute the offset if we can, otherwise just use the element size, + // as this will yield the lowest alignment. + let layout = self.layout.field(bx, 0); + let offset = if let Some(llindex) = bx.const_to_opt_uint(llindex) { + layout.size.checked_mul(llindex, bx).unwrap_or(layout.size) + } else { + layout.size + }; + + PlaceRef { + llval: bx.inbounds_gep( + bx.cx().backend_type(self.layout), + self.llval, + &[bx.cx().const_usize(0), llindex], + ), + llextra: None, + layout, + align: self.align.restrict_for_offset(offset), + } + } + + pub fn project_downcast>( + &self, + bx: &mut Bx, + variant_index: VariantIdx, + ) -> Self { + let mut downcast = *self; + downcast.layout = self.layout.for_variant(bx.cx(), variant_index); + + // Cast to the appropriate variant struct type. + let variant_ty = bx.cx().backend_type(downcast.layout); + downcast.llval = bx.pointercast(downcast.llval, bx.cx().type_ptr_to(variant_ty)); + + downcast + } + + pub fn storage_live>(&self, bx: &mut Bx) { + bx.lifetime_start(self.llval, self.layout.size); + } + + pub fn storage_dead>(&self, bx: &mut Bx) { + bx.lifetime_end(self.llval, self.layout.size); + } +} + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + #[instrument(level = "trace", skip(self, bx))] + pub fn codegen_place( + &mut self, + bx: &mut Bx, + place_ref: mir::PlaceRef<'tcx>, + ) -> PlaceRef<'tcx, Bx::Value> { + let cx = self.cx; + let tcx = self.cx.tcx(); + + let mut base = 0; + let mut cg_base = match self.locals[place_ref.local] { + LocalRef::Place(place) => place, + LocalRef::UnsizedPlace(place) => bx.load_operand(place).deref(cx), + LocalRef::Operand(..) => { + if place_ref.has_deref() { + base = 1; + let cg_base = self.codegen_consume( + bx, + mir::PlaceRef { projection: &place_ref.projection[..0], ..place_ref }, + ); + cg_base.deref(bx.cx()) + } else { + bug!("using operand local {:?} as place", place_ref); + } + } + }; + for elem in place_ref.projection[base..].iter() { + cg_base = match *elem { + mir::ProjectionElem::Deref => bx.load_operand(cg_base).deref(bx.cx()), + mir::ProjectionElem::Field(ref field, _) => { + cg_base.project_field(bx, field.index()) + } + mir::ProjectionElem::Index(index) => { + let index = &mir::Operand::Copy(mir::Place::from(index)); + let index = self.codegen_operand(bx, index); + let llindex = index.immediate(); + cg_base.project_index(bx, llindex) + } + mir::ProjectionElem::ConstantIndex { offset, from_end: false, min_length: _ } => { + let lloffset = bx.cx().const_usize(offset as u64); + cg_base.project_index(bx, lloffset) + } + mir::ProjectionElem::ConstantIndex { offset, from_end: true, min_length: _ } => { + let lloffset = bx.cx().const_usize(offset as u64); + let lllen = cg_base.len(bx.cx()); + let llindex = bx.sub(lllen, lloffset); + cg_base.project_index(bx, llindex) + } + mir::ProjectionElem::Subslice { from, to, from_end } => { + let mut subslice = cg_base.project_index(bx, bx.cx().const_usize(from as u64)); + let projected_ty = + PlaceTy::from_ty(cg_base.layout.ty).projection_ty(tcx, *elem).ty; + subslice.layout = bx.cx().layout_of(self.monomorphize(projected_ty)); + + if subslice.layout.is_unsized() { + assert!(from_end, "slice subslices should be `from_end`"); + subslice.llextra = Some(bx.sub( + cg_base.llextra.unwrap(), + bx.cx().const_usize((from as u64) + (to as u64)), + )); + } + + // Cast the place pointer type to the new + // array or slice type (`*[%_; new_len]`). + subslice.llval = bx.pointercast( + subslice.llval, + bx.cx().type_ptr_to(bx.cx().backend_type(subslice.layout)), + ); + + subslice + } + mir::ProjectionElem::Downcast(_, v) => cg_base.project_downcast(bx, v), + }; + } + debug!("codegen_place(place={:?}) => {:?}", place_ref, cg_base); + cg_base + } + + pub fn monomorphized_place_ty(&self, place_ref: mir::PlaceRef<'tcx>) -> Ty<'tcx> { + let tcx = self.cx.tcx(); + let place_ty = place_ref.ty(self.mir, tcx); + self.monomorphize(place_ty.ty) + } +} + +fn round_up_const_value_to_alignment<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + bx: &mut Bx, + value: Bx::Value, + align: Bx::Value, +) -> Bx::Value { + // In pseudo code: + // + // if value & (align - 1) == 0 { + // value + // } else { + // (value & !(align - 1)) + align + // } + // + // Usually this is written without branches as + // + // (value + align - 1) & !(align - 1) + // + // But this formula cannot take advantage of constant `value`. E.g. if `value` is known + // at compile time to be `1`, this expression should be optimized to `align`. However, + // optimization only holds if `align` is a power of two. Since the optimizer doesn't know + // that `align` is a power of two, it cannot perform this optimization. + // + // Instead we use + // + // value + (-value & (align - 1)) + // + // Since `align` is used only once, the expression can be optimized. For `value = 0` + // its optimized to `0` even in debug mode. + // + // NB: The previous version of this code used + // + // (value + align - 1) & -align + // + // Even though `-align == !(align - 1)`, LLVM failed to optimize this even for + // `value = 0`. Bug report: https://bugs.llvm.org/show_bug.cgi?id=48559 + let one = bx.const_usize(1); + let align_minus_1 = bx.sub(align, one); + let neg_value = bx.neg(value); + let offset = bx.and(neg_value, align_minus_1); + bx.add(value, offset) +} diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs new file mode 100644 index 00000000000..26b9fbf4428 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -0,0 +1,729 @@ +use super::operand::{OperandRef, OperandValue}; +use super::place::PlaceRef; +use super::{FunctionCx, LocalRef}; + +use crate::base; +use crate::common::{self, IntPredicate}; +use crate::traits::*; +use crate::MemFlags; + +use rustc_middle::mir; +use rustc_middle::mir::Operand; +use rustc_middle::ty::cast::{CastTy, IntTy}; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt}; +use rustc_span::source_map::{Span, DUMMY_SP}; + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + #[instrument(level = "trace", skip(self, bx))] + pub fn codegen_rvalue( + &mut self, + mut bx: Bx, + dest: PlaceRef<'tcx, Bx::Value>, + rvalue: &mir::Rvalue<'tcx>, + ) -> Bx { + match *rvalue { + mir::Rvalue::Use(ref operand) => { + let cg_operand = self.codegen_operand(&mut bx, operand); + // FIXME: consider not copying constants through stack. (Fixable by codegen'ing + // constants into `OperandValue::Ref`; why don’t we do that yet if we don’t?) + cg_operand.val.store(&mut bx, dest); + bx + } + + mir::Rvalue::Cast(mir::CastKind::Pointer(PointerCast::Unsize), ref source, _) => { + // The destination necessarily contains a fat pointer, so if + // it's a scalar pair, it's a fat pointer or newtype thereof. + if bx.cx().is_backend_scalar_pair(dest.layout) { + // Into-coerce of a thin pointer to a fat pointer -- just + // use the operand path. + let (mut bx, temp) = self.codegen_rvalue_operand(bx, rvalue); + temp.val.store(&mut bx, dest); + return bx; + } + + // Unsize of a nontrivial struct. I would prefer for + // this to be eliminated by MIR building, but + // `CoerceUnsized` can be passed by a where-clause, + // so the (generic) MIR may not be able to expand it. + let operand = self.codegen_operand(&mut bx, source); + match operand.val { + OperandValue::Pair(..) | OperandValue::Immediate(_) => { + // Unsize from an immediate structure. We don't + // really need a temporary alloca here, but + // avoiding it would require us to have + // `coerce_unsized_into` use `extractvalue` to + // index into the struct, and this case isn't + // important enough for it. + debug!("codegen_rvalue: creating ugly alloca"); + let scratch = PlaceRef::alloca(&mut bx, operand.layout); + scratch.storage_live(&mut bx); + operand.val.store(&mut bx, scratch); + base::coerce_unsized_into(&mut bx, scratch, dest); + scratch.storage_dead(&mut bx); + } + OperandValue::Ref(llref, None, align) => { + let source = PlaceRef::new_sized_aligned(llref, operand.layout, align); + base::coerce_unsized_into(&mut bx, source, dest); + } + OperandValue::Ref(_, Some(_), _) => { + bug!("unsized coercion on an unsized rvalue"); + } + } + bx + } + + mir::Rvalue::Repeat(ref elem, count) => { + let cg_elem = self.codegen_operand(&mut bx, elem); + + // Do not generate the loop for zero-sized elements or empty arrays. + if dest.layout.is_zst() { + return bx; + } + + if let OperandValue::Immediate(v) = cg_elem.val { + let zero = bx.const_usize(0); + let start = dest.project_index(&mut bx, zero).llval; + let size = bx.const_usize(dest.layout.size.bytes()); + + // Use llvm.memset.p0i8.* to initialize all zero arrays + if bx.cx().const_to_opt_uint(v) == Some(0) { + let fill = bx.cx().const_u8(0); + bx.memset(start, fill, size, dest.align, MemFlags::empty()); + return bx; + } + + // Use llvm.memset.p0i8.* to initialize byte arrays + let v = bx.from_immediate(v); + if bx.cx().val_ty(v) == bx.cx().type_i8() { + bx.memset(start, v, size, dest.align, MemFlags::empty()); + return bx; + } + } + + let count = + self.monomorphize(count).eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all()); + + bx.write_operand_repeatedly(cg_elem, count, dest) + } + + mir::Rvalue::Aggregate(ref kind, ref operands) => { + let (dest, active_field_index) = match **kind { + mir::AggregateKind::Adt(adt_did, variant_index, _, _, active_field_index) => { + dest.codegen_set_discr(&mut bx, variant_index); + if bx.tcx().adt_def(adt_did).is_enum() { + (dest.project_downcast(&mut bx, variant_index), active_field_index) + } else { + (dest, active_field_index) + } + } + _ => (dest, None), + }; + for (i, operand) in operands.iter().enumerate() { + let op = self.codegen_operand(&mut bx, operand); + // Do not generate stores and GEPis for zero-sized fields. + if !op.layout.is_zst() { + let field_index = active_field_index.unwrap_or(i); + let field = if let mir::AggregateKind::Array(_) = **kind { + let llindex = bx.cx().const_usize(field_index as u64); + dest.project_index(&mut bx, llindex) + } else { + dest.project_field(&mut bx, field_index) + }; + op.val.store(&mut bx, field); + } + } + bx + } + + _ => { + assert!(self.rvalue_creates_operand(rvalue, DUMMY_SP)); + let (mut bx, temp) = self.codegen_rvalue_operand(bx, rvalue); + temp.val.store(&mut bx, dest); + bx + } + } + } + + pub fn codegen_rvalue_unsized( + &mut self, + mut bx: Bx, + indirect_dest: PlaceRef<'tcx, Bx::Value>, + rvalue: &mir::Rvalue<'tcx>, + ) -> Bx { + debug!( + "codegen_rvalue_unsized(indirect_dest.llval={:?}, rvalue={:?})", + indirect_dest.llval, rvalue + ); + + match *rvalue { + mir::Rvalue::Use(ref operand) => { + let cg_operand = self.codegen_operand(&mut bx, operand); + cg_operand.val.store_unsized(&mut bx, indirect_dest); + bx + } + + _ => bug!("unsized assignment other than `Rvalue::Use`"), + } + } + + pub fn codegen_rvalue_operand( + &mut self, + mut bx: Bx, + rvalue: &mir::Rvalue<'tcx>, + ) -> (Bx, OperandRef<'tcx, Bx::Value>) { + assert!( + self.rvalue_creates_operand(rvalue, DUMMY_SP), + "cannot codegen {:?} to operand", + rvalue, + ); + + match *rvalue { + mir::Rvalue::Cast(ref kind, ref source, mir_cast_ty) => { + let operand = self.codegen_operand(&mut bx, source); + debug!("cast operand is {:?}", operand); + let cast = bx.cx().layout_of(self.monomorphize(mir_cast_ty)); + + let val = match *kind { + mir::CastKind::PointerExposeAddress => { + assert!(bx.cx().is_backend_immediate(cast)); + let llptr = operand.immediate(); + let llcast_ty = bx.cx().immediate_backend_type(cast); + let lladdr = bx.ptrtoint(llptr, llcast_ty); + OperandValue::Immediate(lladdr) + } + mir::CastKind::Pointer(PointerCast::ReifyFnPointer) => { + match *operand.layout.ty.kind() { + ty::FnDef(def_id, substs) => { + let instance = ty::Instance::resolve_for_fn_ptr( + bx.tcx(), + ty::ParamEnv::reveal_all(), + def_id, + substs, + ) + .unwrap() + .polymorphize(bx.cx().tcx()); + OperandValue::Immediate(bx.get_fn_addr(instance)) + } + _ => bug!("{} cannot be reified to a fn ptr", operand.layout.ty), + } + } + mir::CastKind::Pointer(PointerCast::ClosureFnPointer(_)) => { + match *operand.layout.ty.kind() { + ty::Closure(def_id, substs) => { + let instance = Instance::resolve_closure( + bx.cx().tcx(), + def_id, + substs, + ty::ClosureKind::FnOnce, + ) + .expect("failed to normalize and resolve closure during codegen") + .polymorphize(bx.cx().tcx()); + OperandValue::Immediate(bx.cx().get_fn_addr(instance)) + } + _ => bug!("{} cannot be cast to a fn ptr", operand.layout.ty), + } + } + mir::CastKind::Pointer(PointerCast::UnsafeFnPointer) => { + // This is a no-op at the LLVM level. + operand.val + } + mir::CastKind::Pointer(PointerCast::Unsize) => { + assert!(bx.cx().is_backend_scalar_pair(cast)); + let (lldata, llextra) = match operand.val { + OperandValue::Pair(lldata, llextra) => { + // unsize from a fat pointer -- this is a + // "trait-object-to-supertrait" coercion. + (lldata, Some(llextra)) + } + OperandValue::Immediate(lldata) => { + // "standard" unsize + (lldata, None) + } + OperandValue::Ref(..) => { + bug!("by-ref operand {:?} in `codegen_rvalue_operand`", operand); + } + }; + let (lldata, llextra) = + base::unsize_ptr(&mut bx, lldata, operand.layout.ty, cast.ty, llextra); + OperandValue::Pair(lldata, llextra) + } + mir::CastKind::Pointer(PointerCast::MutToConstPointer) + | mir::CastKind::Misc + if bx.cx().is_backend_scalar_pair(operand.layout) => + { + if let OperandValue::Pair(data_ptr, meta) = operand.val { + if bx.cx().is_backend_scalar_pair(cast) { + let data_cast = bx.pointercast( + data_ptr, + bx.cx().scalar_pair_element_backend_type(cast, 0, true), + ); + OperandValue::Pair(data_cast, meta) + } else { + // cast to thin-ptr + // Cast of fat-ptr to thin-ptr is an extraction of data-ptr and + // pointer-cast of that pointer to desired pointer type. + let llcast_ty = bx.cx().immediate_backend_type(cast); + let llval = bx.pointercast(data_ptr, llcast_ty); + OperandValue::Immediate(llval) + } + } else { + bug!("unexpected non-pair operand"); + } + } + mir::CastKind::Pointer( + PointerCast::MutToConstPointer | PointerCast::ArrayToPointer, + ) + | mir::CastKind::Misc + // Since int2ptr can have arbitrary integer types as input (so we have to do + // sign extension and all that), it is currently best handled in the same code + // path as the other integer-to-X casts. + | mir::CastKind::PointerFromExposedAddress => { + assert!(bx.cx().is_backend_immediate(cast)); + let ll_t_out = bx.cx().immediate_backend_type(cast); + if operand.layout.abi.is_uninhabited() { + let val = OperandValue::Immediate(bx.cx().const_undef(ll_t_out)); + return (bx, OperandRef { val, layout: cast }); + } + let r_t_in = + CastTy::from_ty(operand.layout.ty).expect("bad input type for cast"); + let r_t_out = CastTy::from_ty(cast.ty).expect("bad output type for cast"); + let ll_t_in = bx.cx().immediate_backend_type(operand.layout); + let llval = operand.immediate(); + + let newval = match (r_t_in, r_t_out) { + (CastTy::Int(i), CastTy::Int(_)) => { + bx.intcast(llval, ll_t_out, i.is_signed()) + } + (CastTy::Float, CastTy::Float) => { + let srcsz = bx.cx().float_width(ll_t_in); + let dstsz = bx.cx().float_width(ll_t_out); + if dstsz > srcsz { + bx.fpext(llval, ll_t_out) + } else if srcsz > dstsz { + bx.fptrunc(llval, ll_t_out) + } else { + llval + } + } + (CastTy::Int(i), CastTy::Float) => { + if i.is_signed() { + bx.sitofp(llval, ll_t_out) + } else { + bx.uitofp(llval, ll_t_out) + } + } + (CastTy::Ptr(_) | CastTy::FnPtr, CastTy::Ptr(_)) => { + bx.pointercast(llval, ll_t_out) + } + (CastTy::Int(i), CastTy::Ptr(_)) => { + let usize_llval = + bx.intcast(llval, bx.cx().type_isize(), i.is_signed()); + bx.inttoptr(usize_llval, ll_t_out) + } + (CastTy::Float, CastTy::Int(IntTy::I)) => { + bx.cast_float_to_int(true, llval, ll_t_out) + } + (CastTy::Float, CastTy::Int(_)) => { + bx.cast_float_to_int(false, llval, ll_t_out) + } + _ => bug!("unsupported cast: {:?} to {:?}", operand.layout.ty, cast.ty), + }; + OperandValue::Immediate(newval) + } + }; + (bx, OperandRef { val, layout: cast }) + } + + mir::Rvalue::Ref(_, bk, place) => { + let mk_ref = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { + tcx.mk_ref( + tcx.lifetimes.re_erased, + ty::TypeAndMut { ty, mutbl: bk.to_mutbl_lossy() }, + ) + }; + self.codegen_place_to_pointer(bx, place, mk_ref) + } + + mir::Rvalue::CopyForDeref(place) => { + let operand = self.codegen_operand(&mut bx, &Operand::Copy(place)); + (bx, operand) + } + mir::Rvalue::AddressOf(mutability, place) => { + let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { + tcx.mk_ptr(ty::TypeAndMut { ty, mutbl: mutability }) + }; + self.codegen_place_to_pointer(bx, place, mk_ptr) + } + + mir::Rvalue::Len(place) => { + let size = self.evaluate_array_len(&mut bx, place); + let operand = OperandRef { + val: OperandValue::Immediate(size), + layout: bx.cx().layout_of(bx.tcx().types.usize), + }; + (bx, operand) + } + + mir::Rvalue::BinaryOp(op, box (ref lhs, ref rhs)) => { + let lhs = self.codegen_operand(&mut bx, lhs); + let rhs = self.codegen_operand(&mut bx, rhs); + let llresult = match (lhs.val, rhs.val) { + ( + OperandValue::Pair(lhs_addr, lhs_extra), + OperandValue::Pair(rhs_addr, rhs_extra), + ) => self.codegen_fat_ptr_binop( + &mut bx, + op, + lhs_addr, + lhs_extra, + rhs_addr, + rhs_extra, + lhs.layout.ty, + ), + + (OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => { + self.codegen_scalar_binop(&mut bx, op, lhs_val, rhs_val, lhs.layout.ty) + } + + _ => bug!(), + }; + let operand = OperandRef { + val: OperandValue::Immediate(llresult), + layout: bx.cx().layout_of(op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty)), + }; + (bx, operand) + } + mir::Rvalue::CheckedBinaryOp(op, box (ref lhs, ref rhs)) => { + let lhs = self.codegen_operand(&mut bx, lhs); + let rhs = self.codegen_operand(&mut bx, rhs); + let result = self.codegen_scalar_checked_binop( + &mut bx, + op, + lhs.immediate(), + rhs.immediate(), + lhs.layout.ty, + ); + let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty); + let operand_ty = bx.tcx().intern_tup(&[val_ty, bx.tcx().types.bool]); + let operand = OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) }; + + (bx, operand) + } + + mir::Rvalue::UnaryOp(op, ref operand) => { + let operand = self.codegen_operand(&mut bx, operand); + let lloperand = operand.immediate(); + let is_float = operand.layout.ty.is_floating_point(); + let llval = match op { + mir::UnOp::Not => bx.not(lloperand), + mir::UnOp::Neg => { + if is_float { + bx.fneg(lloperand) + } else { + bx.neg(lloperand) + } + } + }; + (bx, OperandRef { val: OperandValue::Immediate(llval), layout: operand.layout }) + } + + mir::Rvalue::Discriminant(ref place) => { + let discr_ty = rvalue.ty(self.mir, bx.tcx()); + let discr_ty = self.monomorphize(discr_ty); + let discr = self + .codegen_place(&mut bx, place.as_ref()) + .codegen_get_discr(&mut bx, discr_ty); + ( + bx, + OperandRef { + val: OperandValue::Immediate(discr), + layout: self.cx.layout_of(discr_ty), + }, + ) + } + + mir::Rvalue::NullaryOp(null_op, ty) => { + let ty = self.monomorphize(ty); + assert!(bx.cx().type_is_sized(ty)); + let layout = bx.cx().layout_of(ty); + let val = match null_op { + mir::NullOp::SizeOf => layout.size.bytes(), + mir::NullOp::AlignOf => layout.align.abi.bytes(), + }; + let val = bx.cx().const_usize(val); + let tcx = self.cx.tcx(); + ( + bx, + OperandRef { + val: OperandValue::Immediate(val), + layout: self.cx.layout_of(tcx.types.usize), + }, + ) + } + + mir::Rvalue::ThreadLocalRef(def_id) => { + assert!(bx.cx().tcx().is_static(def_id)); + let static_ = bx.get_static(def_id); + let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id)); + let operand = OperandRef::from_immediate_or_packed_pair(&mut bx, static_, layout); + (bx, operand) + } + mir::Rvalue::Use(ref operand) => { + let operand = self.codegen_operand(&mut bx, operand); + (bx, operand) + } + mir::Rvalue::Repeat(..) | mir::Rvalue::Aggregate(..) => { + // According to `rvalue_creates_operand`, only ZST + // aggregate rvalues are allowed to be operands. + let ty = rvalue.ty(self.mir, self.cx.tcx()); + let operand = + OperandRef::new_zst(&mut bx, self.cx.layout_of(self.monomorphize(ty))); + (bx, operand) + } + mir::Rvalue::ShallowInitBox(ref operand, content_ty) => { + let operand = self.codegen_operand(&mut bx, operand); + let lloperand = operand.immediate(); + + let content_ty = self.monomorphize(content_ty); + let box_layout = bx.cx().layout_of(bx.tcx().mk_box(content_ty)); + let llty_ptr = bx.cx().backend_type(box_layout); + + let val = bx.pointercast(lloperand, llty_ptr); + let operand = OperandRef { val: OperandValue::Immediate(val), layout: box_layout }; + (bx, operand) + } + } + } + + fn evaluate_array_len(&mut self, bx: &mut Bx, place: mir::Place<'tcx>) -> Bx::Value { + // ZST are passed as operands and require special handling + // because codegen_place() panics if Local is operand. + if let Some(index) = place.as_local() { + if let LocalRef::Operand(Some(op)) = self.locals[index] { + if let ty::Array(_, n) = op.layout.ty.kind() { + let n = n.eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all()); + return bx.cx().const_usize(n); + } + } + } + // use common size calculation for non zero-sized types + let cg_value = self.codegen_place(bx, place.as_ref()); + cg_value.len(bx.cx()) + } + + /// Codegen an `Rvalue::AddressOf` or `Rvalue::Ref` + fn codegen_place_to_pointer( + &mut self, + mut bx: Bx, + place: mir::Place<'tcx>, + mk_ptr_ty: impl FnOnce(TyCtxt<'tcx>, Ty<'tcx>) -> Ty<'tcx>, + ) -> (Bx, OperandRef<'tcx, Bx::Value>) { + let cg_place = self.codegen_place(&mut bx, place.as_ref()); + + let ty = cg_place.layout.ty; + + // Note: places are indirect, so storing the `llval` into the + // destination effectively creates a reference. + let val = if !bx.cx().type_has_metadata(ty) { + OperandValue::Immediate(cg_place.llval) + } else { + OperandValue::Pair(cg_place.llval, cg_place.llextra.unwrap()) + }; + (bx, OperandRef { val, layout: self.cx.layout_of(mk_ptr_ty(self.cx.tcx(), ty)) }) + } + + pub fn codegen_scalar_binop( + &mut self, + bx: &mut Bx, + op: mir::BinOp, + lhs: Bx::Value, + rhs: Bx::Value, + input_ty: Ty<'tcx>, + ) -> Bx::Value { + let is_float = input_ty.is_floating_point(); + let is_signed = input_ty.is_signed(); + match op { + mir::BinOp::Add => { + if is_float { + bx.fadd(lhs, rhs) + } else { + bx.add(lhs, rhs) + } + } + mir::BinOp::Sub => { + if is_float { + bx.fsub(lhs, rhs) + } else { + bx.sub(lhs, rhs) + } + } + mir::BinOp::Mul => { + if is_float { + bx.fmul(lhs, rhs) + } else { + bx.mul(lhs, rhs) + } + } + mir::BinOp::Div => { + if is_float { + bx.fdiv(lhs, rhs) + } else if is_signed { + bx.sdiv(lhs, rhs) + } else { + bx.udiv(lhs, rhs) + } + } + mir::BinOp::Rem => { + if is_float { + bx.frem(lhs, rhs) + } else if is_signed { + bx.srem(lhs, rhs) + } else { + bx.urem(lhs, rhs) + } + } + mir::BinOp::BitOr => bx.or(lhs, rhs), + mir::BinOp::BitAnd => bx.and(lhs, rhs), + mir::BinOp::BitXor => bx.xor(lhs, rhs), + mir::BinOp::Offset => { + let pointee_type = input_ty + .builtin_deref(true) + .unwrap_or_else(|| bug!("deref of non-pointer {:?}", input_ty)) + .ty; + let llty = bx.cx().backend_type(bx.cx().layout_of(pointee_type)); + bx.inbounds_gep(llty, lhs, &[rhs]) + } + mir::BinOp::Shl => common::build_unchecked_lshift(bx, lhs, rhs), + mir::BinOp::Shr => common::build_unchecked_rshift(bx, input_ty, lhs, rhs), + mir::BinOp::Ne + | mir::BinOp::Lt + | mir::BinOp::Gt + | mir::BinOp::Eq + | mir::BinOp::Le + | mir::BinOp::Ge => { + if is_float { + bx.fcmp(base::bin_op_to_fcmp_predicate(op.to_hir_binop()), lhs, rhs) + } else { + bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs) + } + } + } + } + + pub fn codegen_fat_ptr_binop( + &mut self, + bx: &mut Bx, + op: mir::BinOp, + lhs_addr: Bx::Value, + lhs_extra: Bx::Value, + rhs_addr: Bx::Value, + rhs_extra: Bx::Value, + _input_ty: Ty<'tcx>, + ) -> Bx::Value { + match op { + mir::BinOp::Eq => { + let lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr); + let rhs = bx.icmp(IntPredicate::IntEQ, lhs_extra, rhs_extra); + bx.and(lhs, rhs) + } + mir::BinOp::Ne => { + let lhs = bx.icmp(IntPredicate::IntNE, lhs_addr, rhs_addr); + let rhs = bx.icmp(IntPredicate::IntNE, lhs_extra, rhs_extra); + bx.or(lhs, rhs) + } + mir::BinOp::Le | mir::BinOp::Lt | mir::BinOp::Ge | mir::BinOp::Gt => { + // a OP b ~ a.0 STRICT(OP) b.0 | (a.0 == b.0 && a.1 OP a.1) + let (op, strict_op) = match op { + mir::BinOp::Lt => (IntPredicate::IntULT, IntPredicate::IntULT), + mir::BinOp::Le => (IntPredicate::IntULE, IntPredicate::IntULT), + mir::BinOp::Gt => (IntPredicate::IntUGT, IntPredicate::IntUGT), + mir::BinOp::Ge => (IntPredicate::IntUGE, IntPredicate::IntUGT), + _ => bug!(), + }; + let lhs = bx.icmp(strict_op, lhs_addr, rhs_addr); + let and_lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr); + let and_rhs = bx.icmp(op, lhs_extra, rhs_extra); + let rhs = bx.and(and_lhs, and_rhs); + bx.or(lhs, rhs) + } + _ => { + bug!("unexpected fat ptr binop"); + } + } + } + + pub fn codegen_scalar_checked_binop( + &mut self, + bx: &mut Bx, + op: mir::BinOp, + lhs: Bx::Value, + rhs: Bx::Value, + input_ty: Ty<'tcx>, + ) -> OperandValue { + // This case can currently arise only from functions marked + // with #[rustc_inherit_overflow_checks] and inlined from + // another crate (mostly core::num generic/#[inline] fns), + // while the current crate doesn't use overflow checks. + if !bx.cx().check_overflow() { + let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty); + return OperandValue::Pair(val, bx.cx().const_bool(false)); + } + + let (val, of) = match op { + // These are checked using intrinsics + mir::BinOp::Add | mir::BinOp::Sub | mir::BinOp::Mul => { + let oop = match op { + mir::BinOp::Add => OverflowOp::Add, + mir::BinOp::Sub => OverflowOp::Sub, + mir::BinOp::Mul => OverflowOp::Mul, + _ => unreachable!(), + }; + bx.checked_binop(oop, input_ty, lhs, rhs) + } + mir::BinOp::Shl | mir::BinOp::Shr => { + let lhs_llty = bx.cx().val_ty(lhs); + let rhs_llty = bx.cx().val_ty(rhs); + let invert_mask = common::shift_mask_val(bx, lhs_llty, rhs_llty, true); + let outer_bits = bx.and(rhs, invert_mask); + + let of = bx.icmp(IntPredicate::IntNE, outer_bits, bx.cx().const_null(rhs_llty)); + let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty); + + (val, of) + } + _ => bug!("Operator `{:?}` is not a checkable operator", op), + }; + + OperandValue::Pair(val, of) + } +} + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool { + match *rvalue { + mir::Rvalue::Ref(..) | + mir::Rvalue::CopyForDeref(..) | + mir::Rvalue::AddressOf(..) | + mir::Rvalue::Len(..) | + mir::Rvalue::Cast(..) | // (*) + mir::Rvalue::ShallowInitBox(..) | // (*) + mir::Rvalue::BinaryOp(..) | + mir::Rvalue::CheckedBinaryOp(..) | + mir::Rvalue::UnaryOp(..) | + mir::Rvalue::Discriminant(..) | + mir::Rvalue::NullaryOp(..) | + mir::Rvalue::ThreadLocalRef(_) | + mir::Rvalue::Use(..) => // (*) + true, + mir::Rvalue::Repeat(..) | + mir::Rvalue::Aggregate(..) => { + let ty = rvalue.ty(self.mir, self.cx.tcx()); + let ty = self.monomorphize(ty); + self.cx.spanned_layout_of(ty, span).is_zst() + } + } + + // (*) this is only true if the type is suitable + } +} diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs new file mode 100644 index 00000000000..f452f29883f --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -0,0 +1,102 @@ +use rustc_middle::mir; + +use super::FunctionCx; +use super::LocalRef; +use crate::traits::BuilderMethods; +use crate::traits::*; + +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + #[instrument(level = "debug", skip(self, bx))] + pub fn codegen_statement(&mut self, mut bx: Bx, statement: &mir::Statement<'tcx>) -> Bx { + self.set_debug_loc(&mut bx, statement.source_info); + match statement.kind { + mir::StatementKind::Assign(box (ref place, ref rvalue)) => { + if let Some(index) = place.as_local() { + match self.locals[index] { + LocalRef::Place(cg_dest) => self.codegen_rvalue(bx, cg_dest, rvalue), + LocalRef::UnsizedPlace(cg_indirect_dest) => { + self.codegen_rvalue_unsized(bx, cg_indirect_dest, rvalue) + } + LocalRef::Operand(None) => { + let (mut bx, operand) = self.codegen_rvalue_operand(bx, rvalue); + self.locals[index] = LocalRef::Operand(Some(operand)); + self.debug_introduce_local(&mut bx, index); + bx + } + LocalRef::Operand(Some(op)) => { + if !op.layout.is_zst() { + span_bug!( + statement.source_info.span, + "operand {:?} already assigned", + rvalue + ); + } + + // If the type is zero-sized, it's already been set here, + // but we still need to make sure we codegen the operand + self.codegen_rvalue_operand(bx, rvalue).0 + } + } + } else { + let cg_dest = self.codegen_place(&mut bx, place.as_ref()); + self.codegen_rvalue(bx, cg_dest, rvalue) + } + } + mir::StatementKind::SetDiscriminant { box ref place, variant_index } => { + self.codegen_place(&mut bx, place.as_ref()) + .codegen_set_discr(&mut bx, variant_index); + bx + } + mir::StatementKind::Deinit(..) => { + // For now, don't codegen this to anything. In the future it may be worth + // experimenting with what kind of information we can emit to LLVM without hurting + // perf here + bx + } + mir::StatementKind::StorageLive(local) => { + if let LocalRef::Place(cg_place) = self.locals[local] { + cg_place.storage_live(&mut bx); + } else if let LocalRef::UnsizedPlace(cg_indirect_place) = self.locals[local] { + cg_indirect_place.storage_live(&mut bx); + } + bx + } + mir::StatementKind::StorageDead(local) => { + if let LocalRef::Place(cg_place) = self.locals[local] { + cg_place.storage_dead(&mut bx); + } else if let LocalRef::UnsizedPlace(cg_indirect_place) = self.locals[local] { + cg_indirect_place.storage_dead(&mut bx); + } + bx + } + mir::StatementKind::Coverage(box ref coverage) => { + self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope); + bx + } + mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping { + ref src, + ref dst, + ref count, + }) => { + let dst_val = self.codegen_operand(&mut bx, dst); + let src_val = self.codegen_operand(&mut bx, src); + let count = self.codegen_operand(&mut bx, count).immediate(); + let pointee_layout = dst_val + .layout + .pointee_info_at(&bx, rustc_target::abi::Size::ZERO) + .expect("Expected pointer"); + let bytes = bx.mul(count, bx.const_usize(pointee_layout.size.bytes())); + + let align = pointee_layout.align; + let dst = dst_val.immediate(); + let src = src_val.immediate(); + bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty()); + bx + } + mir::StatementKind::FakeRead(..) + | mir::StatementKind::Retag { .. } + | mir::StatementKind::AscribeUserType(..) + | mir::StatementKind::Nop => bx, + } + } +} diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs new file mode 100644 index 00000000000..5006a2157fc --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/mono_item.rs @@ -0,0 +1,147 @@ +use crate::base; +use crate::common; +use crate::traits::*; +use rustc_hir as hir; +use rustc_middle::mir::mono::MonoItem; +use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::ty; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::Instance; + +pub trait MonoItemExt<'a, 'tcx> { + fn define>(&self, cx: &'a Bx::CodegenCx); + fn predefine>( + &self, + cx: &'a Bx::CodegenCx, + linkage: Linkage, + visibility: Visibility, + ); + fn to_raw_string(&self) -> String; +} + +impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> { + fn define>(&self, cx: &'a Bx::CodegenCx) { + debug!( + "BEGIN IMPLEMENTING '{} ({})' in cgu {}", + self, + self.to_raw_string(), + cx.codegen_unit().name() + ); + + match *self { + MonoItem::Static(def_id) => { + cx.codegen_static(def_id, cx.tcx().is_mutable_static(def_id)); + } + MonoItem::GlobalAsm(item_id) => { + let item = cx.tcx().hir().item(item_id); + if let hir::ItemKind::GlobalAsm(ref asm) = item.kind { + let operands: Vec<_> = asm + .operands + .iter() + .map(|(op, op_sp)| match *op { + hir::InlineAsmOperand::Const { ref anon_const } => { + let anon_const_def_id = + cx.tcx().hir().local_def_id(anon_const.hir_id).to_def_id(); + let const_value = + cx.tcx().const_eval_poly(anon_const_def_id).unwrap_or_else( + |_| span_bug!(*op_sp, "asm const cannot be resolved"), + ); + let ty = cx + .tcx() + .typeck_body(anon_const.body) + .node_type(anon_const.hir_id); + let string = common::asm_const_to_str( + cx.tcx(), + *op_sp, + const_value, + cx.layout_of(ty), + ); + GlobalAsmOperandRef::Const { string } + } + hir::InlineAsmOperand::SymFn { ref anon_const } => { + let ty = cx + .tcx() + .typeck_body(anon_const.body) + .node_type(anon_const.hir_id); + let instance = match ty.kind() { + &ty::FnDef(def_id, substs) => Instance::new(def_id, substs), + _ => span_bug!(*op_sp, "asm sym is not a function"), + }; + + GlobalAsmOperandRef::SymFn { instance } + } + hir::InlineAsmOperand::SymStatic { path: _, def_id } => { + GlobalAsmOperandRef::SymStatic { def_id } + } + hir::InlineAsmOperand::In { .. } + | hir::InlineAsmOperand::Out { .. } + | hir::InlineAsmOperand::InOut { .. } + | hir::InlineAsmOperand::SplitInOut { .. } => { + span_bug!(*op_sp, "invalid operand type for global_asm!") + } + }) + .collect(); + + cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans); + } else { + span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type") + } + } + MonoItem::Fn(instance) => { + base::codegen_instance::(&cx, instance); + } + } + + debug!( + "END IMPLEMENTING '{} ({})' in cgu {}", + self, + self.to_raw_string(), + cx.codegen_unit().name() + ); + } + + fn predefine>( + &self, + cx: &'a Bx::CodegenCx, + linkage: Linkage, + visibility: Visibility, + ) { + debug!( + "BEGIN PREDEFINING '{} ({})' in cgu {}", + self, + self.to_raw_string(), + cx.codegen_unit().name() + ); + + let symbol_name = self.symbol_name(cx.tcx()).name; + + debug!("symbol {}", &symbol_name); + + match *self { + MonoItem::Static(def_id) => { + cx.predefine_static(def_id, linkage, visibility, &symbol_name); + } + MonoItem::Fn(instance) => { + cx.predefine_fn(instance, linkage, visibility, &symbol_name); + } + MonoItem::GlobalAsm(..) => {} + } + + debug!( + "END PREDEFINING '{} ({})' in cgu {}", + self, + self.to_raw_string(), + cx.codegen_unit().name() + ); + } + + fn to_raw_string(&self) -> String { + match *self { + MonoItem::Fn(instance) => { + format!("Fn({:?}, {})", instance.def, instance.substs.as_ptr().addr()) + } + MonoItem::Static(id) => format!("Static({:?})", id), + MonoItem::GlobalAsm(id) => format!("GlobalAsm({:?})", id), + } + } +} diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs new file mode 100644 index 00000000000..ecad0518533 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -0,0 +1,308 @@ +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::ty::query::Providers; +use rustc_session::Session; +use rustc_span::symbol::sym; +use rustc_span::symbol::Symbol; + +/// Features that control behaviour of rustc, rather than the codegen. +pub const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"]; + +// When adding features to the below lists +// check whether they're named already elsewhere in rust +// e.g. in stdarch and whether the given name matches LLVM's +// if it doesn't, to_llvm_feature in llvm_util in rustc_codegen_llvm needs to be adapted + +const ARM_ALLOWED_FEATURES: &[(&str, Option)] = &[ + ("aclass", Some(sym::arm_target_feature)), + ("mclass", Some(sym::arm_target_feature)), + ("rclass", Some(sym::arm_target_feature)), + ("dsp", Some(sym::arm_target_feature)), + ("neon", Some(sym::arm_target_feature)), + ("crc", Some(sym::arm_target_feature)), + ("crypto", Some(sym::arm_target_feature)), + ("aes", Some(sym::arm_target_feature)), + ("sha2", Some(sym::arm_target_feature)), + ("i8mm", Some(sym::arm_target_feature)), + ("dotprod", Some(sym::arm_target_feature)), + ("v5te", Some(sym::arm_target_feature)), + ("v6", Some(sym::arm_target_feature)), + ("v6k", Some(sym::arm_target_feature)), + ("v6t2", Some(sym::arm_target_feature)), + ("v7", Some(sym::arm_target_feature)), + ("v8", Some(sym::arm_target_feature)), + ("vfp2", Some(sym::arm_target_feature)), + ("vfp3", Some(sym::arm_target_feature)), + ("vfp4", Some(sym::arm_target_feature)), + ("fp-armv8", Some(sym::arm_target_feature)), + // This is needed for inline assembly, but shouldn't be stabilized as-is + // since it should be enabled per-function using #[instruction_set], not + // #[target_feature]. + ("thumb-mode", Some(sym::arm_target_feature)), + ("thumb2", Some(sym::arm_target_feature)), + ("d32", Some(sym::arm_target_feature)), +]; + +const AARCH64_ALLOWED_FEATURES: &[(&str, Option)] = &[ + // FEAT_AdvSimd & FEAT_FP + ("neon", None), + // FEAT_FP16 + ("fp16", None), + // FEAT_SVE + ("sve", None), + // FEAT_CRC + ("crc", None), + // FEAT_RAS + ("ras", None), + // FEAT_LSE + ("lse", None), + // FEAT_RDM + ("rdm", None), + // FEAT_RCPC + ("rcpc", None), + // FEAT_RCPC2 + ("rcpc2", None), + // FEAT_DotProd + ("dotprod", None), + // FEAT_TME + ("tme", None), + // FEAT_FHM + ("fhm", None), + // FEAT_DIT + ("dit", None), + // FEAT_FLAGM + ("flagm", None), + // FEAT_SSBS + ("ssbs", None), + // FEAT_SB + ("sb", None), + // FEAT_PAUTH (address authentication) + ("paca", None), + // FEAT_PAUTH (generic authentication) + ("pacg", None), + // FEAT_DPB + ("dpb", None), + // FEAT_DPB2 + ("dpb2", None), + // FEAT_SVE2 + ("sve2", None), + // FEAT_SVE2_AES + ("sve2-aes", None), + // FEAT_SVE2_SM4 + ("sve2-sm4", None), + // FEAT_SVE2_SHA3 + ("sve2-sha3", None), + // FEAT_SVE2_BitPerm + ("sve2-bitperm", None), + // FEAT_FRINTTS + ("frintts", None), + // FEAT_I8MM + ("i8mm", None), + // FEAT_F32MM + ("f32mm", None), + // FEAT_F64MM + ("f64mm", None), + // FEAT_BF16 + ("bf16", None), + // FEAT_RAND + ("rand", None), + // FEAT_BTI + ("bti", None), + // FEAT_MTE + ("mte", None), + // FEAT_JSCVT + ("jsconv", None), + // FEAT_FCMA + ("fcma", None), + // FEAT_AES + ("aes", None), + // FEAT_SHA1 & FEAT_SHA256 + ("sha2", None), + // FEAT_SHA512 & FEAT_SHA3 + ("sha3", None), + // FEAT_SM3 & FEAT_SM4 + ("sm4", None), + // FEAT_PAN + ("pan", None), + // FEAT_LOR + ("lor", None), + // FEAT_VHE + ("vh", None), + // FEAT_PMUv3 + ("pmuv3", None), + // FEAT_SPE + ("spe", None), + ("v8.1a", Some(sym::aarch64_ver_target_feature)), + ("v8.2a", Some(sym::aarch64_ver_target_feature)), + ("v8.3a", Some(sym::aarch64_ver_target_feature)), + ("v8.4a", Some(sym::aarch64_ver_target_feature)), + ("v8.5a", Some(sym::aarch64_ver_target_feature)), + ("v8.6a", Some(sym::aarch64_ver_target_feature)), + ("v8.7a", Some(sym::aarch64_ver_target_feature)), +]; + +const AARCH64_TIED_FEATURES: &[&[&str]] = &[ + &["paca", "pacg"], // Together these represent `pauth` in LLVM +]; + +const X86_ALLOWED_FEATURES: &[(&str, Option)] = &[ + ("adx", None), + ("aes", None), + ("avx", None), + ("avx2", None), + ("avx512bf16", Some(sym::avx512_target_feature)), + ("avx512bitalg", Some(sym::avx512_target_feature)), + ("avx512bw", Some(sym::avx512_target_feature)), + ("avx512cd", Some(sym::avx512_target_feature)), + ("avx512dq", Some(sym::avx512_target_feature)), + ("avx512er", Some(sym::avx512_target_feature)), + ("avx512f", Some(sym::avx512_target_feature)), + ("avx512gfni", Some(sym::avx512_target_feature)), + ("avx512ifma", Some(sym::avx512_target_feature)), + ("avx512pf", Some(sym::avx512_target_feature)), + ("avx512vaes", Some(sym::avx512_target_feature)), + ("avx512vbmi", Some(sym::avx512_target_feature)), + ("avx512vbmi2", Some(sym::avx512_target_feature)), + ("avx512vl", Some(sym::avx512_target_feature)), + ("avx512vnni", Some(sym::avx512_target_feature)), + ("avx512vp2intersect", Some(sym::avx512_target_feature)), + ("avx512vpclmulqdq", Some(sym::avx512_target_feature)), + ("avx512vpopcntdq", Some(sym::avx512_target_feature)), + ("bmi1", None), + ("bmi2", None), + ("cmpxchg16b", Some(sym::cmpxchg16b_target_feature)), + ("ermsb", Some(sym::ermsb_target_feature)), + ("f16c", Some(sym::f16c_target_feature)), + ("fma", None), + ("fxsr", None), + ("lzcnt", None), + ("movbe", Some(sym::movbe_target_feature)), + ("pclmulqdq", None), + ("popcnt", None), + ("rdrand", None), + ("rdseed", None), + ("rtm", Some(sym::rtm_target_feature)), + ("sha", None), + ("sse", None), + ("sse2", None), + ("sse3", None), + ("sse4.1", None), + ("sse4.2", None), + ("sse4a", Some(sym::sse4a_target_feature)), + ("ssse3", None), + ("tbm", Some(sym::tbm_target_feature)), + ("xsave", None), + ("xsavec", None), + ("xsaveopt", None), + ("xsaves", None), +]; + +const HEXAGON_ALLOWED_FEATURES: &[(&str, Option)] = &[ + ("hvx", Some(sym::hexagon_target_feature)), + ("hvx-length128b", Some(sym::hexagon_target_feature)), +]; + +const POWERPC_ALLOWED_FEATURES: &[(&str, Option)] = &[ + ("altivec", Some(sym::powerpc_target_feature)), + ("power8-altivec", Some(sym::powerpc_target_feature)), + ("power9-altivec", Some(sym::powerpc_target_feature)), + ("power8-vector", Some(sym::powerpc_target_feature)), + ("power9-vector", Some(sym::powerpc_target_feature)), + ("vsx", Some(sym::powerpc_target_feature)), +]; + +const MIPS_ALLOWED_FEATURES: &[(&str, Option)] = + &[("fp64", Some(sym::mips_target_feature)), ("msa", Some(sym::mips_target_feature))]; + +const RISCV_ALLOWED_FEATURES: &[(&str, Option)] = &[ + ("m", Some(sym::riscv_target_feature)), + ("a", Some(sym::riscv_target_feature)), + ("c", Some(sym::riscv_target_feature)), + ("f", Some(sym::riscv_target_feature)), + ("d", Some(sym::riscv_target_feature)), + ("e", Some(sym::riscv_target_feature)), + ("v", Some(sym::riscv_target_feature)), + ("zfinx", Some(sym::riscv_target_feature)), + ("zdinx", Some(sym::riscv_target_feature)), + ("zhinx", Some(sym::riscv_target_feature)), + ("zhinxmin", Some(sym::riscv_target_feature)), + ("zfh", Some(sym::riscv_target_feature)), + ("zfhmin", Some(sym::riscv_target_feature)), + ("zbkb", Some(sym::riscv_target_feature)), + ("zbkc", Some(sym::riscv_target_feature)), + ("zbkx", Some(sym::riscv_target_feature)), + ("zknd", Some(sym::riscv_target_feature)), + ("zkne", Some(sym::riscv_target_feature)), + ("zknh", Some(sym::riscv_target_feature)), + ("zksed", Some(sym::riscv_target_feature)), + ("zksh", Some(sym::riscv_target_feature)), + ("zkr", Some(sym::riscv_target_feature)), + ("zkn", Some(sym::riscv_target_feature)), + ("zks", Some(sym::riscv_target_feature)), + ("zk", Some(sym::riscv_target_feature)), + ("zkt", Some(sym::riscv_target_feature)), +]; + +const WASM_ALLOWED_FEATURES: &[(&str, Option)] = &[ + ("simd128", None), + ("atomics", Some(sym::wasm_target_feature)), + ("nontrapping-fptoint", Some(sym::wasm_target_feature)), + ("bulk-memory", Some(sym::wasm_target_feature)), + ("mutable-globals", Some(sym::wasm_target_feature)), + ("reference-types", Some(sym::wasm_target_feature)), + ("sign-ext", Some(sym::wasm_target_feature)), +]; + +const BPF_ALLOWED_FEATURES: &[(&str, Option)] = &[("alu32", Some(sym::bpf_target_feature))]; + +/// When rustdoc is running, provide a list of all known features so that all their respective +/// primitives may be documented. +/// +/// IMPORTANT: If you're adding another feature list above, make sure to add it to this iterator! +pub fn all_known_features() -> impl Iterator)> { + std::iter::empty() + .chain(ARM_ALLOWED_FEATURES.iter()) + .chain(AARCH64_ALLOWED_FEATURES.iter()) + .chain(X86_ALLOWED_FEATURES.iter()) + .chain(HEXAGON_ALLOWED_FEATURES.iter()) + .chain(POWERPC_ALLOWED_FEATURES.iter()) + .chain(MIPS_ALLOWED_FEATURES.iter()) + .chain(RISCV_ALLOWED_FEATURES.iter()) + .chain(WASM_ALLOWED_FEATURES.iter()) + .chain(BPF_ALLOWED_FEATURES.iter()) + .cloned() +} + +pub fn supported_target_features(sess: &Session) -> &'static [(&'static str, Option)] { + match &*sess.target.arch { + "arm" => ARM_ALLOWED_FEATURES, + "aarch64" => AARCH64_ALLOWED_FEATURES, + "x86" | "x86_64" => X86_ALLOWED_FEATURES, + "hexagon" => HEXAGON_ALLOWED_FEATURES, + "mips" | "mips64" => MIPS_ALLOWED_FEATURES, + "powerpc" | "powerpc64" => POWERPC_ALLOWED_FEATURES, + "riscv32" | "riscv64" => RISCV_ALLOWED_FEATURES, + "wasm32" | "wasm64" => WASM_ALLOWED_FEATURES, + "bpf" => BPF_ALLOWED_FEATURES, + _ => &[], + } +} + +pub fn tied_target_features(sess: &Session) -> &'static [&'static [&'static str]] { + match &*sess.target.arch { + "aarch64" => AARCH64_TIED_FEATURES, + _ => &[], + } +} + +pub(crate) fn provide(providers: &mut Providers) { + providers.supported_target_features = |tcx, cnum| { + assert_eq!(cnum, LOCAL_CRATE); + if tcx.sess.opts.actually_rustdoc { + // rustdoc needs to be able to document functions that use all the features, so + // whitelist them all + all_known_features().map(|(a, b)| (a.to_string(), b)).collect() + } else { + supported_target_features(tcx.sess).iter().map(|&(a, b)| (a.to_string(), b)).collect() + } + }; +} diff --git a/compiler/rustc_codegen_ssa/src/traits/abi.rs b/compiler/rustc_codegen_ssa/src/traits/abi.rs new file mode 100644 index 00000000000..a00d78daf4d --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/abi.rs @@ -0,0 +1,8 @@ +use super::BackendTypes; +use rustc_middle::ty::Ty; +use rustc_target::abi::call::FnAbi; + +pub trait AbiBuilderMethods<'tcx>: BackendTypes { + fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value); + fn get_param(&mut self, index: usize) -> Self::Value; +} diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs new file mode 100644 index 00000000000..c2ae74b18d8 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/asm.rs @@ -0,0 +1,66 @@ +use super::BackendTypes; +use crate::mir::operand::OperandRef; +use crate::mir::place::PlaceRef; +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_hir::def_id::DefId; +use rustc_middle::ty::Instance; +use rustc_span::Span; +use rustc_target::asm::InlineAsmRegOrRegClass; + +#[derive(Debug)] +pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> { + In { + reg: InlineAsmRegOrRegClass, + value: OperandRef<'tcx, B::Value>, + }, + Out { + reg: InlineAsmRegOrRegClass, + late: bool, + place: Option>, + }, + InOut { + reg: InlineAsmRegOrRegClass, + late: bool, + in_value: OperandRef<'tcx, B::Value>, + out_place: Option>, + }, + Const { + string: String, + }, + SymFn { + instance: Instance<'tcx>, + }, + SymStatic { + def_id: DefId, + }, +} + +#[derive(Debug)] +pub enum GlobalAsmOperandRef<'tcx> { + Const { string: String }, + SymFn { instance: Instance<'tcx> }, + SymStatic { def_id: DefId }, +} + +pub trait AsmBuilderMethods<'tcx>: BackendTypes { + /// Take an inline assembly expression and splat it out via LLVM + fn codegen_inline_asm( + &mut self, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperandRef<'tcx, Self>], + options: InlineAsmOptions, + line_spans: &[Span], + instance: Instance<'_>, + dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + ); +} + +pub trait AsmMethods<'tcx> { + fn codegen_global_asm( + &self, + template: &[InlineAsmTemplatePiece], + operands: &[GlobalAsmOperandRef<'tcx>], + options: InlineAsmOptions, + line_spans: &[Span], + ); +} diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs new file mode 100644 index 00000000000..779bd3ea278 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs @@ -0,0 +1,161 @@ +use super::write::WriteBackendMethods; +use super::CodegenObject; +use crate::back::write::TargetMachineFactoryFn; +use crate::{CodegenResults, ModuleCodegen}; + +use rustc_ast::expand::allocator::AllocatorKind; +use rustc_data_structures::fx::FxHashMap; +use rustc_errors::ErrorGuaranteed; +use rustc_metadata::EncodedMetadata; +use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf, TyAndLayout}; +use rustc_middle::ty::query::{ExternProviders, Providers}; +use rustc_middle::ty::{Ty, TyCtxt}; +use rustc_session::{ + config::{self, OutputFilenames, PrintRequest}, + cstore::MetadataLoaderDyn, + Session, +}; +use rustc_span::symbol::Symbol; +use rustc_target::abi::call::FnAbi; +use rustc_target::spec::Target; + +pub use rustc_data_structures::sync::MetadataRef; + +use std::any::Any; + +pub trait BackendTypes { + type Value: CodegenObject; + type Function: CodegenObject; + + type BasicBlock: Copy; + type Type: CodegenObject; + type Funclet; + + // FIXME(eddyb) find a common convention for all of the debuginfo-related + // names (choose between `Dbg`, `Debug`, `DebugInfo`, `DI` etc.). + type DIScope: Copy; + type DILocation: Copy; + type DIVariable: Copy; +} + +pub trait Backend<'tcx>: + Sized + + BackendTypes + + HasTyCtxt<'tcx> + + LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>> + + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>> +{ +} + +impl<'tcx, T> Backend<'tcx> for T where + Self: BackendTypes + + HasTyCtxt<'tcx> + + LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>> + + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>> +{ +} + +pub trait CodegenBackend { + fn init(&self, _sess: &Session) {} + fn print(&self, _req: PrintRequest, _sess: &Session) {} + fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec { + vec![] + } + fn print_passes(&self) {} + fn print_version(&self) {} + + /// If this plugin provides additional builtin targets, provide the one enabled by the options here. + /// Be careful: this is called *before* init() is called. + fn target_override(&self, _opts: &config::Options) -> Option { + None + } + + /// The metadata loader used to load rlib and dylib metadata. + /// + /// Alternative codegen backends may want to use different rlib or dylib formats than the + /// default native static archives and dynamic libraries. + fn metadata_loader(&self) -> Box { + Box::new(crate::back::metadata::DefaultMetadataLoader) + } + + fn provide(&self, _providers: &mut Providers) {} + fn provide_extern(&self, _providers: &mut ExternProviders) {} + fn codegen_crate<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + metadata: EncodedMetadata, + need_metadata_module: bool, + ) -> Box; + + /// This is called on the returned `Box` from `codegen_backend` + /// + /// # Panics + /// + /// Panics when the passed `Box` was not returned by `codegen_backend`. + fn join_codegen( + &self, + ongoing_codegen: Box, + sess: &Session, + outputs: &OutputFilenames, + ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed>; + + /// This is called on the returned `Box` from `join_codegen` + /// + /// # Panics + /// + /// Panics when the passed `Box` was not returned by `join_codegen`. + fn link( + &self, + sess: &Session, + codegen_results: CodegenResults, + outputs: &OutputFilenames, + ) -> Result<(), ErrorGuaranteed>; +} + +pub trait ExtraBackendMethods: CodegenBackend + WriteBackendMethods + Sized + Send + Sync { + fn codegen_allocator<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + module_name: &str, + kind: AllocatorKind, + has_alloc_error_handler: bool, + ) -> Self::Module; + /// This generates the codegen unit and returns it along with + /// a `u64` giving an estimate of the unit's processing cost. + fn compile_codegen_unit( + &self, + tcx: TyCtxt<'_>, + cgu_name: Symbol, + ) -> (ModuleCodegen, u64); + fn target_machine_factory( + &self, + sess: &Session, + opt_level: config::OptLevel, + target_features: &[String], + ) -> TargetMachineFactoryFn; + fn target_cpu<'b>(&self, sess: &'b Session) -> &'b str; + fn tune_cpu<'b>(&self, sess: &'b Session) -> Option<&'b str>; + + fn spawn_thread(_time_trace: bool, f: F) -> std::thread::JoinHandle + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send + 'static, + { + std::thread::spawn(f) + } + + fn spawn_named_thread( + _time_trace: bool, + name: String, + f: F, + ) -> std::io::Result> + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send + 'static, + { + std::thread::Builder::new().name(name).spawn(f) + } +} diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs new file mode 100644 index 00000000000..9f49749bb41 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -0,0 +1,481 @@ +use super::abi::AbiBuilderMethods; +use super::asm::AsmBuilderMethods; +use super::consts::ConstMethods; +use super::coverageinfo::CoverageInfoBuilderMethods; +use super::debuginfo::DebugInfoBuilderMethods; +use super::intrinsic::IntrinsicCallMethods; +use super::misc::MiscMethods; +use super::type_::{ArgAbiMethods, BaseTypeMethods}; +use super::{HasCodegen, StaticBuilderMethods}; + +use crate::common::{ + AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind, +}; +use crate::mir::operand::OperandRef; +use crate::mir::place::PlaceRef; +use crate::MemFlags; + +use rustc_apfloat::{ieee, Float, Round, Status}; +use rustc_middle::ty::layout::{HasParamEnv, TyAndLayout}; +use rustc_middle::ty::Ty; +use rustc_span::Span; +use rustc_target::abi::{Abi, Align, Scalar, Size, WrappingRange}; +use rustc_target::spec::HasTargetSpec; + +#[derive(Copy, Clone)] +pub enum OverflowOp { + Add, + Sub, + Mul, +} + +pub trait BuilderMethods<'a, 'tcx>: + HasCodegen<'tcx> + + CoverageInfoBuilderMethods<'tcx> + + DebugInfoBuilderMethods + + ArgAbiMethods<'tcx> + + AbiBuilderMethods<'tcx> + + IntrinsicCallMethods<'tcx> + + AsmBuilderMethods<'tcx> + + StaticBuilderMethods + + HasParamEnv<'tcx> + + HasTargetSpec +{ + fn build(cx: &'a Self::CodegenCx, llbb: Self::BasicBlock) -> Self; + + fn cx(&self) -> &Self::CodegenCx; + fn llbb(&self) -> Self::BasicBlock; + + fn set_span(&mut self, span: Span); + + // FIXME(eddyb) replace uses of this with `append_sibling_block`. + fn append_block(cx: &'a Self::CodegenCx, llfn: Self::Function, name: &str) -> Self::BasicBlock; + + fn append_sibling_block(&mut self, name: &str) -> Self::BasicBlock; + + fn switch_to_block(&mut self, llbb: Self::BasicBlock); + + fn ret_void(&mut self); + fn ret(&mut self, v: Self::Value); + fn br(&mut self, dest: Self::BasicBlock); + fn cond_br( + &mut self, + cond: Self::Value, + then_llbb: Self::BasicBlock, + else_llbb: Self::BasicBlock, + ); + fn switch( + &mut self, + v: Self::Value, + else_llbb: Self::BasicBlock, + cases: impl ExactSizeIterator, + ); + fn invoke( + &mut self, + llty: Self::Type, + llfn: Self::Value, + args: &[Self::Value], + then: Self::BasicBlock, + catch: Self::BasicBlock, + funclet: Option<&Self::Funclet>, + ) -> Self::Value; + fn unreachable(&mut self); + + fn add(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fadd_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn sub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fsub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fsub_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn mul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fmul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fmul_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn udiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn exactudiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn sdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn exactsdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fdiv_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn urem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn srem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn frem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn frem_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn shl(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn lshr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn unchecked_ssub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn unchecked_usub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn unchecked_smul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn and(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn or(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn xor(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn neg(&mut self, v: Self::Value) -> Self::Value; + fn fneg(&mut self, v: Self::Value) -> Self::Value; + fn not(&mut self, v: Self::Value) -> Self::Value; + + fn checked_binop( + &mut self, + oop: OverflowOp, + ty: Ty<'_>, + lhs: Self::Value, + rhs: Self::Value, + ) -> (Self::Value, Self::Value); + + fn from_immediate(&mut self, val: Self::Value) -> Self::Value; + fn to_immediate(&mut self, val: Self::Value, layout: TyAndLayout<'_>) -> Self::Value { + if let Abi::Scalar(scalar) = layout.abi { + self.to_immediate_scalar(val, scalar) + } else { + val + } + } + fn to_immediate_scalar(&mut self, val: Self::Value, scalar: Scalar) -> Self::Value; + + fn alloca(&mut self, ty: Self::Type, align: Align) -> Self::Value; + fn dynamic_alloca(&mut self, ty: Self::Type, align: Align) -> Self::Value; + fn array_alloca(&mut self, ty: Self::Type, len: Self::Value, align: Align) -> Self::Value; + + fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value; + fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value; + fn atomic_load( + &mut self, + ty: Self::Type, + ptr: Self::Value, + order: AtomicOrdering, + size: Size, + ) -> Self::Value; + fn load_operand(&mut self, place: PlaceRef<'tcx, Self::Value>) + -> OperandRef<'tcx, Self::Value>; + + /// Called for Rvalue::Repeat when the elem is neither a ZST nor optimizable using memset. + fn write_operand_repeatedly( + self, + elem: OperandRef<'tcx, Self::Value>, + count: u64, + dest: PlaceRef<'tcx, Self::Value>, + ) -> Self; + + fn range_metadata(&mut self, load: Self::Value, range: WrappingRange); + fn nonnull_metadata(&mut self, load: Self::Value); + + fn store(&mut self, val: Self::Value, ptr: Self::Value, align: Align) -> Self::Value; + fn store_with_flags( + &mut self, + val: Self::Value, + ptr: Self::Value, + align: Align, + flags: MemFlags, + ) -> Self::Value; + fn atomic_store( + &mut self, + val: Self::Value, + ptr: Self::Value, + order: AtomicOrdering, + size: Size, + ); + + fn gep(&mut self, ty: Self::Type, ptr: Self::Value, indices: &[Self::Value]) -> Self::Value; + fn inbounds_gep( + &mut self, + ty: Self::Type, + ptr: Self::Value, + indices: &[Self::Value], + ) -> Self::Value; + fn struct_gep(&mut self, ty: Self::Type, ptr: Self::Value, idx: u64) -> Self::Value; + + fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option; + fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option; + fn fptoui(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn fptosi(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn uitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn sitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn fptrunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn fpext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn ptrtoint(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn inttoptr(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn bitcast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn intcast(&mut self, val: Self::Value, dest_ty: Self::Type, is_signed: bool) -> Self::Value; + fn pointercast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + + fn cast_float_to_int( + &mut self, + signed: bool, + x: Self::Value, + dest_ty: Self::Type, + ) -> Self::Value { + let in_ty = self.cx().val_ty(x); + let (float_ty, int_ty) = if self.cx().type_kind(dest_ty) == TypeKind::Vector + && self.cx().type_kind(in_ty) == TypeKind::Vector + { + (self.cx().element_type(in_ty), self.cx().element_type(dest_ty)) + } else { + (in_ty, dest_ty) + }; + assert!(matches!(self.cx().type_kind(float_ty), TypeKind::Float | TypeKind::Double)); + assert_eq!(self.cx().type_kind(int_ty), TypeKind::Integer); + + if let Some(false) = self.cx().sess().opts.unstable_opts.saturating_float_casts { + return if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) }; + } + + let try_sat_result = + if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) }; + if let Some(try_sat_result) = try_sat_result { + return try_sat_result; + } + + let int_width = self.cx().int_width(int_ty); + let float_width = self.cx().float_width(float_ty); + // LLVM's fpto[su]i returns undef when the input x is infinite, NaN, or does not fit into the + // destination integer type after rounding towards zero. This `undef` value can cause UB in + // safe code (see issue #10184), so we implement a saturating conversion on top of it: + // Semantically, the mathematical value of the input is rounded towards zero to the next + // mathematical integer, and then the result is clamped into the range of the destination + // integer type. Positive and negative infinity are mapped to the maximum and minimum value of + // the destination integer type. NaN is mapped to 0. + // + // Define f_min and f_max as the largest and smallest (finite) floats that are exactly equal to + // a value representable in int_ty. + // They are exactly equal to int_ty::{MIN,MAX} if float_ty has enough significand bits. + // Otherwise, int_ty::MAX must be rounded towards zero, as it is one less than a power of two. + // int_ty::MIN, however, is either zero or a negative power of two and is thus exactly + // representable. Note that this only works if float_ty's exponent range is sufficiently large. + // f16 or 256 bit integers would break this property. Right now the smallest float type is f32 + // with exponents ranging up to 127, which is barely enough for i128::MIN = -2^127. + // On the other hand, f_max works even if int_ty::MAX is greater than float_ty::MAX. Because + // we're rounding towards zero, we just get float_ty::MAX (which is always an integer). + // This already happens today with u128::MAX = 2^128 - 1 > f32::MAX. + let int_max = |signed: bool, int_width: u64| -> u128 { + let shift_amount = 128 - int_width; + if signed { i128::MAX as u128 >> shift_amount } else { u128::MAX >> shift_amount } + }; + let int_min = |signed: bool, int_width: u64| -> i128 { + if signed { i128::MIN >> (128 - int_width) } else { 0 } + }; + + let compute_clamp_bounds_single = |signed: bool, int_width: u64| -> (u128, u128) { + let rounded_min = + ieee::Single::from_i128_r(int_min(signed, int_width), Round::TowardZero); + assert_eq!(rounded_min.status, Status::OK); + let rounded_max = + ieee::Single::from_u128_r(int_max(signed, int_width), Round::TowardZero); + assert!(rounded_max.value.is_finite()); + (rounded_min.value.to_bits(), rounded_max.value.to_bits()) + }; + let compute_clamp_bounds_double = |signed: bool, int_width: u64| -> (u128, u128) { + let rounded_min = + ieee::Double::from_i128_r(int_min(signed, int_width), Round::TowardZero); + assert_eq!(rounded_min.status, Status::OK); + let rounded_max = + ieee::Double::from_u128_r(int_max(signed, int_width), Round::TowardZero); + assert!(rounded_max.value.is_finite()); + (rounded_min.value.to_bits(), rounded_max.value.to_bits()) + }; + // To implement saturation, we perform the following steps: + // + // 1. Cast x to an integer with fpto[su]i. This may result in undef. + // 2. Compare x to f_min and f_max, and use the comparison results to select: + // a) int_ty::MIN if x < f_min or x is NaN + // b) int_ty::MAX if x > f_max + // c) the result of fpto[su]i otherwise + // 3. If x is NaN, return 0.0, otherwise return the result of step 2. + // + // This avoids resulting undef because values in range [f_min, f_max] by definition fit into the + // destination type. It creates an undef temporary, but *producing* undef is not UB. Our use of + // undef does not introduce any non-determinism either. + // More importantly, the above procedure correctly implements saturating conversion. + // Proof (sketch): + // If x is NaN, 0 is returned by definition. + // Otherwise, x is finite or infinite and thus can be compared with f_min and f_max. + // This yields three cases to consider: + // (1) if x in [f_min, f_max], the result of fpto[su]i is returned, which agrees with + // saturating conversion for inputs in that range. + // (2) if x > f_max, then x is larger than int_ty::MAX. This holds even if f_max is rounded + // (i.e., if f_max < int_ty::MAX) because in those cases, nextUp(f_max) is already larger + // than int_ty::MAX. Because x is larger than int_ty::MAX, the return value of int_ty::MAX + // is correct. + // (3) if x < f_min, then x is smaller than int_ty::MIN. As shown earlier, f_min exactly equals + // int_ty::MIN and therefore the return value of int_ty::MIN is correct. + // QED. + + let float_bits_to_llval = |bx: &mut Self, bits| { + let bits_llval = match float_width { + 32 => bx.cx().const_u32(bits as u32), + 64 => bx.cx().const_u64(bits as u64), + n => bug!("unsupported float width {}", n), + }; + bx.bitcast(bits_llval, float_ty) + }; + let (f_min, f_max) = match float_width { + 32 => compute_clamp_bounds_single(signed, int_width), + 64 => compute_clamp_bounds_double(signed, int_width), + n => bug!("unsupported float width {}", n), + }; + let f_min = float_bits_to_llval(self, f_min); + let f_max = float_bits_to_llval(self, f_max); + let int_max = self.cx().const_uint_big(int_ty, int_max(signed, int_width)); + let int_min = self.cx().const_uint_big(int_ty, int_min(signed, int_width) as u128); + let zero = self.cx().const_uint(int_ty, 0); + + // If we're working with vectors, constants must be "splatted": the constant is duplicated + // into each lane of the vector. The algorithm stays the same, we are just using the + // same constant across all lanes. + let maybe_splat = |bx: &mut Self, val| { + if bx.cx().type_kind(dest_ty) == TypeKind::Vector { + bx.vector_splat(bx.vector_length(dest_ty), val) + } else { + val + } + }; + let f_min = maybe_splat(self, f_min); + let f_max = maybe_splat(self, f_max); + let int_max = maybe_splat(self, int_max); + let int_min = maybe_splat(self, int_min); + let zero = maybe_splat(self, zero); + + // Step 1 ... + let fptosui_result = if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) }; + let less_or_nan = self.fcmp(RealPredicate::RealULT, x, f_min); + let greater = self.fcmp(RealPredicate::RealOGT, x, f_max); + + // Step 2: We use two comparisons and two selects, with %s1 being the + // result: + // %less_or_nan = fcmp ult %x, %f_min + // %greater = fcmp olt %x, %f_max + // %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result + // %s1 = select %greater, int_ty::MAX, %s0 + // Note that %less_or_nan uses an *unordered* comparison. This + // comparison is true if the operands are not comparable (i.e., if x is + // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if + // x is NaN. + // + // Performance note: Unordered comparison can be lowered to a "flipped" + // comparison and a negation, and the negation can be merged into the + // select. Therefore, it not necessarily any more expensive than an + // ordered ("normal") comparison. Whether these optimizations will be + // performed is ultimately up to the backend, but at least x86 does + // perform them. + let s0 = self.select(less_or_nan, int_min, fptosui_result); + let s1 = self.select(greater, int_max, s0); + + // Step 3: NaN replacement. + // For unsigned types, the above step already yielded int_ty::MIN == 0 if x is NaN. + // Therefore we only need to execute this step for signed integer types. + if signed { + // LLVM has no isNaN predicate, so we use (x == x) instead + let cmp = self.fcmp(RealPredicate::RealOEQ, x, x); + self.select(cmp, s1, zero) + } else { + s1 + } + } + + fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn fcmp(&mut self, op: RealPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + + fn memcpy( + &mut self, + dst: Self::Value, + dst_align: Align, + src: Self::Value, + src_align: Align, + size: Self::Value, + flags: MemFlags, + ); + fn memmove( + &mut self, + dst: Self::Value, + dst_align: Align, + src: Self::Value, + src_align: Align, + size: Self::Value, + flags: MemFlags, + ); + fn memset( + &mut self, + ptr: Self::Value, + fill_byte: Self::Value, + size: Self::Value, + align: Align, + flags: MemFlags, + ); + + fn select( + &mut self, + cond: Self::Value, + then_val: Self::Value, + else_val: Self::Value, + ) -> Self::Value; + + fn va_arg(&mut self, list: Self::Value, ty: Self::Type) -> Self::Value; + fn extract_element(&mut self, vec: Self::Value, idx: Self::Value) -> Self::Value; + fn vector_splat(&mut self, num_elts: usize, elt: Self::Value) -> Self::Value; + fn extract_value(&mut self, agg_val: Self::Value, idx: u64) -> Self::Value; + fn insert_value(&mut self, agg_val: Self::Value, elt: Self::Value, idx: u64) -> Self::Value; + + fn set_personality_fn(&mut self, personality: Self::Value); + + // These are used by everyone except msvc + fn cleanup_landing_pad(&mut self, ty: Self::Type, pers_fn: Self::Value) -> Self::Value; + fn resume(&mut self, exn: Self::Value); + + // These are used only by msvc + fn cleanup_pad(&mut self, parent: Option, args: &[Self::Value]) -> Self::Funclet; + fn cleanup_ret(&mut self, funclet: &Self::Funclet, unwind: Option); + fn catch_pad(&mut self, parent: Self::Value, args: &[Self::Value]) -> Self::Funclet; + fn catch_switch( + &mut self, + parent: Option, + unwind: Option, + handlers: &[Self::BasicBlock], + ) -> Self::Value; + + fn atomic_cmpxchg( + &mut self, + dst: Self::Value, + cmp: Self::Value, + src: Self::Value, + order: AtomicOrdering, + failure_order: AtomicOrdering, + weak: bool, + ) -> Self::Value; + fn atomic_rmw( + &mut self, + op: AtomicRmwBinOp, + dst: Self::Value, + src: Self::Value, + order: AtomicOrdering, + ) -> Self::Value; + fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope); + fn set_invariant_load(&mut self, load: Self::Value); + + /// Called for `StorageLive` + fn lifetime_start(&mut self, ptr: Self::Value, size: Size); + + /// Called for `StorageDead` + fn lifetime_end(&mut self, ptr: Self::Value, size: Size); + + fn instrprof_increment( + &mut self, + fn_name: Self::Value, + hash: Self::Value, + num_counters: Self::Value, + index: Self::Value, + ); + + fn call( + &mut self, + llty: Self::Type, + llfn: Self::Value, + args: &[Self::Value], + funclet: Option<&Self::Funclet>, + ) -> Self::Value; + fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + + fn do_not_inline(&mut self, llret: Self::Value); +} diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs new file mode 100644 index 00000000000..8a91d4735ba --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs @@ -0,0 +1,41 @@ +use super::BackendTypes; +use crate::mir::place::PlaceRef; +use rustc_middle::mir::interpret::{ConstAllocation, Scalar}; +use rustc_middle::ty::layout::TyAndLayout; +use rustc_target::abi::{self, Size}; + +pub trait ConstMethods<'tcx>: BackendTypes { + // Constant constructors + fn const_null(&self, t: Self::Type) -> Self::Value; + fn const_undef(&self, t: Self::Type) -> Self::Value; + fn const_int(&self, t: Self::Type, i: i64) -> Self::Value; + fn const_uint(&self, t: Self::Type, i: u64) -> Self::Value; + fn const_uint_big(&self, t: Self::Type, u: u128) -> Self::Value; + fn const_bool(&self, val: bool) -> Self::Value; + fn const_i16(&self, i: i16) -> Self::Value; + fn const_i32(&self, i: i32) -> Self::Value; + fn const_u32(&self, i: u32) -> Self::Value; + fn const_u64(&self, i: u64) -> Self::Value; + fn const_usize(&self, i: u64) -> Self::Value; + fn const_u8(&self, i: u8) -> Self::Value; + fn const_real(&self, t: Self::Type, val: f64) -> Self::Value; + + fn const_str(&self, s: &str) -> (Self::Value, Self::Value); + fn const_struct(&self, elts: &[Self::Value], packed: bool) -> Self::Value; + + fn const_to_opt_uint(&self, v: Self::Value) -> Option; + fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option; + + fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value; + + fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value; + fn zst_to_backend(&self, llty: Self::Type) -> Self::Value; + fn from_const_alloc( + &self, + layout: TyAndLayout<'tcx>, + alloc: ConstAllocation<'tcx>, + offset: Size, + ) -> PlaceRef<'tcx, Self::Value>; + + fn const_ptrcast(&self, val: Self::Value, ty: Self::Type) -> Self::Value; +} diff --git a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs new file mode 100644 index 00000000000..e77201cf0c8 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs @@ -0,0 +1,57 @@ +use super::BackendTypes; +use rustc_hir::def_id::DefId; +use rustc_middle::mir::coverage::*; +use rustc_middle::ty::Instance; + +pub trait CoverageInfoMethods<'tcx>: BackendTypes { + fn coverageinfo_finalize(&self); + + /// Codegen a small function that will never be called, with one counter + /// that will never be incremented, that gives LLVM coverage tools a + /// function definition it needs in order to resolve coverage map references + /// to unused functions. This is necessary so unused functions will appear + /// as uncovered (coverage execution count `0`) in LLVM coverage reports. + fn define_unused_fn(&self, def_id: DefId); + + /// For LLVM codegen, returns a function-specific `Value` for a global + /// string, to hold the function name passed to LLVM intrinsic + /// `instrprof.increment()`. The `Value` is only created once per instance. + /// Multiple invocations with the same instance return the same `Value`. + fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> Self::Value; +} + +pub trait CoverageInfoBuilderMethods<'tcx>: BackendTypes { + /// Returns true if the function source hash was added to the coverage map (even if it had + /// already been added, for this instance). Returns false *only* if `-C instrument-coverage` is + /// not enabled (a coverage map is not being generated). + fn set_function_source_hash( + &mut self, + instance: Instance<'tcx>, + function_source_hash: u64, + ) -> bool; + + /// Returns true if the counter was added to the coverage map; false if `-C instrument-coverage` + /// is not enabled (a coverage map is not being generated). + fn add_coverage_counter( + &mut self, + instance: Instance<'tcx>, + index: CounterValueReference, + region: CodeRegion, + ) -> bool; + + /// Returns true if the expression was added to the coverage map; false if + /// `-C instrument-coverage` is not enabled (a coverage map is not being generated). + fn add_coverage_counter_expression( + &mut self, + instance: Instance<'tcx>, + id: InjectedExpressionId, + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + region: Option, + ) -> bool; + + /// Returns true if the region was added to the coverage map; false if `-C instrument-coverage` + /// is not enabled (a coverage map is not being generated). + fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool; +} diff --git a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs new file mode 100644 index 00000000000..f310789d144 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs @@ -0,0 +1,79 @@ +use super::BackendTypes; +use crate::mir::debuginfo::{FunctionDebugContext, VariableKind}; +use rustc_middle::mir; +use rustc_middle::ty::{Instance, PolyExistentialTraitRef, Ty}; +use rustc_span::{SourceFile, Span, Symbol}; +use rustc_target::abi::call::FnAbi; +use rustc_target::abi::Size; + +pub trait DebugInfoMethods<'tcx>: BackendTypes { + fn create_vtable_debuginfo( + &self, + ty: Ty<'tcx>, + trait_ref: Option>, + vtable: Self::Value, + ); + + /// Creates the function-specific debug context. + /// + /// Returns the FunctionDebugContext for the function which holds state needed + /// for debug info creation, if it is enabled. + fn create_function_debug_context( + &self, + instance: Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + llfn: Self::Function, + mir: &mir::Body<'tcx>, + ) -> Option>; + + // FIXME(eddyb) find a common convention for all of the debuginfo-related + // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). + fn dbg_scope_fn( + &self, + instance: Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + maybe_definition_llfn: Option, + ) -> Self::DIScope; + + fn dbg_loc( + &self, + scope: Self::DIScope, + inlined_at: Option, + span: Span, + ) -> Self::DILocation; + + fn extend_scope_to_file( + &self, + scope_metadata: Self::DIScope, + file: &SourceFile, + ) -> Self::DIScope; + fn debuginfo_finalize(&self); + + // FIXME(eddyb) find a common convention for all of the debuginfo-related + // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). + fn create_dbg_var( + &self, + variable_name: Symbol, + variable_type: Ty<'tcx>, + scope_metadata: Self::DIScope, + variable_kind: VariableKind, + span: Span, + ) -> Self::DIVariable; +} + +pub trait DebugInfoBuilderMethods: BackendTypes { + // FIXME(eddyb) find a common convention for all of the debuginfo-related + // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). + fn dbg_var_addr( + &mut self, + dbg_var: Self::DIVariable, + dbg_loc: Self::DILocation, + variable_alloca: Self::Value, + direct_offset: Size, + // NB: each offset implies a deref (i.e. they're steps in a pointer chain). + indirect_offsets: &[Size], + ); + fn set_dbg_loc(&mut self, dbg_loc: Self::DILocation); + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self); + fn set_var_name(&mut self, value: Self::Value, name: &str); +} diff --git a/compiler/rustc_codegen_ssa/src/traits/declare.rs b/compiler/rustc_codegen_ssa/src/traits/declare.rs new file mode 100644 index 00000000000..655afcd17f0 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/declare.rs @@ -0,0 +1,21 @@ +use super::BackendTypes; +use rustc_hir::def_id::DefId; +use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::ty::Instance; + +pub trait PreDefineMethods<'tcx>: BackendTypes { + fn predefine_static( + &self, + def_id: DefId, + linkage: Linkage, + visibility: Visibility, + symbol_name: &str, + ); + fn predefine_fn( + &self, + instance: Instance<'tcx>, + linkage: Linkage, + visibility: Visibility, + symbol_name: &str, + ); +} diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs new file mode 100644 index 00000000000..7755e67938c --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs @@ -0,0 +1,39 @@ +use super::BackendTypes; +use crate::mir::operand::OperandRef; +use rustc_middle::ty::{self, Ty}; +use rustc_span::Span; +use rustc_target::abi::call::FnAbi; + +pub trait IntrinsicCallMethods<'tcx>: BackendTypes { + /// Remember to add all intrinsics here, in `compiler/rustc_typeck/src/check/mod.rs`, + /// and in `library/core/src/intrinsics.rs`; if you need access to any LLVM intrinsics, + /// add them to `compiler/rustc_codegen_llvm/src/context.rs`. + fn codegen_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + args: &[OperandRef<'tcx, Self::Value>], + llresult: Self::Value, + span: Span, + ); + + fn abort(&mut self); + fn assume(&mut self, val: Self::Value); + fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value; + /// Trait method used to test whether a given pointer is associated with a type identifier. + fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value; + /// Trait method used to load a function while testing if it is associated with a type + /// identifier. + fn type_checked_load( + &mut self, + llvtable: Self::Value, + vtable_byte_offset: u64, + typeid: Self::Value, + ) -> Self::Value; + /// Trait method used to inject `va_start` on the "spoofed" `VaListImpl` in + /// Rust defined C-variadic functions. + fn va_start(&mut self, val: Self::Value) -> Self::Value; + /// Trait method used to inject `va_end` on the "spoofed" `VaListImpl` before + /// Rust defined C-variadic functions return. + fn va_end(&mut self, val: Self::Value) -> Self::Value; +} diff --git a/compiler/rustc_codegen_ssa/src/traits/misc.rs b/compiler/rustc_codegen_ssa/src/traits/misc.rs new file mode 100644 index 00000000000..4266e42ec2b --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/misc.rs @@ -0,0 +1,26 @@ +use super::BackendTypes; +use rustc_data_structures::fx::FxHashMap; +use rustc_middle::mir::mono::CodegenUnit; +use rustc_middle::ty::{self, Instance, Ty}; +use rustc_session::Session; +use std::cell::RefCell; + +pub trait MiscMethods<'tcx>: BackendTypes { + fn vtables( + &self, + ) -> &RefCell, Option>), Self::Value>>; + fn check_overflow(&self) -> bool; + fn get_fn(&self, instance: Instance<'tcx>) -> Self::Function; + fn get_fn_addr(&self, instance: Instance<'tcx>) -> Self::Value; + fn eh_personality(&self) -> Self::Value; + fn sess(&self) -> &Session; + fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx>; + fn used_statics(&self) -> &RefCell>; + fn compiler_used_statics(&self) -> &RefCell>; + fn set_frame_pointer_type(&self, llfn: Self::Function); + fn apply_target_cpu_attr(&self, llfn: Self::Function); + fn create_used_variable(&self); + fn create_compiler_used_variable(&self); + /// Declares the extern "C" main function for the entry point. Returns None if the symbol already exists. + fn declare_c_main(&self, fn_type: Self::Type) -> Option; +} diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs new file mode 100644 index 00000000000..782fdadbfb8 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs @@ -0,0 +1,102 @@ +//! Interface of a Rust codegen backend +//! +//! This crate defines all the traits that have to be implemented by a codegen backend in order to +//! use the backend-agnostic codegen code in `rustc_codegen_ssa`. +//! +//! The interface is designed around two backend-specific data structures, the codegen context and +//! the builder. The codegen context is supposed to be read-only after its creation and during the +//! actual codegen, while the builder stores the information about the function during codegen and +//! is used to produce the instructions of the backend IR. +//! +//! Finally, a third `Backend` structure has to implement methods related to how codegen information +//! is passed to the backend, especially for asynchronous compilation. +//! +//! The traits contain associated types that are backend-specific, such as the backend's value or +//! basic blocks. + +mod abi; +mod asm; +mod backend; +mod builder; +mod consts; +mod coverageinfo; +mod debuginfo; +mod declare; +mod intrinsic; +mod misc; +mod statics; +mod type_; +mod write; + +pub use self::abi::AbiBuilderMethods; +pub use self::asm::{AsmBuilderMethods, AsmMethods, GlobalAsmOperandRef, InlineAsmOperandRef}; +pub use self::backend::{Backend, BackendTypes, CodegenBackend, ExtraBackendMethods}; +pub use self::builder::{BuilderMethods, OverflowOp}; +pub use self::consts::ConstMethods; +pub use self::coverageinfo::{CoverageInfoBuilderMethods, CoverageInfoMethods}; +pub use self::debuginfo::{DebugInfoBuilderMethods, DebugInfoMethods}; +pub use self::declare::PreDefineMethods; +pub use self::intrinsic::IntrinsicCallMethods; +pub use self::misc::MiscMethods; +pub use self::statics::{StaticBuilderMethods, StaticMethods}; +pub use self::type_::{ + ArgAbiMethods, BaseTypeMethods, DerivedTypeMethods, LayoutTypeMethods, TypeMembershipMethods, + TypeMethods, +}; +pub use self::write::{ModuleBufferMethods, ThinBufferMethods, WriteBackendMethods}; + +use rustc_middle::ty::layout::{HasParamEnv, HasTyCtxt}; +use rustc_target::spec::HasTargetSpec; + +use std::fmt; + +pub trait CodegenObject: Copy + PartialEq + fmt::Debug {} +impl CodegenObject for T {} + +pub trait CodegenMethods<'tcx>: + Backend<'tcx> + + TypeMethods<'tcx> + + MiscMethods<'tcx> + + ConstMethods<'tcx> + + StaticMethods + + CoverageInfoMethods<'tcx> + + DebugInfoMethods<'tcx> + + AsmMethods<'tcx> + + PreDefineMethods<'tcx> + + HasParamEnv<'tcx> + + HasTyCtxt<'tcx> + + HasTargetSpec +{ +} + +impl<'tcx, T> CodegenMethods<'tcx> for T where + Self: Backend<'tcx> + + TypeMethods<'tcx> + + MiscMethods<'tcx> + + ConstMethods<'tcx> + + StaticMethods + + CoverageInfoMethods<'tcx> + + DebugInfoMethods<'tcx> + + AsmMethods<'tcx> + + PreDefineMethods<'tcx> + + HasParamEnv<'tcx> + + HasTyCtxt<'tcx> + + HasTargetSpec +{ +} + +pub trait HasCodegen<'tcx>: + Backend<'tcx> + std::ops::Deref>::CodegenCx> +{ + type CodegenCx: CodegenMethods<'tcx> + + BackendTypes< + Value = Self::Value, + Function = Self::Function, + BasicBlock = Self::BasicBlock, + Type = Self::Type, + Funclet = Self::Funclet, + DIScope = Self::DIScope, + DILocation = Self::DILocation, + DIVariable = Self::DIVariable, + >; +} diff --git a/compiler/rustc_codegen_ssa/src/traits/statics.rs b/compiler/rustc_codegen_ssa/src/traits/statics.rs new file mode 100644 index 00000000000..413d31bb942 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/statics.rs @@ -0,0 +1,24 @@ +use super::BackendTypes; +use rustc_hir::def_id::DefId; +use rustc_target::abi::Align; + +pub trait StaticMethods: BackendTypes { + fn static_addr_of(&self, cv: Self::Value, align: Align, kind: Option<&str>) -> Self::Value; + fn codegen_static(&self, def_id: DefId, is_mutable: bool); + + /// Mark the given global value as "used", to prevent the compiler and linker from potentially + /// removing a static variable that may otherwise appear unused. + fn add_used_global(&self, global: Self::Value); + + /// Same as add_used_global(), but only prevent the compiler from potentially removing an + /// otherwise unused symbol. The linker is still permitted to drop it. + /// + /// This corresponds to the documented semantics of the `#[used]` attribute, although + /// on some targets (non-ELF), we may use `add_used_global` for `#[used]` statics + /// instead. + fn add_compiler_used_global(&self, global: Self::Value); +} + +pub trait StaticBuilderMethods: BackendTypes { + fn get_static(&mut self, def_id: DefId) -> Self::Value; +} diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs new file mode 100644 index 00000000000..8158e8dd011 --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -0,0 +1,151 @@ +use super::misc::MiscMethods; +use super::Backend; +use super::HasCodegen; +use crate::common::TypeKind; +use crate::mir::place::PlaceRef; +use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::ty::{self, Ty}; +use rustc_span::DUMMY_SP; +use rustc_target::abi::call::{ArgAbi, CastTarget, FnAbi, Reg}; +use rustc_target::abi::{AddressSpace, Integer}; + +// This depends on `Backend` and not `BackendTypes`, because consumers will probably want to use +// `LayoutOf` or `HasTyCtxt`. This way, they don't have to add a constraint on it themselves. +pub trait BaseTypeMethods<'tcx>: Backend<'tcx> { + fn type_i1(&self) -> Self::Type; + fn type_i8(&self) -> Self::Type; + fn type_i16(&self) -> Self::Type; + fn type_i32(&self) -> Self::Type; + fn type_i64(&self) -> Self::Type; + fn type_i128(&self) -> Self::Type; + fn type_isize(&self) -> Self::Type; + + fn type_f32(&self) -> Self::Type; + fn type_f64(&self) -> Self::Type; + + fn type_func(&self, args: &[Self::Type], ret: Self::Type) -> Self::Type; + fn type_struct(&self, els: &[Self::Type], packed: bool) -> Self::Type; + fn type_kind(&self, ty: Self::Type) -> TypeKind; + fn type_ptr_to(&self, ty: Self::Type) -> Self::Type; + fn type_ptr_to_ext(&self, ty: Self::Type, address_space: AddressSpace) -> Self::Type; + fn element_type(&self, ty: Self::Type) -> Self::Type; + + /// Returns the number of elements in `self` if it is a LLVM vector type. + fn vector_length(&self, ty: Self::Type) -> usize; + + fn float_width(&self, ty: Self::Type) -> usize; + + /// Retrieves the bit width of the integer type `self`. + fn int_width(&self, ty: Self::Type) -> u64; + + fn val_ty(&self, v: Self::Value) -> Self::Type; +} + +pub trait DerivedTypeMethods<'tcx>: BaseTypeMethods<'tcx> + MiscMethods<'tcx> { + fn type_i8p(&self) -> Self::Type { + self.type_i8p_ext(AddressSpace::DATA) + } + + fn type_i8p_ext(&self, address_space: AddressSpace) -> Self::Type { + self.type_ptr_to_ext(self.type_i8(), address_space) + } + + fn type_int(&self) -> Self::Type { + match &self.sess().target.c_int_width[..] { + "16" => self.type_i16(), + "32" => self.type_i32(), + "64" => self.type_i64(), + width => bug!("Unsupported c_int_width: {}", width), + } + } + + fn type_from_integer(&self, i: Integer) -> Self::Type { + use Integer::*; + match i { + I8 => self.type_i8(), + I16 => self.type_i16(), + I32 => self.type_i32(), + I64 => self.type_i64(), + I128 => self.type_i128(), + } + } + + fn type_needs_drop(&self, ty: Ty<'tcx>) -> bool { + ty.needs_drop(self.tcx(), ty::ParamEnv::reveal_all()) + } + + fn type_is_sized(&self, ty: Ty<'tcx>) -> bool { + ty.is_sized(self.tcx().at(DUMMY_SP), ty::ParamEnv::reveal_all()) + } + + fn type_is_freeze(&self, ty: Ty<'tcx>) -> bool { + ty.is_freeze(self.tcx().at(DUMMY_SP), ty::ParamEnv::reveal_all()) + } + + fn type_has_metadata(&self, ty: Ty<'tcx>) -> bool { + let param_env = ty::ParamEnv::reveal_all(); + if ty.is_sized(self.tcx().at(DUMMY_SP), param_env) { + return false; + } + + let tail = self.tcx().struct_tail_erasing_lifetimes(ty, param_env); + match tail.kind() { + ty::Foreign(..) => false, + ty::Str | ty::Slice(..) | ty::Dynamic(..) => true, + _ => bug!("unexpected unsized tail: {:?}", tail), + } + } +} + +impl<'tcx, T> DerivedTypeMethods<'tcx> for T where Self: BaseTypeMethods<'tcx> + MiscMethods<'tcx> {} + +pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> { + fn backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type; + fn cast_backend_type(&self, ty: &CastTarget) -> Self::Type; + fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; + fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; + fn reg_backend_type(&self, ty: &Reg) -> Self::Type; + fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type; + fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool; + fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool; + fn backend_field_index(&self, layout: TyAndLayout<'tcx>, index: usize) -> u64; + fn scalar_pair_element_backend_type( + &self, + layout: TyAndLayout<'tcx>, + index: usize, + immediate: bool, + ) -> Self::Type; +} + +// For backends that support CFI using type membership (i.e., testing whether a given pointer is +// associated with a type identifier). +pub trait TypeMembershipMethods<'tcx>: Backend<'tcx> { + fn set_type_metadata(&self, function: Self::Function, typeid: String); + fn typeid_metadata(&self, typeid: String) -> Self::Value; +} + +pub trait ArgAbiMethods<'tcx>: HasCodegen<'tcx> { + fn store_fn_arg( + &mut self, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + idx: &mut usize, + dst: PlaceRef<'tcx, Self::Value>, + ); + fn store_arg( + &mut self, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + val: Self::Value, + dst: PlaceRef<'tcx, Self::Value>, + ); + fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> Self::Type; +} + +pub trait TypeMethods<'tcx>: + DerivedTypeMethods<'tcx> + LayoutTypeMethods<'tcx> + TypeMembershipMethods<'tcx> +{ +} + +impl<'tcx, T> TypeMethods<'tcx> for T where + Self: DerivedTypeMethods<'tcx> + LayoutTypeMethods<'tcx> + TypeMembershipMethods<'tcx> +{ +} diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs new file mode 100644 index 00000000000..e54ec34f1ce --- /dev/null +++ b/compiler/rustc_codegen_ssa/src/traits/write.rs @@ -0,0 +1,68 @@ +use crate::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; +use crate::back::write::{CodegenContext, FatLTOInput, ModuleConfig}; +use crate::{CompiledModule, ModuleCodegen}; + +use rustc_errors::{FatalError, Handler}; +use rustc_middle::dep_graph::WorkProduct; + +pub trait WriteBackendMethods: 'static + Sized + Clone { + type Module: Send + Sync; + type TargetMachine; + type ModuleBuffer: ModuleBufferMethods; + type Context: ?Sized; + type ThinData: Send + Sync; + type ThinBuffer: ThinBufferMethods; + + /// Merge all modules into main_module and returning it + fn run_link( + cgcx: &CodegenContext, + diag_handler: &Handler, + modules: Vec>, + ) -> Result, FatalError>; + /// Performs fat LTO by merging all modules into a single one and returning it + /// for further optimization. + fn run_fat_lto( + cgcx: &CodegenContext, + modules: Vec>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + ) -> Result, FatalError>; + /// Performs thin LTO by performing necessary global analysis and returning two + /// lists, one of the modules that need optimization and another for modules that + /// can simply be copied over from the incr. comp. cache. + fn run_thin_lto( + cgcx: &CodegenContext, + modules: Vec<(String, Self::ThinBuffer)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + ) -> Result<(Vec>, Vec), FatalError>; + fn print_pass_timings(&self); + unsafe fn optimize( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: &ModuleCodegen, + config: &ModuleConfig, + ) -> Result<(), FatalError>; + fn optimize_fat( + cgcx: &CodegenContext, + llmod: &mut ModuleCodegen, + ) -> Result<(), FatalError>; + unsafe fn optimize_thin( + cgcx: &CodegenContext, + thin: ThinModule, + ) -> Result, FatalError>; + unsafe fn codegen( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: ModuleCodegen, + config: &ModuleConfig, + ) -> Result; + fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer); + fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer); +} + +pub trait ThinBufferMethods: Send + Sync { + fn data(&self) -> &[u8]; +} + +pub trait ModuleBufferMethods: Send + Sync { + fn data(&self) -> &[u8]; +} From baaa94ce3a85b83e054cd5464b6b01d3e3f6345a Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 6 Dec 2023 02:49:01 +0000 Subject: [PATCH 08/70] auto formatting --- compiler/rustc_llvm/src/lib.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs index 8eade02a408..cc712da5ef0 100644 --- a/compiler/rustc_llvm/src/lib.rs +++ b/compiler/rustc_llvm/src/lib.rs @@ -23,11 +23,7 @@ impl RustString { /// Appending to a Rust string -- used by RawRustStringOstream. #[no_mangle] -pub unsafe extern "C" fn LLVMRustStringWriteImpl( - sr: &RustString, - ptr: *const c_char, - size: size_t, -) { +pub unsafe extern "C" fn LLVMRustStringWriteImpl(sr: &RustString, ptr: *const c_char, size: size_t) { let slice = slice::from_raw_parts(ptr as *const u8, size as usize); sr.bytes.borrow_mut().extend_from_slice(slice); From 25a9eed0298ee8b8d7c93e5b5c052a90b25f7343 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 6 Dec 2023 02:50:33 +0000 Subject: [PATCH 09/70] coverage interfaces in rust --- compiler/rustc_llvm_coverage/Cargo.toml | 10 + compiler/rustc_llvm_coverage/src/ffi.rs | 53 ++++ compiler/rustc_llvm_coverage/src/lib.rs | 130 +++++++++ compiler/rustc_llvm_coverage/src/types.rs | 307 ++++++++++++++++++++++ 4 files changed, 500 insertions(+) create mode 100644 compiler/rustc_llvm_coverage/Cargo.toml create mode 100644 compiler/rustc_llvm_coverage/src/ffi.rs create mode 100644 compiler/rustc_llvm_coverage/src/lib.rs create mode 100644 compiler/rustc_llvm_coverage/src/types.rs diff --git a/compiler/rustc_llvm_coverage/Cargo.toml b/compiler/rustc_llvm_coverage/Cargo.toml new file mode 100644 index 00000000000..5248cb58f9b --- /dev/null +++ b/compiler/rustc_llvm_coverage/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rustc_llvm_coverage" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +libc = "0.2" +rustc_llvm = { path = "../rustc_llvm" } diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs new file mode 100644 index 00000000000..ac06f94cd97 --- /dev/null +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -0,0 +1,53 @@ +#![allow(dead_code, unused_variables)] + +// This FFI links to the static library built by `rustc_llvm` +// +// Function interface definitions are taken from [here](https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_llvm/src/llvm/ffi.rs#L1864). +// +use libc::{c_char, c_uint, c_void, size_t}; + +use rustc_llvm::RustString; + +use super::types::*; + +#[repr(C)] +pub struct Module(c_void); + +#[repr(C)] +pub struct Value(c_void); + +#[link(name = "llvm-wrapper")] +extern "C" { + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( + Filenames: *const *const c_char, + FilenamesLen: size_t, + BufferOut: &RustString, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMappingToBuffer( + VirtualFileMappingIDs: *const c_uint, + NumVirtualFileMappingIDs: c_uint, + Expressions: *const CounterExpression, + NumExpressions: c_uint, + MappingRegions: *const CounterMappingRegion, + NumMappingRegions: c_uint, + BufferOut: &RustString, + ); + + pub fn LLVMRustCoverageCreatePGOFuncNameVar(F: &Value, FuncName: *const c_char) -> &Value; + pub fn LLVMRustCoverageHashCString(StrVal: *const c_char) -> u64; + pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); + + pub fn LLVMRustCoverageMappingVersion() -> u32; +} diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs new file mode 100644 index 00000000000..977296498ec --- /dev/null +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -0,0 +1,130 @@ +// use rustc_llvm::RustString; + +mod ffi; +mod types; + +// use crate::abi::Abi; +// use crate::llvm; +// use libc::c_uint; + +// use std::ffi::CString; + +// /* +// * Many of the functions in this file have been adapted from the +// * `rustc` implementation of LLVM code coverage. +// * +// * https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs#L220-L221 +// */ +// /// Calls llvm::createPGOFuncNameVar() with the given function instance's +// /// mangled function name. The LLVM API returns an llvm::GlobalVariable +// /// containing the function name, with the specific variable name and linkage +// /// required by LLVM InstrProf source-based coverage instrumentation. Use +// /// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per +// /// `Instance`. +// // fn create_pgo_func_name_var<'ll, 'tcx>( +// // cx: &CodegenCx<'ll, 'tcx>, +// // instance: Instance<'tcx>, +// // ) -> &'ll llvm::Value { +// // let mangled_fn_name = +// // CString::new(cx.tcx.symbol_name(instance).name).expect("error converting function name to C string"); +// // let llfn = cx.get_fn(instance); +// // unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } +// // } + +// pub(crate) fn write_filenames_section_to_buffer<'a>( +// filenames: impl IntoIterator, +// buffer: &RustString, +// ) { +// let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::>(); +// unsafe { +// llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer(c_str_vec.as_ptr(), c_str_vec.len(), buffer); +// } +// } + +// pub(crate) fn write_mapping_to_buffer( +// virtual_file_mapping: Vec, +// expressions: Vec, +// mapping_regions: Vec, +// buffer: &RustString, +// ) { +// unsafe { +// llvm::LLVMRustCoverageWriteMappingToBuffer( +// virtual_file_mapping.as_ptr(), +// virtual_file_mapping.len() as c_uint, +// expressions.as_ptr(), +// expressions.len() as c_uint, +// mapping_regions.as_ptr(), +// mapping_regions.len() as c_uint, +// buffer, +// ); +// } +// } + +// pub(crate) fn hash_str(strval: &str) -> u64 { +// let strval = CString::new(strval).expect("null error converting hashable str to C string"); +// unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } +// } + +// pub(crate) fn hash_bytes(bytes: Vec) -> u64 { +// unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } +// } + +pub fn mapping_version() -> u32 { + unsafe { ffi::LLVMRustCoverageMappingVersion() } +} + +// pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, cov_data_val: &'ll llvm::Value) { +// let covmap_var_name = llvm::build_string(|s| unsafe { +// llvm::LLVMRustCoverageWriteMappingVarNameToString(s); +// }) +// .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); +// debug!("covmap var name: {:?}", covmap_var_name); + +// let covmap_section_name = llvm::build_string(|s| unsafe { +// llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); +// }) +// .expect("Rust Coverage section name failed UTF-8 conversion"); +// debug!("covmap section name: {:?}", covmap_section_name); + +// let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); +// llvm::set_initializer(llglobal, cov_data_val); +// llvm::set_global_constant(llglobal, true); +// llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); +// llvm::set_section(llglobal, &covmap_section_name); +// llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); +// cx.add_used_global(llglobal); +// } + +// pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( +// cx: &CodegenCx<'ll, 'tcx>, +// func_name_hash: u64, +// func_record_val: &'ll llvm::Value, +// is_used: bool, +// ) { +// // Assign a name to the function record. This is used to merge duplicates. +// // +// // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that +// // are included-but-not-used. If (or when) Rust generates functions that are +// // included-but-not-used, note that a dummy description for a function included-but-not-used +// // in a Crate can be replaced by full description provided by a different Crate. The two kinds +// // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by +// // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. +// let func_record_var_name = format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); +// debug!("function record var name: {:?}", func_record_var_name); + +// let func_record_section_name = llvm::build_string(|s| unsafe { +// llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); +// }) +// .expect("Rust Coverage function record section name failed UTF-8 conversion"); +// debug!("function record section name: {:?}", func_record_section_name); + +// let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); +// llvm::set_initializer(llglobal, func_record_val); +// llvm::set_global_constant(llglobal, true); +// llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); +// llvm::set_visibility(llglobal, llvm::Visibility::Hidden); +// llvm::set_section(llglobal, &func_record_section_name); +// llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); +// llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); +// cx.add_used_global(llglobal); +// } diff --git a/compiler/rustc_llvm_coverage/src/types.rs b/compiler/rustc_llvm_coverage/src/types.rs new file mode 100644 index 00000000000..3318ce5a6cb --- /dev/null +++ b/compiler/rustc_llvm_coverage/src/types.rs @@ -0,0 +1,307 @@ +#![allow(dead_code, unused_variables)] + +// These data structures provide definitions for the wrapped C LLVM interface. +// +// These type definitions are taken from: +// - [`rustc_codegen_ssa/src/coverageinfo/ffi.rs`](https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs#L4-L5) +// - [`rustc_codegen_llvm/src/coverageinfo/ffi.rs`](https://github.com/rust-lang/rust/blob/56278a6e2824acc96b222e5816bf2d74e85dab93/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs#L4) +// - [`rustc_middle/src/mir/coverage.rs`](https://github.com/rust-lang/rust/blob/56278a6e2824acc96b222e5816bf2d74e85dab93/compiler/rustc_middle/src/mir/coverage.rs#L9) +// +#[derive(Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct CounterId(u32); + +impl CounterId { + pub const START: Self = Self(0); + + pub fn new(value: u32) -> Self { + CounterId(value) + } + + pub fn from_u32(value: u32) -> Self { + CounterId::new(value) + } + + pub fn as_u32(&self) -> u32 { + self.0 + } +} + +#[derive(Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct ExpressionId(u32); + +impl ExpressionId { + pub const START: Self = Self(0); + + pub fn new(value: u32) -> Self { + ExpressionId(value) + } + + pub fn from_u32(value: u32) -> Self { + ExpressionId::new(value) + } + + pub fn as_u32(&self) -> u32 { + self.0 + } +} + +/// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. +/// +/// Must match the layout of `LLVMRustCounterMappingRegionKind`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub enum RegionKind { + /// A CodeRegion associates some code with a counter + CodeRegion = 0, + + /// An ExpansionRegion represents a file expansion region that associates + /// a source range with the expansion of a virtual source file, such as + /// for a macro instantiation or #include file. + ExpansionRegion = 1, + + /// A SkippedRegion represents a source range with code that was skipped + /// by a preprocessor or similar means. + SkippedRegion = 2, + + /// A GapRegion is like a CodeRegion, but its count is only set as the + /// line execution count when its the only region in the line. + GapRegion = 3, + + /// A BranchRegion represents leaf-level boolean expressions and is + /// associated with two counters, each representing the number of times the + /// expression evaluates to true or false. + BranchRegion = 4, +} + +/// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the +/// coverage map, in accordance with the +/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). +/// The struct composes fields representing the `Counter` type and value(s) (injected counter +/// ID, or expression type and operands), the source file (an indirect index into a "filenames +/// array", encoded separately), and source location (start and end positions of the represented +/// code region). +/// +/// Corresponds to struct `llvm::coverage::CounterMappingRegion`. +/// +/// Must match the layout of `LLVMRustCounterMappingRegion`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub struct CounterMappingRegion { + /// The counter type and type-dependent counter data, if any. + counter: Counter, + + /// If the `RegionKind` is a `BranchRegion`, this represents the counter + /// for the false branch of the region. + false_counter: Counter, + + /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the + /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes + /// that, in turn, are used to look up the filename for this region. + file_id: u32, + + /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find + /// the mapping regions created as a result of macro expansion, by checking if their file id + /// matches the expanded file id. + expanded_file_id: u32, + + /// 1-based starting line of the mapping region. + start_line: u32, + + /// 1-based starting column of the mapping region. + start_col: u32, + + /// 1-based ending line of the mapping region. + end_line: u32, + + /// 1-based ending column of the mapping region. If the high bit is set, the current + /// mapping region is a gap area. + end_col: u32, + + kind: RegionKind, +} + +impl CounterMappingRegion { + pub(crate) fn code_region( + counter: Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter: Counter::ZERO, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::CodeRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn branch_region( + counter: Counter, + false_counter: Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::BranchRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn expansion_region( + file_id: u32, + expanded_file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter: Counter::ZERO, + false_counter: Counter::ZERO, + file_id, + expanded_file_id, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::ExpansionRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn skipped_region( + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter: Counter::ZERO, + false_counter: Counter::ZERO, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::SkippedRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn gap_region( + counter: Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter: Counter::ZERO, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col: (1_u32 << 31) | end_col, + kind: RegionKind::GapRegion, + } + } +} + +/// Aligns with [llvm::coverage::Counter::CounterKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L95) +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub enum CounterKind { + Zero = 0, + CounterValueReference = 1, + Expression = 2, +} + +/// A reference to an instance of an abstract "counter" that will yield a value in a coverage +/// report. Note that `id` has different interpretations, depending on the `kind`: +/// * For `CounterKind::Zero`, `id` is assumed to be `0` +/// * For `CounterKind::CounterValueReference`, `id` matches the `counter_id` of the injected +/// instrumentation counter (the `index` argument to the LLVM intrinsic +/// `instrprof.increment()`) +/// * For `CounterKind::Expression`, `id` is the index into the coverage map's array of +/// counter expressions. +/// Aligns with [llvm::coverage::Counter](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L102-L103) +/// Important: The Rust struct layout (order and types of fields) must match its C++ counterpart. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub struct Counter { + // Important: The layout (order and types of fields) must match its C++ counterpart. + pub kind: CounterKind, + id: u32, +} + +impl Counter { + /// A `Counter` of kind `Zero`. For this counter kind, the `id` is not used. + pub(crate) const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 }; + + /// Constructs a new `Counter` of kind `CounterValueReference`. + pub fn counter_value_reference(counter_id: CounterId) -> Self { + Self { kind: CounterKind::CounterValueReference, id: counter_id.as_u32() } + } + + /// Constructs a new `Counter` of kind `Expression`. + pub(crate) fn expression(expression_id: ExpressionId) -> Self { + Self { kind: CounterKind::Expression, id: expression_id.as_u32() } + } +} + +/// Aligns with [llvm::coverage::CounterExpression::ExprKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L150) +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub enum ExprKind { + Subtract = 0, + Add = 1, +} + +/// Aligns with [llvm::coverage::CounterExpression](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L152) +/// Important: The Rust struct layout (order and types of fields) must match its C++ +/// counterpart. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub struct CounterExpression { + pub kind: ExprKind, + pub lhs: Counter, + pub rhs: Counter, +} + +impl CounterExpression { + pub fn new(lhs: Counter, kind: ExprKind, rhs: Counter) -> Self { + Self { kind, lhs, rhs } + } +} From 20c5941c81f446e59f227cadec74efeb65489e86 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 6 Dec 2023 02:52:23 +0000 Subject: [PATCH 10/70] remove first attempt files --- compiler/rustc_codegen_llvm/Cargo.toml | 36 - compiler/rustc_codegen_llvm/README.md | 7 - compiler/rustc_codegen_llvm/src/abi.rs | 599 ---- compiler/rustc_codegen_llvm/src/allocator.rs | 157 - compiler/rustc_codegen_llvm/src/asm.rs | 1037 ------ compiler/rustc_codegen_llvm/src/attributes.rs | 449 --- .../rustc_codegen_llvm/src/back/archive.rs | 383 --- compiler/rustc_codegen_llvm/src/back/lto.rs | 936 ------ .../rustc_codegen_llvm/src/back/profiling.rs | 58 - compiler/rustc_codegen_llvm/src/back/write.rs | 1212 ------- compiler/rustc_codegen_llvm/src/base.rs | 173 - compiler/rustc_codegen_llvm/src/builder.rs | 1508 --------- compiler/rustc_codegen_llvm/src/callee.rs | 194 -- compiler/rustc_codegen_llvm/src/common.rs | 359 --- compiler/rustc_codegen_llvm/src/consts.rs | 577 ---- compiler/rustc_codegen_llvm/src/context.rs | 1014 ------ .../src/coverageinfo/mapgen.rs | 334 -- .../src/coverageinfo/mod.rs | 385 --- .../src/debuginfo/create_scope_map.rs | 126 - .../rustc_codegen_llvm/src/debuginfo/doc.md | 131 - .../rustc_codegen_llvm/src/debuginfo/gdb.rs | 120 - .../src/debuginfo/metadata.rs | 1618 ---------- .../src/debuginfo/metadata/enums/cpp_like.rs | 514 --- .../src/debuginfo/metadata/enums/mod.rs | 437 --- .../src/debuginfo/metadata/enums/native.rs | 441 --- .../src/debuginfo/metadata/type_map.rs | 267 -- .../rustc_codegen_llvm/src/debuginfo/mod.rs | 614 ---- .../src/debuginfo/namespace.rs | 48 - .../rustc_codegen_llvm/src/debuginfo/utils.rs | 99 - compiler/rustc_codegen_llvm/src/declare.rs | 146 - compiler/rustc_codegen_llvm/src/intrinsic.rs | 1924 ----------- compiler/rustc_codegen_llvm/src/lib.rs | 442 --- .../rustc_codegen_llvm/src/llvm/archive_ro.rs | 105 - .../rustc_codegen_llvm/src/llvm/diagnostic.rs | 213 -- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2547 --------------- compiler/rustc_codegen_llvm/src/llvm/mod.rs | 318 -- compiler/rustc_codegen_llvm/src/llvm_util.rs | 562 ---- compiler/rustc_codegen_llvm/src/mono_item.rs | 150 - compiler/rustc_codegen_llvm/src/type_.rs | 319 -- compiler/rustc_codegen_llvm/src/type_of.rs | 418 --- compiler/rustc_codegen_llvm/src/va_arg.rs | 214 -- compiler/rustc_codegen_llvm/src/value.rs | 32 - compiler/rustc_codegen_ssa/Cargo.toml | 48 - compiler/rustc_codegen_ssa/README.md | 3 - .../rustc_codegen_ssa/src/back/archive.rs | 69 - .../rustc_codegen_ssa/src/back/command.rs | 178 -- compiler/rustc_codegen_ssa/src/back/link.rs | 2800 ----------------- compiler/rustc_codegen_ssa/src/back/linker.rs | 1788 ----------- compiler/rustc_codegen_ssa/src/back/lto.rs | 104 - .../rustc_codegen_ssa/src/back/metadata.rs | 314 -- compiler/rustc_codegen_ssa/src/back/mod.rs | 9 - compiler/rustc_codegen_ssa/src/back/rpath.rs | 114 - .../rustc_codegen_ssa/src/back/rpath/tests.rs | 72 - .../src/back/symbol_export.rs | 590 ---- compiler/rustc_codegen_ssa/src/back/write.rs | 2015 ------------ compiler/rustc_codegen_ssa/src/base.rs | 961 ------ compiler/rustc_codegen_ssa/src/common.rs | 223 -- .../rustc_codegen_ssa/src/coverageinfo/ffi.rs | 85 - .../rustc_codegen_ssa/src/coverageinfo/map.rs | 347 -- .../rustc_codegen_ssa/src/coverageinfo/mod.rs | 2 - .../rustc_codegen_ssa/src/debuginfo/mod.rs | 34 - .../src/debuginfo/type_names.rs | 821 ----- compiler/rustc_codegen_ssa/src/glue.rs | 123 - compiler/rustc_codegen_ssa/src/lib.rs | 244 -- compiler/rustc_codegen_ssa/src/meth.rs | 116 - compiler/rustc_codegen_ssa/src/mir/analyze.rs | 368 --- compiler/rustc_codegen_ssa/src/mir/block.rs | 1654 ---------- .../rustc_codegen_ssa/src/mir/constant.rs | 90 - .../rustc_codegen_ssa/src/mir/coverageinfo.rs | 55 - .../rustc_codegen_ssa/src/mir/debuginfo.rs | 418 --- .../rustc_codegen_ssa/src/mir/intrinsic.rs | 636 ---- compiler/rustc_codegen_ssa/src/mir/mod.rs | 410 --- compiler/rustc_codegen_ssa/src/mir/operand.rs | 461 --- compiler/rustc_codegen_ssa/src/mir/place.rs | 549 ---- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 729 ----- .../rustc_codegen_ssa/src/mir/statement.rs | 102 - compiler/rustc_codegen_ssa/src/mono_item.rs | 147 - .../rustc_codegen_ssa/src/target_features.rs | 308 -- compiler/rustc_codegen_ssa/src/traits/abi.rs | 8 - compiler/rustc_codegen_ssa/src/traits/asm.rs | 66 - .../rustc_codegen_ssa/src/traits/backend.rs | 161 - .../rustc_codegen_ssa/src/traits/builder.rs | 481 --- .../rustc_codegen_ssa/src/traits/consts.rs | 41 - .../src/traits/coverageinfo.rs | 57 - .../rustc_codegen_ssa/src/traits/debuginfo.rs | 79 - .../rustc_codegen_ssa/src/traits/declare.rs | 21 - .../rustc_codegen_ssa/src/traits/intrinsic.rs | 39 - compiler/rustc_codegen_ssa/src/traits/misc.rs | 26 - compiler/rustc_codegen_ssa/src/traits/mod.rs | 102 - .../rustc_codegen_ssa/src/traits/statics.rs | 24 - .../rustc_codegen_ssa/src/traits/type_.rs | 151 - .../rustc_codegen_ssa/src/traits/write.rs | 68 - 92 files changed, 39534 deletions(-) delete mode 100644 compiler/rustc_codegen_llvm/Cargo.toml delete mode 100644 compiler/rustc_codegen_llvm/README.md delete mode 100644 compiler/rustc_codegen_llvm/src/abi.rs delete mode 100644 compiler/rustc_codegen_llvm/src/allocator.rs delete mode 100644 compiler/rustc_codegen_llvm/src/asm.rs delete mode 100644 compiler/rustc_codegen_llvm/src/attributes.rs delete mode 100644 compiler/rustc_codegen_llvm/src/back/archive.rs delete mode 100644 compiler/rustc_codegen_llvm/src/back/lto.rs delete mode 100644 compiler/rustc_codegen_llvm/src/back/profiling.rs delete mode 100644 compiler/rustc_codegen_llvm/src/back/write.rs delete mode 100644 compiler/rustc_codegen_llvm/src/base.rs delete mode 100644 compiler/rustc_codegen_llvm/src/builder.rs delete mode 100644 compiler/rustc_codegen_llvm/src/callee.rs delete mode 100644 compiler/rustc_codegen_llvm/src/common.rs delete mode 100644 compiler/rustc_codegen_llvm/src/consts.rs delete mode 100644 compiler/rustc_codegen_llvm/src/context.rs delete mode 100644 compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs delete mode 100644 compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/doc.md delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/mod.rs delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs delete mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/utils.rs delete mode 100644 compiler/rustc_codegen_llvm/src/declare.rs delete mode 100644 compiler/rustc_codegen_llvm/src/intrinsic.rs delete mode 100644 compiler/rustc_codegen_llvm/src/lib.rs delete mode 100644 compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs delete mode 100644 compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs delete mode 100644 compiler/rustc_codegen_llvm/src/llvm/ffi.rs delete mode 100644 compiler/rustc_codegen_llvm/src/llvm/mod.rs delete mode 100644 compiler/rustc_codegen_llvm/src/llvm_util.rs delete mode 100644 compiler/rustc_codegen_llvm/src/mono_item.rs delete mode 100644 compiler/rustc_codegen_llvm/src/type_.rs delete mode 100644 compiler/rustc_codegen_llvm/src/type_of.rs delete mode 100644 compiler/rustc_codegen_llvm/src/va_arg.rs delete mode 100644 compiler/rustc_codegen_llvm/src/value.rs delete mode 100644 compiler/rustc_codegen_ssa/Cargo.toml delete mode 100644 compiler/rustc_codegen_ssa/README.md delete mode 100644 compiler/rustc_codegen_ssa/src/back/archive.rs delete mode 100644 compiler/rustc_codegen_ssa/src/back/command.rs delete mode 100644 compiler/rustc_codegen_ssa/src/back/link.rs delete mode 100644 compiler/rustc_codegen_ssa/src/back/linker.rs delete mode 100644 compiler/rustc_codegen_ssa/src/back/lto.rs delete mode 100644 compiler/rustc_codegen_ssa/src/back/metadata.rs delete mode 100644 compiler/rustc_codegen_ssa/src/back/mod.rs delete mode 100644 compiler/rustc_codegen_ssa/src/back/rpath.rs delete mode 100644 compiler/rustc_codegen_ssa/src/back/rpath/tests.rs delete mode 100644 compiler/rustc_codegen_ssa/src/back/symbol_export.rs delete mode 100644 compiler/rustc_codegen_ssa/src/back/write.rs delete mode 100644 compiler/rustc_codegen_ssa/src/base.rs delete mode 100644 compiler/rustc_codegen_ssa/src/common.rs delete mode 100644 compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs delete mode 100644 compiler/rustc_codegen_ssa/src/coverageinfo/map.rs delete mode 100644 compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs delete mode 100644 compiler/rustc_codegen_ssa/src/debuginfo/mod.rs delete mode 100644 compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs delete mode 100644 compiler/rustc_codegen_ssa/src/glue.rs delete mode 100644 compiler/rustc_codegen_ssa/src/lib.rs delete mode 100644 compiler/rustc_codegen_ssa/src/meth.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/analyze.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/block.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/constant.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/debuginfo.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/intrinsic.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/mod.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/operand.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/place.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/rvalue.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mir/statement.rs delete mode 100644 compiler/rustc_codegen_ssa/src/mono_item.rs delete mode 100644 compiler/rustc_codegen_ssa/src/target_features.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/abi.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/asm.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/backend.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/builder.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/consts.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/debuginfo.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/declare.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/intrinsic.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/misc.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/mod.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/statics.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/type_.rs delete mode 100644 compiler/rustc_codegen_ssa/src/traits/write.rs diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml deleted file mode 100644 index f9a5463efcd..00000000000 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ /dev/null @@ -1,36 +0,0 @@ -[package] -name = "rustc_codegen_llvm" -version = "0.0.0" -edition = "2021" - -[lib] -test = false -doctest = false - -[dependencies] -bitflags = "1.0" -cstr = "0.2" -libc = "0.2" -libloading = "0.7.1" -measureme = "10.0.0" -tracing = "0.1" -rustc_middle = { path = "../rustc_middle" } -rustc-demangle = "0.1.21" -rustc_attr = { path = "../rustc_attr" } -rustc_codegen_ssa = { path = "../rustc_codegen_ssa" } -rustc_data_structures = { path = "../rustc_data_structures" } -rustc_errors = { path = "../rustc_errors" } -rustc_fs_util = { path = "../rustc_fs_util" } -rustc_hir = { path = "../rustc_hir" } -rustc_index = { path = "../rustc_index" } -rustc_llvm = { path = "../rustc_llvm" } -rustc_macros = { path = "../rustc_macros" } -rustc_metadata = { path = "../rustc_metadata" } -rustc_query_system = { path = "../rustc_query_system" } -rustc_session = { path = "../rustc_session" } -rustc_serialize = { path = "../rustc_serialize" } -rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } -rustc_target = { path = "../rustc_target" } -smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } -rustc_ast = { path = "../rustc_ast" } -rustc_span = { path = "../rustc_span" } diff --git a/compiler/rustc_codegen_llvm/README.md b/compiler/rustc_codegen_llvm/README.md deleted file mode 100644 index afec60d017e..00000000000 --- a/compiler/rustc_codegen_llvm/README.md +++ /dev/null @@ -1,7 +0,0 @@ -The `codegen` crate contains the code to convert from MIR into LLVM IR, -and then from LLVM IR into machine code. In general it contains code -that runs towards the end of the compilation process. - -For more information about how codegen works, see the [rustc dev guide]. - -[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/backend/codegen.html diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs deleted file mode 100644 index 9eb3574e77b..00000000000 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ /dev/null @@ -1,599 +0,0 @@ -use crate::attributes; -use crate::builder::Builder; -use crate::context::CodegenCx; -use crate::llvm::{self, Attribute, AttributePlace}; -use crate::type_::Type; -use crate::type_of::LayoutLlvmExt; -use crate::value::Value; - -use rustc_codegen_ssa::mir::operand::OperandValue; -use rustc_codegen_ssa::mir::place::PlaceRef; -use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::MemFlags; -use rustc_middle::bug; -use rustc_middle::ty::layout::LayoutOf; -pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; -use rustc_middle::ty::Ty; -use rustc_session::config; -use rustc_target::abi::call::ArgAbi; -pub use rustc_target::abi::call::*; -use rustc_target::abi::{self, HasDataLayout, Int}; -pub use rustc_target::spec::abi::Abi; - -use libc::c_uint; -use smallvec::SmallVec; - -pub trait ArgAttributesExt { - fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); - fn apply_attrs_to_callsite( - &self, - idx: AttributePlace, - cx: &CodegenCx<'_, '_>, - callsite: &Value, - ); -} - -fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { - // LLVM prior to version 12 had known miscompiles in the presence of - // noalias attributes (see #54878), but we don't support earlier - // versions at all anymore. We now enable mutable noalias by default. - cx.tcx.sess.opts.unstable_opts.mutable_noalias.unwrap_or(true) -} - -const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = - [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; - -const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [ - (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), - (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture), - (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), - (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), - (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), -]; - -fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> { - let mut regular = this.regular; - - let mut attrs = SmallVec::new(); - - // ABI-affecting attributes must always be applied - for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES { - if regular.contains(attr) { - attrs.push(llattr.create_attr(cx.llcx)); - } - } - if let Some(align) = this.pointee_align { - attrs.push(llvm::CreateAlignmentAttr(cx.llcx, align.bytes())); - } - match this.arg_ext { - ArgExtension::None => {} - ArgExtension::Zext => attrs.push(llvm::AttributeKind::ZExt.create_attr(cx.llcx)), - ArgExtension::Sext => attrs.push(llvm::AttributeKind::SExt.create_attr(cx.llcx)), - } - - // Only apply remaining attributes when optimizing - if cx.sess().opts.optimize != config::OptLevel::No { - let deref = this.pointee_size.bytes(); - if deref != 0 { - if regular.contains(ArgAttribute::NonNull) { - attrs.push(llvm::CreateDereferenceableAttr(cx.llcx, deref)); - } else { - attrs.push(llvm::CreateDereferenceableOrNullAttr(cx.llcx, deref)); - } - regular -= ArgAttribute::NonNull; - } - for (attr, llattr) in OPTIMIZATION_ATTRIBUTES { - if regular.contains(attr) { - attrs.push(llattr.create_attr(cx.llcx)); - } - } - if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { - attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx)); - } - } - - attrs -} - -impl ArgAttributesExt for ArgAttributes { - fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) { - let attrs = get_attrs(self, cx); - attributes::apply_to_llfn(llfn, idx, &attrs); - } - - fn apply_attrs_to_callsite( - &self, - idx: AttributePlace, - cx: &CodegenCx<'_, '_>, - callsite: &Value, - ) { - let attrs = get_attrs(self, cx); - attributes::apply_to_callsite(callsite, idx, &attrs); - } -} - -pub trait LlvmType { - fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type; -} - -impl LlvmType for Reg { - fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { - match self.kind { - RegKind::Integer => cx.type_ix(self.size.bits()), - RegKind::Float => match self.size.bits() { - 32 => cx.type_f32(), - 64 => cx.type_f64(), - _ => bug!("unsupported float: {:?}", self), - }, - RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()), - } - } -} - -impl LlvmType for CastTarget { - fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { - let rest_ll_unit = self.rest.unit.llvm_type(cx); - let (rest_count, rem_bytes) = if self.rest.unit.size.bytes() == 0 { - (0, 0) - } else { - ( - self.rest.total.bytes() / self.rest.unit.size.bytes(), - self.rest.total.bytes() % self.rest.unit.size.bytes(), - ) - }; - - if self.prefix.iter().all(|x| x.is_none()) { - // Simplify to a single unit when there is no prefix and size <= unit size - if self.rest.total <= self.rest.unit.size { - return rest_ll_unit; - } - - // Simplify to array when all chunks are the same size and type - if rem_bytes == 0 { - return cx.type_array(rest_ll_unit, rest_count); - } - } - - // Create list of fields in the main structure - let mut args: Vec<_> = self - .prefix - .iter() - .flat_map(|option_reg| option_reg.map(|reg| reg.llvm_type(cx))) - .chain((0..rest_count).map(|_| rest_ll_unit)) - .collect(); - - // Append final integer - if rem_bytes != 0 { - // Only integers can be really split further. - assert_eq!(self.rest.unit.kind, RegKind::Integer); - args.push(cx.type_ix(rem_bytes * 8)); - } - - cx.type_struct(&args, false) - } -} - -pub trait ArgAbiExt<'ll, 'tcx> { - fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; - fn store( - &self, - bx: &mut Builder<'_, 'll, 'tcx>, - val: &'ll Value, - dst: PlaceRef<'tcx, &'ll Value>, - ); - fn store_fn_arg( - &self, - bx: &mut Builder<'_, 'll, 'tcx>, - idx: &mut usize, - dst: PlaceRef<'tcx, &'ll Value>, - ); -} - -impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { - /// Gets the LLVM type for a place of the original Rust type of - /// this argument/return, i.e., the result of `type_of::type_of`. - fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { - self.layout.llvm_type(cx) - } - - /// Stores a direct/indirect value described by this ArgAbi into a - /// place for the original Rust type of this argument/return. - /// Can be used for both storing formal arguments into Rust variables - /// or results of call/invoke instructions into their destinations. - fn store( - &self, - bx: &mut Builder<'_, 'll, 'tcx>, - val: &'ll Value, - dst: PlaceRef<'tcx, &'ll Value>, - ) { - if self.is_ignore() { - return; - } - if self.is_sized_indirect() { - OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) - } else if self.is_unsized_indirect() { - bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); - } else if let PassMode::Cast(cast) = self.mode { - // FIXME(eddyb): Figure out when the simpler Store is safe, clang - // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. - let can_store_through_cast_ptr = false; - if can_store_through_cast_ptr { - let cast_ptr_llty = bx.type_ptr_to(cast.llvm_type(bx)); - let cast_dst = bx.pointercast(dst.llval, cast_ptr_llty); - bx.store(val, cast_dst, self.layout.align.abi); - } else { - // The actual return type is a struct, but the ABI - // adaptation code has cast it into some scalar type. The - // code that follows is the only reliable way I have - // found to do a transform like i64 -> {i32,i32}. - // Basically we dump the data onto the stack then memcpy it. - // - // Other approaches I tried: - // - Casting rust ret pointer to the foreign type and using Store - // is (a) unsafe if size of foreign type > size of rust type and - // (b) runs afoul of strict aliasing rules, yielding invalid - // assembly under -O (specifically, the store gets removed). - // - Truncating foreign type to correct integral type and then - // bitcasting to the struct type yields invalid cast errors. - - // We instead thus allocate some scratch space... - let scratch_size = cast.size(bx); - let scratch_align = cast.align(bx); - let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); - bx.lifetime_start(llscratch, scratch_size); - - // ... where we first store the value... - bx.store(val, llscratch, scratch_align); - - // ... and then memcpy it to the intended destination. - bx.memcpy( - dst.llval, - self.layout.align.abi, - llscratch, - scratch_align, - bx.const_usize(self.layout.size.bytes()), - MemFlags::empty(), - ); - - bx.lifetime_end(llscratch, scratch_size); - } - } else { - OperandValue::Immediate(val).store(bx, dst); - } - } - - fn store_fn_arg( - &self, - bx: &mut Builder<'_, 'll, 'tcx>, - idx: &mut usize, - dst: PlaceRef<'tcx, &'ll Value>, - ) { - let mut next = || { - let val = llvm::get_param(bx.llfn(), *idx as c_uint); - *idx += 1; - val - }; - match self.mode { - PassMode::Ignore => {} - PassMode::Pair(..) => { - OperandValue::Pair(next(), next()).store(bx, dst); - } - PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { - OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst); - } - PassMode::Direct(_) - | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } - | PassMode::Cast(_) => { - let next_arg = next(); - self.store(bx, next_arg, dst); - } - } - } -} - -impl<'ll, 'tcx> ArgAbiMethods<'tcx> for Builder<'_, 'll, 'tcx> { - fn store_fn_arg( - &mut self, - arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, - idx: &mut usize, - dst: PlaceRef<'tcx, Self::Value>, - ) { - arg_abi.store_fn_arg(self, idx, dst) - } - fn store_arg( - &mut self, - arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, - val: &'ll Value, - dst: PlaceRef<'tcx, &'ll Value>, - ) { - arg_abi.store(self, val, dst) - } - fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> &'ll Type { - arg_abi.memory_ty(self) - } -} - -pub trait FnAbiLlvmExt<'ll, 'tcx> { - fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; - fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; - fn llvm_cconv(&self) -> llvm::CallConv; - fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value); - fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value); -} - -impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { - fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { - // Ignore "extra" args from the call site for C variadic functions. - // Only the "fixed" args are part of the LLVM function signature. - let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; - - let args_capacity: usize = args.iter().map(|arg| - if arg.pad.is_some() { 1 } else { 0 } + - if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } - ).sum(); - let mut llargument_tys = Vec::with_capacity( - if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, - ); - - let llreturn_ty = match self.ret.mode { - PassMode::Ignore => cx.type_void(), - PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), - PassMode::Cast(cast) => cast.llvm_type(cx), - PassMode::Indirect { .. } => { - llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); - cx.type_void() - } - }; - - for arg in args { - // add padding - if let Some(ty) = arg.pad { - llargument_tys.push(ty.llvm_type(cx)); - } - - let llarg_ty = match arg.mode { - PassMode::Ignore => continue, - PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), - PassMode::Pair(..) => { - llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true)); - llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); - continue; - } - PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { - let ptr_ty = cx.tcx.mk_mut_ptr(arg.layout.ty); - let ptr_layout = cx.layout_of(ptr_ty); - llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); - llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); - continue; - } - PassMode::Cast(cast) => cast.llvm_type(cx), - PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { - cx.type_ptr_to(arg.memory_ty(cx)) - } - }; - llargument_tys.push(llarg_ty); - } - - if self.c_variadic { - cx.type_variadic_func(&llargument_tys, llreturn_ty) - } else { - cx.type_func(&llargument_tys, llreturn_ty) - } - } - - fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { - unsafe { - llvm::LLVMPointerType( - self.llvm_type(cx), - cx.data_layout().instruction_address_space.0 as c_uint, - ) - } - } - - fn llvm_cconv(&self) -> llvm::CallConv { - match self.conv { - Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, - Conv::RustCold => llvm::ColdCallConv, - Conv::AmdGpuKernel => llvm::AmdGpuKernel, - Conv::AvrInterrupt => llvm::AvrInterrupt, - Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, - Conv::ArmAapcs => llvm::ArmAapcsCallConv, - Conv::Msp430Intr => llvm::Msp430Intr, - Conv::PtxKernel => llvm::PtxKernel, - Conv::X86Fastcall => llvm::X86FastcallCallConv, - Conv::X86Intr => llvm::X86_Intr, - Conv::X86Stdcall => llvm::X86StdcallCallConv, - Conv::X86ThisCall => llvm::X86_ThisCall, - Conv::X86VectorCall => llvm::X86_VectorCall, - Conv::X86_64SysV => llvm::X86_64_SysV, - Conv::X86_64Win64 => llvm::X86_64_Win64, - } - } - - fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { - let mut func_attrs = SmallVec::<[_; 2]>::new(); - if self.ret.layout.abi.is_uninhabited() { - func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); - } - if !self.can_unwind { - func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx)); - } - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs }); - - let mut i = 0; - let mut apply = |attrs: &ArgAttributes| { - attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn); - i += 1; - i - 1 - }; - match self.ret.mode { - PassMode::Direct(ref attrs) => { - attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); - } - PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { - assert!(!on_stack); - let i = apply(attrs); - let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); - } - PassMode::Cast(cast) => { - cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); - } - _ => {} - } - for arg in &self.args { - if arg.pad.is_some() { - apply(&ArgAttributes::new()); - } - match arg.mode { - PassMode::Ignore => {} - PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { - let i = apply(attrs); - let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); - } - PassMode::Direct(ref attrs) - | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { - apply(attrs); - } - PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { - assert!(!on_stack); - apply(attrs); - apply(extra_attrs); - } - PassMode::Pair(ref a, ref b) => { - apply(a); - apply(b); - } - PassMode::Cast(cast) => { - apply(&cast.attrs); - } - } - } - } - - fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { - let mut func_attrs = SmallVec::<[_; 2]>::new(); - if self.ret.layout.abi.is_uninhabited() { - func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx)); - } - if !self.can_unwind { - func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(bx.cx.llcx)); - } - attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &{ func_attrs }); - - let mut i = 0; - let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { - attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite); - i += 1; - i - 1 - }; - match self.ret.mode { - PassMode::Direct(ref attrs) => { - attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); - } - PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { - assert!(!on_stack); - let i = apply(bx.cx, attrs); - let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); - attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); - } - PassMode::Cast(cast) => { - cast.attrs.apply_attrs_to_callsite( - llvm::AttributePlace::ReturnValue, - &bx.cx, - callsite, - ); - } - _ => {} - } - if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { - // If the value is a boolean, the range is 0..2 and that ultimately - // become 0..0 when the type becomes i1, which would be rejected - // by the LLVM verifier. - if let Int(..) = scalar.primitive() { - if !scalar.is_bool() && !scalar.is_always_valid(bx) { - bx.range_metadata(callsite, scalar.valid_range(bx)); - } - } - } - for arg in &self.args { - if arg.pad.is_some() { - apply(bx.cx, &ArgAttributes::new()); - } - match arg.mode { - PassMode::Ignore => {} - PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { - let i = apply(bx.cx, attrs); - let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); - attributes::apply_to_callsite( - callsite, - llvm::AttributePlace::Argument(i), - &[byval], - ); - } - PassMode::Direct(ref attrs) - | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { - apply(bx.cx, attrs); - } - PassMode::Indirect { - ref attrs, - extra_attrs: Some(ref extra_attrs), - on_stack: _, - } => { - apply(bx.cx, attrs); - apply(bx.cx, extra_attrs); - } - PassMode::Pair(ref a, ref b) => { - apply(bx.cx, a); - apply(bx.cx, b); - } - PassMode::Cast(cast) => { - apply(bx.cx, &cast.attrs); - } - } - } - - let cconv = self.llvm_cconv(); - if cconv != llvm::CCallConv { - llvm::SetInstructionCallConv(callsite, cconv); - } - - if self.conv == Conv::CCmseNonSecureCall { - // This will probably get ignored on all targets but those supporting the TrustZone-M - // extension (thumbv8m targets). - let cmse_nonsecure_call = llvm::CreateAttrString(bx.cx.llcx, "cmse_nonsecure_call"); - attributes::apply_to_callsite( - callsite, - llvm::AttributePlace::Function, - &[cmse_nonsecure_call], - ); - } - - // Some intrinsics require that an elementtype attribute (with the pointee type of a - // pointer argument) is added to the callsite. - let element_type_index = unsafe { llvm::LLVMRustGetElementTypeArgIndex(callsite) }; - if element_type_index >= 0 { - let arg_ty = self.args[element_type_index as usize].layout.ty; - let pointee_ty = arg_ty.builtin_deref(true).expect("Must be pointer argument").ty; - let element_type_attr = unsafe { - llvm::LLVMRustCreateElementTypeAttr(bx.llcx, bx.layout_of(pointee_ty).llvm_type(bx)) - }; - attributes::apply_to_callsite( - callsite, - llvm::AttributePlace::Argument(element_type_index as u32), - &[element_type_attr], - ); - } - } -} - -impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { - fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value) { - fn_abi.apply_attrs_callsite(self, callsite) - } - - fn get_param(&mut self, index: usize) -> Self::Value { - llvm::get_param(self.llfn(), index as c_uint) - } -} diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs deleted file mode 100644 index 72961ae888e..00000000000 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ /dev/null @@ -1,157 +0,0 @@ -use crate::attributes; -use libc::c_uint; -use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; -use rustc_middle::bug; -use rustc_middle::ty::TyCtxt; -use rustc_session::config::{DebugInfo, OomStrategy}; -use rustc_span::symbol::sym; - -use crate::debuginfo; -use crate::llvm::{self, False, True}; -use crate::ModuleLlvm; - -pub(crate) unsafe fn codegen( - tcx: TyCtxt<'_>, - module_llvm: &mut ModuleLlvm, - module_name: &str, - kind: AllocatorKind, - has_alloc_error_handler: bool, -) { - let llcx = &*module_llvm.llcx; - let llmod = module_llvm.llmod(); - let usize = match tcx.sess.target.pointer_width { - 16 => llvm::LLVMInt16TypeInContext(llcx), - 32 => llvm::LLVMInt32TypeInContext(llcx), - 64 => llvm::LLVMInt64TypeInContext(llcx), - tws => bug!("Unsupported target word size for int: {}", tws), - }; - let i8 = llvm::LLVMInt8TypeInContext(llcx); - let i8p = llvm::LLVMPointerType(i8, 0); - let void = llvm::LLVMVoidTypeInContext(llcx); - - for method in ALLOCATOR_METHODS { - let mut args = Vec::with_capacity(method.inputs.len()); - for ty in method.inputs.iter() { - match *ty { - AllocatorTy::Layout => { - args.push(usize); // size - args.push(usize); // align - } - AllocatorTy::Ptr => args.push(i8p), - AllocatorTy::Usize => args.push(usize), - - AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), - } - } - let output = match method.output { - AllocatorTy::ResultPtr => Some(i8p), - AllocatorTy::Unit => None, - - AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { - panic!("invalid allocator output") - } - }; - let ty = llvm::LLVMFunctionType( - output.unwrap_or(void), - args.as_ptr(), - args.len() as c_uint, - False, - ); - let name = format!("__rust_{}", method.name); - let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); - - if tcx.sess.target.default_hidden_visibility { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } - if tcx.sess.must_emit_unwind_tables() { - let uwtable = attributes::uwtable_attr(llcx); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - - let callee = kind.fn_name(method.name); - let callee = - llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); - llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); - - let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); - llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); - let args = args - .iter() - .enumerate() - .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) - .collect::>(); - let ret = llvm::LLVMRustBuildCall( - llbuilder, - ty, - callee, - args.as_ptr(), - args.len() as c_uint, - None, - ); - llvm::LLVMSetTailCall(ret, True); - if output.is_some() { - llvm::LLVMBuildRet(llbuilder, ret); - } else { - llvm::LLVMBuildRetVoid(llbuilder); - } - llvm::LLVMDisposeBuilder(llbuilder); - } - - // rust alloc error handler - let args = [usize, usize]; // size, align - - let ty = llvm::LLVMFunctionType(void, args.as_ptr(), args.len() as c_uint, False); - let name = "__rust_alloc_error_handler"; - let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); - // -> ! DIFlagNoReturn - let no_return = llvm::AttributeKind::NoReturn.create_attr(llcx); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[no_return]); - - if tcx.sess.target.default_hidden_visibility { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } - if tcx.sess.must_emit_unwind_tables() { - let uwtable = attributes::uwtable_attr(llcx); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - - let kind = if has_alloc_error_handler { AllocatorKind::Global } else { AllocatorKind::Default }; - let callee = kind.fn_name(sym::oom); - let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); - // -> ! DIFlagNoReturn - attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); - llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); - - let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); - llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); - let args = args - .iter() - .enumerate() - .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) - .collect::>(); - let ret = - llvm::LLVMRustBuildCall(llbuilder, ty, callee, args.as_ptr(), args.len() as c_uint, None); - llvm::LLVMSetTailCall(ret, True); - llvm::LLVMBuildRetVoid(llbuilder); - llvm::LLVMDisposeBuilder(llbuilder); - - // __rust_alloc_error_handler_should_panic - let name = OomStrategy::SYMBOL; - let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - if tcx.sess.target.default_hidden_visibility { - llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); - } - let val = tcx.sess.opts.unstable_opts.oom.should_panic(); - let llval = llvm::LLVMConstInt(i8, val as u64, False); - llvm::LLVMSetInitializer(ll_g, llval); - - if tcx.sess.opts.debuginfo != DebugInfo::None { - let dbg_cx = debuginfo::CodegenUnitDebugContext::new(llmod); - debuginfo::metadata::build_compile_unit_di_node(tcx, module_name, &dbg_cx); - dbg_cx.finalize(tcx.sess); - } -} diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs deleted file mode 100644 index a53946995ee..00000000000 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ /dev/null @@ -1,1037 +0,0 @@ -use crate::attributes; -use crate::builder::Builder; -use crate::common::Funclet; -use crate::context::CodegenCx; -use crate::llvm; -use crate::llvm_util; -use crate::type_::Type; -use crate::type_of::LayoutLlvmExt; -use crate::value::Value; - -use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; -use rustc_codegen_ssa::mir::operand::OperandValue; -use rustc_codegen_ssa::traits::*; -use rustc_data_structures::fx::FxHashMap; -use rustc_middle::ty::layout::TyAndLayout; -use rustc_middle::{bug, span_bug, ty::Instance}; -use rustc_span::{Pos, Span}; -use rustc_target::abi::*; -use rustc_target::asm::*; - -use libc::{c_char, c_uint}; -use smallvec::SmallVec; -use tracing::debug; - -impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { - fn codegen_inline_asm( - &mut self, - template: &[InlineAsmTemplatePiece], - operands: &[InlineAsmOperandRef<'tcx, Self>], - options: InlineAsmOptions, - line_spans: &[Span], - instance: Instance<'_>, - dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, - ) { - let asm_arch = self.tcx.sess.asm_arch.unwrap(); - - // Collect the types of output operands - let mut constraints = vec![]; - let mut clobbers = vec![]; - let mut output_types = vec![]; - let mut op_idx = FxHashMap::default(); - let mut clobbered_x87 = false; - for (idx, op) in operands.iter().enumerate() { - match *op { - InlineAsmOperandRef::Out { reg, late, place } => { - let is_target_supported = |reg_class: InlineAsmRegClass| { - for &(_, feature) in reg_class.supported_types(asm_arch) { - if let Some(feature) = feature { - let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id()); - if self.tcx.sess.target_features.contains(&feature) - || codegen_fn_attrs.target_features.contains(&feature) - { - return true; - } - } else { - // Register class is unconditionally supported - return true; - } - } - false - }; - - let mut layout = None; - let ty = if let Some(ref place) = place { - layout = Some(&place.layout); - llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout) - } else if matches!( - reg.reg_class(), - InlineAsmRegClass::X86( - X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg - ) - ) { - // Special handling for x87/mmx registers: we always - // clobber the whole set if one register is marked as - // clobbered. This is due to the way LLVM handles the - // FP stack in inline assembly. - if !clobbered_x87 { - clobbered_x87 = true; - clobbers.push("~{st}".to_string()); - for i in 1..=7 { - clobbers.push(format!("~{{st({})}}", i)); - } - } - continue; - } else if !is_target_supported(reg.reg_class()) - || reg.reg_class().is_clobber_only(asm_arch) - { - // We turn discarded outputs into clobber constraints - // if the target feature needed by the register class is - // disabled. This is necessary otherwise LLVM will try - // to actually allocate a register for the dummy output. - assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_))); - clobbers.push(format!("~{}", reg_to_llvm(reg, None))); - continue; - } else { - // If the output is discarded, we don't really care what - // type is used. We're just using this to tell LLVM to - // reserve the register. - dummy_output_type(self.cx, reg.reg_class()) - }; - output_types.push(ty); - op_idx.insert(idx, constraints.len()); - let prefix = if late { "=" } else { "=&" }; - constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout))); - } - InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { - let layout = if let Some(ref out_place) = out_place { - &out_place.layout - } else { - // LLVM required tied operands to have the same type, - // so we just use the type of the input. - &in_value.layout - }; - let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout); - output_types.push(ty); - op_idx.insert(idx, constraints.len()); - let prefix = if late { "=" } else { "=&" }; - constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout)))); - } - _ => {} - } - } - - // Collect input operands - let mut inputs = vec![]; - for (idx, op) in operands.iter().enumerate() { - match *op { - InlineAsmOperandRef::In { reg, value } => { - let llval = - llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout); - inputs.push(llval); - op_idx.insert(idx, constraints.len()); - constraints.push(reg_to_llvm(reg, Some(&value.layout))); - } - InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => { - let value = llvm_fixup_input( - self, - in_value.immediate(), - reg.reg_class(), - &in_value.layout, - ); - inputs.push(value); - constraints.push(format!("{}", op_idx[&idx])); - } - InlineAsmOperandRef::SymFn { instance } => { - inputs.push(self.cx.get_fn(instance)); - op_idx.insert(idx, constraints.len()); - constraints.push("s".to_string()); - } - InlineAsmOperandRef::SymStatic { def_id } => { - inputs.push(self.cx.get_static(def_id)); - op_idx.insert(idx, constraints.len()); - constraints.push("s".to_string()); - } - _ => {} - } - } - - // Build the template string - let mut template_str = String::new(); - for piece in template { - match *piece { - InlineAsmTemplatePiece::String(ref s) => { - if s.contains('$') { - for c in s.chars() { - if c == '$' { - template_str.push_str("$$"); - } else { - template_str.push(c); - } - } - } else { - template_str.push_str(s) - } - } - InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { - match operands[operand_idx] { - InlineAsmOperandRef::In { reg, .. } - | InlineAsmOperandRef::Out { reg, .. } - | InlineAsmOperandRef::InOut { reg, .. } => { - let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier); - if let Some(modifier) = modifier { - template_str.push_str(&format!( - "${{{}:{}}}", - op_idx[&operand_idx], modifier - )); - } else { - template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx])); - } - } - InlineAsmOperandRef::Const { ref string } => { - // Const operands get injected directly into the template - template_str.push_str(string); - } - InlineAsmOperandRef::SymFn { .. } - | InlineAsmOperandRef::SymStatic { .. } => { - // Only emit the raw symbol name - template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); - } - } - } - } - } - - constraints.append(&mut clobbers); - if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { - match asm_arch { - InlineAsmArch::AArch64 | InlineAsmArch::Arm => { - constraints.push("~{cc}".to_string()); - } - InlineAsmArch::X86 | InlineAsmArch::X86_64 => { - constraints.extend_from_slice(&[ - "~{dirflag}".to_string(), - "~{fpsr}".to_string(), - "~{flags}".to_string(), - ]); - } - InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { - constraints.extend_from_slice(&[ - "~{vtype}".to_string(), - "~{vl}".to_string(), - "~{vxsat}".to_string(), - "~{vxrm}".to_string(), - ]); - } - InlineAsmArch::Avr => { - constraints.push("~{sreg}".to_string()); - } - InlineAsmArch::Nvptx64 => {} - InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} - InlineAsmArch::Hexagon => {} - InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} - InlineAsmArch::S390x => {} - InlineAsmArch::SpirV => {} - InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} - InlineAsmArch::Bpf => {} - InlineAsmArch::Msp430 => { - constraints.push("~{sr}".to_string()); - } - } - } - if !options.contains(InlineAsmOptions::NOMEM) { - // This is actually ignored by LLVM, but it's probably best to keep - // it just in case. LLVM instead uses the ReadOnly/ReadNone - // attributes on the call instruction to optimize. - constraints.push("~{memory}".to_string()); - } - let volatile = !options.contains(InlineAsmOptions::PURE); - let alignstack = !options.contains(InlineAsmOptions::NOSTACK); - let output_type = match &output_types[..] { - [] => self.type_void(), - [ty] => ty, - tys => self.type_struct(tys, false), - }; - let dialect = match asm_arch { - InlineAsmArch::X86 | InlineAsmArch::X86_64 - if !options.contains(InlineAsmOptions::ATT_SYNTAX) => - { - llvm::AsmDialect::Intel - } - _ => llvm::AsmDialect::Att, - }; - let result = inline_asm_call( - self, - &template_str, - &constraints.join(","), - &inputs, - output_type, - volatile, - alignstack, - dialect, - line_spans, - options.contains(InlineAsmOptions::MAY_UNWIND), - dest_catch_funclet, - ) - .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); - - let mut attrs = SmallVec::<[_; 2]>::new(); - if options.contains(InlineAsmOptions::PURE) { - if options.contains(InlineAsmOptions::NOMEM) { - attrs.push(llvm::AttributeKind::ReadNone.create_attr(self.cx.llcx)); - } else if options.contains(InlineAsmOptions::READONLY) { - attrs.push(llvm::AttributeKind::ReadOnly.create_attr(self.cx.llcx)); - } - attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); - } else if options.contains(InlineAsmOptions::NOMEM) { - attrs.push(llvm::AttributeKind::InaccessibleMemOnly.create_attr(self.cx.llcx)); - } else { - // LLVM doesn't have an attribute to represent ReadOnly + SideEffect - } - attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); - - // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` - if let Some((dest, _, _)) = dest_catch_funclet { - self.switch_to_block(dest); - } - - // Write results to outputs - for (idx, op) in operands.iter().enumerate() { - if let InlineAsmOperandRef::Out { reg, place: Some(place), .. } - | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op - { - let value = if output_types.len() == 1 { - result - } else { - self.extract_value(result, op_idx[&idx] as u64) - }; - let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout); - OperandValue::Immediate(value).store(self, place); - } - } - } -} - -impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { - fn codegen_global_asm( - &self, - template: &[InlineAsmTemplatePiece], - operands: &[GlobalAsmOperandRef<'tcx>], - options: InlineAsmOptions, - _line_spans: &[Span], - ) { - let asm_arch = self.tcx.sess.asm_arch.unwrap(); - - // Default to Intel syntax on x86 - let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) - && !options.contains(InlineAsmOptions::ATT_SYNTAX); - - // Build the template string - let mut template_str = String::new(); - if intel_syntax { - template_str.push_str(".intel_syntax\n"); - } - for piece in template { - match *piece { - InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s), - InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => { - match operands[operand_idx] { - GlobalAsmOperandRef::Const { ref string } => { - // Const operands get injected directly into the - // template. Note that we don't need to escape $ - // here unlike normal inline assembly. - template_str.push_str(string); - } - GlobalAsmOperandRef::SymFn { instance } => { - let llval = self.get_fn(instance); - self.add_compiler_used_global(llval); - let symbol = llvm::build_string(|s| unsafe { - llvm::LLVMRustGetMangledName(llval, s); - }) - .expect("symbol is not valid UTF-8"); - template_str.push_str(&symbol); - } - GlobalAsmOperandRef::SymStatic { def_id } => { - let llval = self - .renamed_statics - .borrow() - .get(&def_id) - .copied() - .unwrap_or_else(|| self.get_static(def_id)); - self.add_compiler_used_global(llval); - let symbol = llvm::build_string(|s| unsafe { - llvm::LLVMRustGetMangledName(llval, s); - }) - .expect("symbol is not valid UTF-8"); - template_str.push_str(&symbol); - } - } - } - } - } - if intel_syntax { - template_str.push_str("\n.att_syntax\n"); - } - - unsafe { - llvm::LLVMRustAppendModuleInlineAsm( - self.llmod, - template_str.as_ptr().cast(), - template_str.len(), - ); - } - } -} - -pub(crate) fn inline_asm_call<'ll>( - bx: &mut Builder<'_, 'll, '_>, - asm: &str, - cons: &str, - inputs: &[&'ll Value], - output: &'ll llvm::Type, - volatile: bool, - alignstack: bool, - dia: llvm::AsmDialect, - line_spans: &[Span], - unwind: bool, - dest_catch_funclet: Option<( - &'ll llvm::BasicBlock, - &'ll llvm::BasicBlock, - Option<&Funclet<'ll>>, - )>, -) -> Option<&'ll Value> { - let volatile = if volatile { llvm::True } else { llvm::False }; - let alignstack = if alignstack { llvm::True } else { llvm::False }; - let can_throw = if unwind { llvm::True } else { llvm::False }; - - let argtys = inputs - .iter() - .map(|v| { - debug!("Asm Input Type: {:?}", *v); - bx.cx.val_ty(*v) - }) - .collect::>(); - - debug!("Asm Output Type: {:?}", output); - let fty = bx.cx.type_func(&argtys, output); - unsafe { - // Ask LLVM to verify that the constraints are well-formed. - let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); - debug!("constraint verification result: {:?}", constraints_ok); - if constraints_ok { - if unwind && llvm_util::get_version() < (13, 0, 0) { - bx.cx.sess().span_fatal( - line_spans[0], - "unwinding from inline assembly is only supported on llvm >= 13.", - ); - } - - let v = llvm::LLVMRustInlineAsm( - fty, - asm.as_ptr().cast(), - asm.len(), - cons.as_ptr().cast(), - cons.len(), - volatile, - alignstack, - dia, - can_throw, - ); - - let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, v, inputs, dest, catch, funclet) - } else { - bx.call(fty, v, inputs, None) - }; - - // Store mark in a metadata node so we can map LLVM errors - // back to source locations. See #17552. - let key = "srcloc"; - let kind = llvm::LLVMGetMDKindIDInContext( - bx.llcx, - key.as_ptr() as *const c_char, - key.len() as c_uint, - ); - - // srcloc contains one integer for each line of assembly code. - // Unfortunately this isn't enough to encode a full span so instead - // we just encode the start position of each line. - // FIXME: Figure out a way to pass the entire line spans. - let mut srcloc = vec![]; - if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 { - // LLVM inserts an extra line to add the ".intel_syntax", so add - // a dummy srcloc entry for it. - // - // Don't do this if we only have 1 line span since that may be - // due to the asm template string coming from a macro. LLVM will - // default to the first srcloc for lines that don't have an - // associated srcloc. - srcloc.push(bx.const_i32(0)); - } - srcloc.extend(line_spans.iter().map(|span| bx.const_i32(span.lo().to_u32() as i32))); - let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32); - llvm::LLVMSetMetadata(call, kind, md); - - Some(call) - } else { - // LLVM has detected an issue with our constraints, bail out - None - } - } -} - -/// If the register is an xmm/ymm/zmm register then return its index. -fn xmm_reg_index(reg: InlineAsmReg) -> Option { - match reg { - InlineAsmReg::X86(reg) - if reg as u32 >= X86InlineAsmReg::xmm0 as u32 - && reg as u32 <= X86InlineAsmReg::xmm15 as u32 => - { - Some(reg as u32 - X86InlineAsmReg::xmm0 as u32) - } - InlineAsmReg::X86(reg) - if reg as u32 >= X86InlineAsmReg::ymm0 as u32 - && reg as u32 <= X86InlineAsmReg::ymm15 as u32 => - { - Some(reg as u32 - X86InlineAsmReg::ymm0 as u32) - } - InlineAsmReg::X86(reg) - if reg as u32 >= X86InlineAsmReg::zmm0 as u32 - && reg as u32 <= X86InlineAsmReg::zmm31 as u32 => - { - Some(reg as u32 - X86InlineAsmReg::zmm0 as u32) - } - _ => None, - } -} - -/// If the register is an AArch64 vector register then return its index. -fn a64_vreg_index(reg: InlineAsmReg) -> Option { - match reg { - InlineAsmReg::AArch64(reg) - if reg as u32 >= AArch64InlineAsmReg::v0 as u32 - && reg as u32 <= AArch64InlineAsmReg::v31 as u32 => - { - Some(reg as u32 - AArch64InlineAsmReg::v0 as u32) - } - _ => None, - } -} - -/// Converts a register class to an LLVM constraint code. -fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String { - match reg { - // For vector registers LLVM wants the register name to match the type size. - InlineAsmRegOrRegClass::Reg(reg) => { - if let Some(idx) = xmm_reg_index(reg) { - let class = if let Some(layout) = layout { - match layout.size.bytes() { - 64 => 'z', - 32 => 'y', - _ => 'x', - } - } else { - // We use f32 as the type for discarded outputs - 'x' - }; - format!("{{{}mm{}}}", class, idx) - } else if let Some(idx) = a64_vreg_index(reg) { - let class = if let Some(layout) = layout { - match layout.size.bytes() { - 16 => 'q', - 8 => 'd', - 4 => 's', - 2 => 'h', - 1 => 'd', // We fixup i8 to i8x8 - _ => unreachable!(), - } - } else { - // We use i64x2 as the type for discarded outputs - 'q' - }; - format!("{{{}{}}}", class, idx) - } else if reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { - // LLVM doesn't recognize x30 - "{lr}".to_string() - } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) { - // LLVM doesn't recognize r14 - "{lr}".to_string() - } else { - format!("{{{}}}", reg.name()) - } - } - InlineAsmRegOrRegClass::RegClass(reg) => match reg { - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { - unreachable!("clobber-only") - } - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r", - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t", - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x", - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", - InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", - InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", - InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", - InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", - InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r", - InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l", - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r", - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f", - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) - | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { - unreachable!("clobber-only") - } - InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r", - InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f", - InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { - unreachable!("clobber-only") - } - InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", - InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q", - InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q", - InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) - | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x", - InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", - InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", - InlineAsmRegClass::X86( - X86InlineAsmRegClass::x87_reg - | X86InlineAsmRegClass::mmx_reg - | X86InlineAsmRegClass::kreg0 - | X86InlineAsmRegClass::tmm_reg, - ) => unreachable!("clobber-only"), - InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", - InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r", - InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w", - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r", - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d", - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r", - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w", - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e", - InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", - InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", - InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", - InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { - bug!("LLVM backend does not support SPIR-V") - } - InlineAsmRegClass::Err => unreachable!(), - } - .to_string(), - } -} - -/// Converts a modifier into LLVM's equivalent modifier. -fn modifier_to_llvm( - arch: InlineAsmArch, - reg: InlineAsmRegClass, - modifier: Option, -) -> Option { - match reg { - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) - | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { - if modifier == Some('v') { None } else { modifier } - } - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { - unreachable!("clobber-only") - } - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None, - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None, - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'), - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { - if modifier.is_none() { - Some('q') - } else { - modifier - } - } - InlineAsmRegClass::Hexagon(_) => None, - InlineAsmRegClass::Mips(_) => None, - InlineAsmRegClass::Nvptx(_) => None, - InlineAsmRegClass::PowerPC(_) => None, - InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) - | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None, - InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { - unreachable!("clobber-only") - } - InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) - | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier { - None if arch == InlineAsmArch::X86_64 => Some('q'), - None => Some('k'), - Some('l') => Some('b'), - Some('h') => Some('h'), - Some('x') => Some('w'), - Some('e') => Some('k'), - Some('r') => Some('q'), - _ => unreachable!(), - }, - InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None, - InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg) - | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg) - | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) { - (X86InlineAsmRegClass::xmm_reg, None) => Some('x'), - (X86InlineAsmRegClass::ymm_reg, None) => Some('t'), - (X86InlineAsmRegClass::zmm_reg, None) => Some('g'), - (_, Some('x')) => Some('x'), - (_, Some('y')) => Some('t'), - (_, Some('z')) => Some('g'), - _ => unreachable!(), - }, - InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, - InlineAsmRegClass::X86( - X86InlineAsmRegClass::x87_reg - | X86InlineAsmRegClass::mmx_reg - | X86InlineAsmRegClass::kreg0 - | X86InlineAsmRegClass::tmm_reg, - ) => { - unreachable!("clobber-only") - } - InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None, - InlineAsmRegClass::Bpf(_) => None, - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) - | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) - | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier { - Some('h') => Some('B'), - Some('l') => Some('A'), - _ => None, - }, - InlineAsmRegClass::Avr(_) => None, - InlineAsmRegClass::S390x(_) => None, - InlineAsmRegClass::Msp430(_) => None, - InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { - bug!("LLVM backend does not support SPIR-V") - } - InlineAsmRegClass::Err => unreachable!(), - } -} - -/// Type to use for outputs that are discarded. It doesn't really matter what -/// the type is, as long as it is valid for the constraint code. -fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type { - match reg { - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) - | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { - cx.type_vector(cx.type_i64(), 2) - } - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { - unreachable!("clobber-only") - } - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(), - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(), - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) - | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { - cx.type_vector(cx.type_i64(), 2) - } - InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), - InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), - InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), - InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), - InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), - InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) - | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { - unreachable!("clobber-only") - } - InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), - InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), - InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { - unreachable!("clobber-only") - } - InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) - | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(), - InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(), - InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) - | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) - | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), - InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), - InlineAsmRegClass::X86( - X86InlineAsmRegClass::x87_reg - | X86InlineAsmRegClass::mmx_reg - | X86InlineAsmRegClass::kreg0 - | X86InlineAsmRegClass::tmm_reg, - ) => { - unreachable!("clobber-only") - } - InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), - InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(), - InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(), - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(), - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(), - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(), - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(), - InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), - InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), - InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), - InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), - InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { - bug!("LLVM backend does not support SPIR-V") - } - InlineAsmRegClass::Err => unreachable!(), - } -} - -/// Helper function to get the LLVM type for a Scalar. Pointers are returned as -/// the equivalent integer type. -fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type { - match scalar.primitive() { - Primitive::Int(Integer::I8, _) => cx.type_i8(), - Primitive::Int(Integer::I16, _) => cx.type_i16(), - Primitive::Int(Integer::I32, _) => cx.type_i32(), - Primitive::Int(Integer::I64, _) => cx.type_i64(), - Primitive::F32 => cx.type_f32(), - Primitive::F64 => cx.type_f64(), - Primitive::Pointer => cx.type_isize(), - _ => unreachable!(), - } -} - -/// Fix up an input value to work around LLVM bugs. -fn llvm_fixup_input<'ll, 'tcx>( - bx: &mut Builder<'_, 'll, 'tcx>, - mut value: &'ll Value, - reg: InlineAsmRegClass, - layout: &TyAndLayout<'tcx>, -) -> &'ll Value { - match (reg, layout.abi) { - (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { - if let Primitive::Int(Integer::I8, _) = s.primitive() { - let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); - bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) - } else { - value - } - } - (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { - let elem_ty = llvm_asm_scalar_type(bx.cx, s); - let count = 16 / layout.size.bytes(); - let vec_ty = bx.cx.type_vector(elem_ty, count); - if let Primitive::Pointer = s.primitive() { - value = bx.ptrtoint(value, bx.cx.type_isize()); - } - bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) - } - ( - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), - Abi::Vector { element, count }, - ) if layout.size.bytes() == 8 => { - let elem_ty = llvm_asm_scalar_type(bx.cx, element); - let vec_ty = bx.cx.type_vector(elem_ty, count); - let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); - bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) - } - (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) - if s.primitive() == Primitive::F64 => - { - bx.bitcast(value, bx.cx.type_i64()) - } - ( - InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - Abi::Vector { .. }, - ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)), - ( - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), - Abi::Scalar(s), - ) => { - if let Primitive::Int(Integer::I32, _) = s.primitive() { - bx.bitcast(value, bx.cx.type_f32()) - } else { - value - } - } - ( - InlineAsmRegClass::Arm( - ArmInlineAsmRegClass::dreg - | ArmInlineAsmRegClass::dreg_low8 - | ArmInlineAsmRegClass::dreg_low16, - ), - Abi::Scalar(s), - ) => { - if let Primitive::Int(Integer::I64, _) = s.primitive() { - bx.bitcast(value, bx.cx.type_f64()) - } else { - value - } - } - (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { - match s.primitive() { - // MIPS only supports register-length arithmetics. - Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), - Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()), - Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()), - _ => value, - } - } - _ => value, - } -} - -/// Fix up an output value to work around LLVM bugs. -fn llvm_fixup_output<'ll, 'tcx>( - bx: &mut Builder<'_, 'll, 'tcx>, - mut value: &'ll Value, - reg: InlineAsmRegClass, - layout: &TyAndLayout<'tcx>, -) -> &'ll Value { - match (reg, layout.abi) { - (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { - if let Primitive::Int(Integer::I8, _) = s.primitive() { - bx.extract_element(value, bx.const_i32(0)) - } else { - value - } - } - (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { - value = bx.extract_element(value, bx.const_i32(0)); - if let Primitive::Pointer = s.primitive() { - value = bx.inttoptr(value, layout.llvm_type(bx.cx)); - } - value - } - ( - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), - Abi::Vector { element, count }, - ) if layout.size.bytes() == 8 => { - let elem_ty = llvm_asm_scalar_type(bx.cx, element); - let vec_ty = bx.cx.type_vector(elem_ty, count * 2); - let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); - bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) - } - (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) - if s.primitive() == Primitive::F64 => - { - bx.bitcast(value, bx.cx.type_f64()) - } - ( - InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - Abi::Vector { .. }, - ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)), - ( - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), - Abi::Scalar(s), - ) => { - if let Primitive::Int(Integer::I32, _) = s.primitive() { - bx.bitcast(value, bx.cx.type_i32()) - } else { - value - } - } - ( - InlineAsmRegClass::Arm( - ArmInlineAsmRegClass::dreg - | ArmInlineAsmRegClass::dreg_low8 - | ArmInlineAsmRegClass::dreg_low16, - ), - Abi::Scalar(s), - ) => { - if let Primitive::Int(Integer::I64, _) = s.primitive() { - bx.bitcast(value, bx.cx.type_i64()) - } else { - value - } - } - (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { - match s.primitive() { - // MIPS only supports register-length arithmetics. - Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), - Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()), - Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()), - Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()), - _ => value, - } - } - _ => value, - } -} - -/// Output type to use for llvm_fixup_output. -fn llvm_fixup_output_type<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - reg: InlineAsmRegClass, - layout: &TyAndLayout<'tcx>, -) -> &'ll Type { - match (reg, layout.abi) { - (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { - if let Primitive::Int(Integer::I8, _) = s.primitive() { - cx.type_vector(cx.type_i8(), 8) - } else { - layout.llvm_type(cx) - } - } - (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { - let elem_ty = llvm_asm_scalar_type(cx, s); - let count = 16 / layout.size.bytes(); - cx.type_vector(elem_ty, count) - } - ( - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), - Abi::Vector { element, count }, - ) if layout.size.bytes() == 8 => { - let elem_ty = llvm_asm_scalar_type(cx, element); - cx.type_vector(elem_ty, count * 2) - } - (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) - if s.primitive() == Primitive::F64 => - { - cx.type_i64() - } - ( - InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - Abi::Vector { .. }, - ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8), - ( - InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), - Abi::Scalar(s), - ) => { - if let Primitive::Int(Integer::I32, _) = s.primitive() { - cx.type_f32() - } else { - layout.llvm_type(cx) - } - } - ( - InlineAsmRegClass::Arm( - ArmInlineAsmRegClass::dreg - | ArmInlineAsmRegClass::dreg_low8 - | ArmInlineAsmRegClass::dreg_low16, - ), - Abi::Scalar(s), - ) => { - if let Primitive::Int(Integer::I64, _) = s.primitive() { - cx.type_f64() - } else { - layout.llvm_type(cx) - } - } - (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { - match s.primitive() { - // MIPS only supports register-length arithmetics. - Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), - Primitive::F32 => cx.type_i32(), - Primitive::F64 => cx.type_i64(), - _ => layout.llvm_type(cx), - } - } - _ => layout.llvm_type(cx), - } -} diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs deleted file mode 100644 index aabbe8ac276..00000000000 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ /dev/null @@ -1,449 +0,0 @@ -//! Set and unset common attributes on LLVM values. - -use rustc_codegen_ssa::traits::*; -use rustc_data_structures::small_str::SmallStr; -use rustc_hir::def_id::DefId; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::ty::{self, TyCtxt}; -use rustc_session::config::OptLevel; -use rustc_span::symbol::sym; -use rustc_target::spec::abi::Abi; -use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; -use smallvec::SmallVec; - -use crate::attributes; -use crate::llvm::AttributePlace::Function; -use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace}; -use crate::llvm_util; -pub use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr}; - -use crate::context::CodegenCx; -use crate::value::Value; - -pub fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { - if !attrs.is_empty() { - llvm::AddFunctionAttributes(llfn, idx, attrs); - } -} - -pub fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attribute]) { - if !attrs.is_empty() { - llvm::AddCallSiteAttributes(callsite, idx, attrs); - } -} - -/// Get LLVM attribute for the provided inline heuristic. -#[inline] -fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { - match inline { - InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), - InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), - InlineAttr::Never => { - if cx.sess().target.arch != "amdgpu" { - Some(AttributeKind::NoInline.create_attr(cx.llcx)) - } else { - None - } - } - InlineAttr::None => None, - } -} - -/// Get LLVM sanitize attributes. -#[inline] -pub fn sanitize_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, - no_sanitize: SanitizerSet, -) -> SmallVec<[&'ll Attribute; 4]> { - let mut attrs = SmallVec::new(); - let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; - if enabled.contains(SanitizerSet::ADDRESS) { - attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); - } - if enabled.contains(SanitizerSet::MEMORY) { - attrs.push(llvm::AttributeKind::SanitizeMemory.create_attr(cx.llcx)); - } - if enabled.contains(SanitizerSet::THREAD) { - attrs.push(llvm::AttributeKind::SanitizeThread.create_attr(cx.llcx)); - } - if enabled.contains(SanitizerSet::HWADDRESS) { - attrs.push(llvm::AttributeKind::SanitizeHWAddress.create_attr(cx.llcx)); - } - if enabled.contains(SanitizerSet::SHADOWCALLSTACK) { - attrs.push(llvm::AttributeKind::ShadowCallStack.create_attr(cx.llcx)); - } - if enabled.contains(SanitizerSet::MEMTAG) { - // Check to make sure the mte target feature is actually enabled. - let features = cx.tcx.global_backend_features(()); - let mte_feature = - features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); - if let None | Some("-mte") = mte_feature { - cx.tcx.sess.err("`-Zsanitizer=memtag` requires `-Ctarget-feature=+mte`"); - } - - attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); - } - attrs -} - -/// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. -#[inline] -pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { - // NOTE: We should determine if we even need async unwind tables, as they - // take have more overhead and if we can use sync unwind tables we - // probably should. - llvm::CreateUWTableAttr(llcx, true) -} - -pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let mut fp = cx.sess().target.frame_pointer; - // "mcount" function relies on stack pointer. - // See . - if cx.sess().instrument_mcount() || matches!(cx.sess().opts.cg.force_frame_pointers, Some(true)) - { - fp = FramePointer::Always; - } - let attr_value = match fp { - FramePointer::Always => "all", - FramePointer::NonLeaf => "non-leaf", - FramePointer::MayOmit => return None, - }; - Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value)) -} - -/// Tell LLVM what instrument function to insert. -#[inline] -fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if cx.sess().instrument_mcount() { - // Similar to `clang -pg` behavior. Handled by the - // `post-inline-ee-instrument` LLVM pass. - - // The function name varies on platforms. - // See test/CodeGen/mcount.c in clang. - let mcount_name = cx.sess().target.mcount.as_ref(); - - Some(llvm::CreateAttrStringValue( - cx.llcx, - "instrument-function-entry-inlined", - &mcount_name, - )) - } else { - None - } -} - -fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - // Currently stack probes seem somewhat incompatible with the address - // sanitizer and thread sanitizer. With asan we're already protected from - // stack overflow anyway so we don't really need stack probes regardless. - if cx - .sess() - .opts - .unstable_opts - .sanitizer - .intersects(SanitizerSet::ADDRESS | SanitizerSet::THREAD) - { - return None; - } - - // probestack doesn't play nice either with `-C profile-generate`. - if cx.sess().opts.cg.profile_generate.enabled() { - return None; - } - - // probestack doesn't play nice either with gcov profiling. - if cx.sess().opts.unstable_opts.profile { - return None; - } - - let attr_value = match cx.sess().target.stack_probes { - StackProbeType::None => return None, - // Request LLVM to generate the probes inline. If the given LLVM version does not support - // this, no probe is generated at all (even if the attribute is specified). - StackProbeType::Inline => "inline-asm", - // Flag our internal `__rust_probestack` function as the stack probe symbol. - // This is defined in the `compiler-builtins` crate for each architecture. - StackProbeType::Call => "__rust_probestack", - // Pick from the two above based on the LLVM version. - StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { - if llvm_util::get_version() < min_llvm_version_for_inline { - "__rust_probestack" - } else { - "inline-asm" - } - } - }; - Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value)) -} - -fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let sspattr = match cx.sess().stack_protector() { - StackProtector::None => return None, - StackProtector::All => AttributeKind::StackProtectReq, - StackProtector::Strong => AttributeKind::StackProtectStrong, - StackProtector::Basic => AttributeKind::StackProtect, - }; - - Some(sspattr.create_attr(cx.llcx)) -} - -pub fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { - let target_cpu = llvm_util::target_cpu(cx.tcx.sess); - llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) -} - -pub fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - llvm_util::tune_cpu(cx.tcx.sess) - .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) -} - -/// Get the `NonLazyBind` LLVM attribute, -/// if the codegen options allow skipping the PLT. -pub fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - // Don't generate calls through PLT if it's not necessary - if !cx.sess().needs_plt() { - Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) - } else { - None - } -} - -/// Get the default optimizations attrs for a function. -#[inline] -pub(crate) fn default_optimisation_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, -) -> SmallVec<[&'ll Attribute; 2]> { - let mut attrs = SmallVec::new(); - match cx.sess().opts.optimize { - OptLevel::Size => { - attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); - } - OptLevel::SizeMin => { - attrs.push(llvm::AttributeKind::MinSize.create_attr(cx.llcx)); - attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); - } - _ => {} - } - attrs -} - -fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { - llvm::CreateAttrStringValue(llcx, "alloc-family", "__rust_alloc") -} - -/// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) -/// attributes. -pub fn from_fn_attrs<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - llfn: &'ll Value, - instance: ty::Instance<'tcx>, -) { - let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); - - let mut to_add = SmallVec::<[_; 16]>::new(); - - match codegen_fn_attrs.optimize { - OptimizeAttr::None => { - to_add.extend(default_optimisation_attrs(cx)); - } - OptimizeAttr::Size => { - to_add.push(llvm::AttributeKind::MinSize.create_attr(cx.llcx)); - to_add.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); - } - OptimizeAttr::Speed => {} - } - - let inline = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { - InlineAttr::Never - } else if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { - InlineAttr::Hint - } else { - codegen_fn_attrs.inline - }; - to_add.extend(inline_attr(cx, inline)); - - // The `uwtable` attribute according to LLVM is: - // - // This attribute indicates that the ABI being targeted requires that an - // unwind table entry be produced for this function even if we can show - // that no exceptions passes by it. This is normally the case for the - // ELF x86-64 abi, but it can be disabled for some compilation units. - // - // Typically when we're compiling with `-C panic=abort` (which implies this - // `no_landing_pads` check) we don't need `uwtable` because we can't - // generate any exceptions! On Windows, however, exceptions include other - // events such as illegal instructions, segfaults, etc. This means that on - // Windows we end up still needing the `uwtable` attribute even if the `-C - // panic=abort` flag is passed. - // - // You can also find more info on why Windows always requires uwtables here: - // https://bugzilla.mozilla.org/show_bug.cgi?id=1302078 - if cx.sess().must_emit_unwind_tables() { - to_add.push(uwtable_attr(cx.llcx)); - } - - if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { - to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); - } - - // FIXME: none of these three functions interact with source level attributes. - to_add.extend(frame_pointer_type_attr(cx)); - to_add.extend(instrument_function_attr(cx)); - to_add.extend(probestack_attr(cx)); - to_add.extend(stackprotector_attr(cx)); - - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { - to_add.push(AttributeKind::Cold.create_attr(cx.llcx)); - } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_RETURNS_TWICE) { - to_add.push(AttributeKind::ReturnsTwice.create_attr(cx.llcx)); - } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_PURE) { - to_add.push(AttributeKind::ReadOnly.create_attr(cx.llcx)); - } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_CONST) { - to_add.push(AttributeKind::ReadNone.create_attr(cx.llcx)); - } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { - to_add.push(AttributeKind::Naked.create_attr(cx.llcx)); - // HACK(jubilee): "indirect branch tracking" works by attaching prologues to functions. - // And it is a module-level attribute, so the alternative is pulling naked functions into new LLVM modules. - // Otherwise LLVM's "naked" functions come with endbr prefixes per https://github.com/rust-lang/rust/issues/98768 - to_add.push(AttributeKind::NoCfCheck.create_attr(cx.llcx)); - // Need this for AArch64. - to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false")); - } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) - || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) - { - if llvm_util::get_version() >= (15, 0, 0) { - to_add.push(create_alloc_family_attr(cx.llcx)); - // apply to argument place instead of function - let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); - to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); - let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { - flags |= AllocKindFlags::Uninitialized; - } else { - flags |= AllocKindFlags::Zeroed; - } - to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); - } - // apply to return place instead of function (unlike all other attributes applied in this function) - let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); - } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::REALLOCATOR) { - if llvm_util::get_version() >= (15, 0, 0) { - to_add.push(create_alloc_family_attr(cx.llcx)); - to_add.push(llvm::CreateAllocKindAttr( - cx.llcx, - AllocKindFlags::Realloc | AllocKindFlags::Aligned, - )); - // applies to argument place instead of function place - let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); - // apply to argument place instead of function - let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); - to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); - } - let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); - } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::DEALLOCATOR) { - if llvm_util::get_version() >= (15, 0, 0) { - to_add.push(create_alloc_family_attr(cx.llcx)); - to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); - // applies to argument place instead of function place - let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); - } - } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { - to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry")); - } - if let Some(align) = codegen_fn_attrs.alignment { - llvm::set_alignment(llfn, align as usize); - } - to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); - - // Always annotate functions with the target-cpu they are compiled for. - // Without this, ThinLTO won't inline Rust functions into Clang generated - // functions (because Clang annotates functions this way too). - to_add.push(target_cpu_attr(cx)); - // tune-cpu is only conveyed through the attribute for our purpose. - // The target doesn't care; the subtarget reads our attribute. - to_add.extend(tune_cpu_attr(cx)); - - let function_features = - codegen_fn_attrs.target_features.iter().map(|f| f.as_str()).collect::>(); - - if let Some(f) = llvm_util::check_tied_features( - cx.tcx.sess, - &function_features.iter().map(|f| (*f, true)).collect(), - ) { - let span = cx - .tcx - .get_attr(instance.def_id(), sym::target_feature) - .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); - let msg = format!( - "the target features {} must all be either enabled or disabled together", - f.join(", ") - ); - let mut err = cx.tcx.sess.struct_span_err(span, &msg); - err.help("add the missing features in a `target_feature` attribute"); - err.emit(); - return; - } - - let mut function_features = function_features - .iter() - .flat_map(|feat| { - llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{}", f)) - }) - .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { - InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(), - InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(), - })) - .collect::>(); - - if cx.tcx.sess.target.is_like_wasm { - // If this function is an import from the environment but the wasm - // import has a specific module/name, apply them here. - if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { - to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", &module)); - - let name = - codegen_fn_attrs.link_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); - let name = name.as_str(); - to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); - } - - // The `"wasm"` abi on wasm targets automatically enables the - // `+multivalue` feature because the purpose of the wasm abi is to match - // the WebAssembly specification, which has this feature. This won't be - // needed when LLVM enables this `multivalue` feature by default. - if !cx.tcx.is_closure(instance.def_id()) { - let abi = cx.tcx.fn_sig(instance.def_id()).abi(); - if abi == Abi::Wasm { - function_features.push("+multivalue".to_string()); - } - } - } - - let global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str()); - let function_features = function_features.iter().map(|s| s.as_str()); - let target_features = - global_features.chain(function_features).intersperse(",").collect::>(); - if !target_features.is_empty() { - to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)); - } - - attributes::apply_to_llfn(llfn, Function, &to_add); -} - -fn wasm_import_module(tcx: TyCtxt<'_>, id: DefId) -> Option<&String> { - tcx.wasm_import_module_map(id.krate).get(&id) -} diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs deleted file mode 100644 index 27039cda253..00000000000 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ /dev/null @@ -1,383 +0,0 @@ -//! A helper class for dealing with static archives - -use std::env; -use std::ffi::{CStr, CString, OsString}; -use std::io; -use std::mem; -use std::path::{Path, PathBuf}; -use std::ptr; -use std::str; - -use crate::llvm::archive_ro::{ArchiveRO, Child}; -use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; -use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder}; -use rustc_session::cstore::{DllCallingConvention, DllImport}; -use rustc_session::Session; - -/// Helper for adding many files to an archive. -#[must_use = "must call build() to finish building the archive"] -pub struct LlvmArchiveBuilder<'a> { - sess: &'a Session, - additions: Vec, -} - -enum Addition { - File { path: PathBuf, name_in_archive: String }, - Archive { path: PathBuf, archive: ArchiveRO, skip: Box bool> }, -} - -impl Addition { - fn path(&self) -> &Path { - match self { - Addition::File { path, .. } | Addition::Archive { path, .. } => path, - } - } -} - -fn is_relevant_child(c: &Child<'_>) -> bool { - match c.name() { - Some(name) => !name.contains("SYMDEF"), - None => false, - } -} - -/// Map machine type strings to values of LLVM's MachineTypes enum. -fn llvm_machine_type(cpu: &str) -> LLVMMachineType { - match cpu { - "x86_64" => LLVMMachineType::AMD64, - "x86" => LLVMMachineType::I386, - "aarch64" => LLVMMachineType::ARM64, - "arm" => LLVMMachineType::ARM, - _ => panic!("unsupported cpu type {}", cpu), - } -} - -impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { - fn add_archive( - &mut self, - archive: &Path, - skip: Box bool + 'static>, - ) -> io::Result<()> { - let archive_ro = match ArchiveRO::open(archive) { - Ok(ar) => ar, - Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), - }; - if self.additions.iter().any(|ar| ar.path() == archive) { - return Ok(()); - } - self.additions.push(Addition::Archive { - path: archive.to_path_buf(), - archive: archive_ro, - skip: Box::new(skip), - }); - Ok(()) - } - - /// Adds an arbitrary file to this archive - fn add_file(&mut self, file: &Path) { - let name = file.file_name().unwrap().to_str().unwrap(); - self.additions - .push(Addition::File { path: file.to_path_buf(), name_in_archive: name.to_owned() }); - } - - /// Combine the provided files, rlibs, and native libraries into a single - /// `Archive`. - fn build(mut self: Box, output: &Path) -> bool { - match self.build_with_llvm(output) { - Ok(any_members) => any_members, - Err(e) => self.sess.fatal(&format!("failed to build archive: {}", e)), - } - } -} - -pub struct LlvmArchiveBuilderBuilder; - -impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { - fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box + 'a> { - Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) - } - - fn create_dll_import_lib( - &self, - sess: &Session, - lib_name: &str, - dll_imports: &[DllImport], - tmpdir: &Path, - ) -> PathBuf { - let output_path = { - let mut output_path: PathBuf = tmpdir.to_path_buf(); - output_path.push(format!("{}_imports", lib_name)); - output_path.with_extension("lib") - }; - - let target = &sess.target; - let mingw_gnu_toolchain = target.vendor == "pc" - && target.os == "windows" - && target.env == "gnu" - && target.abi.is_empty(); - - let import_name_and_ordinal_vector: Vec<(String, Option)> = dll_imports - .iter() - .map(|import: &DllImport| { - if sess.target.arch == "x86" { - ( - LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), - import.ordinal, - ) - } else { - (import.name.to_string(), import.ordinal) - } - }) - .collect(); - - if mingw_gnu_toolchain { - // The binutils linker used on -windows-gnu targets cannot read the import - // libraries generated by LLVM: in our attempts, the linker produced an .EXE - // that loaded but crashed with an AV upon calling one of the imported - // functions. Therefore, use binutils to create the import library instead, - // by writing a .DEF file to the temp dir and calling binutils's dlltool. - let def_file_path = tmpdir.join(format!("{}_imports", lib_name)).with_extension("def"); - - let def_file_content = format!( - "EXPORTS\n{}", - import_name_and_ordinal_vector - .into_iter() - .map(|(name, ordinal)| { - match ordinal { - Some(n) => format!("{} @{} NONAME", name, n), - None => name, - } - }) - .collect::>() - .join("\n") - ); - - match std::fs::write(&def_file_path, def_file_content) { - Ok(_) => {} - Err(e) => { - sess.fatal(&format!("Error writing .DEF file: {}", e)); - } - }; - - let dlltool = find_binutils_dlltool(sess); - let result = std::process::Command::new(dlltool) - .args([ - "-d", - def_file_path.to_str().unwrap(), - "-D", - lib_name, - "-l", - output_path.to_str().unwrap(), - ]) - .output(); - - match result { - Err(e) => { - sess.fatal(&format!("Error calling dlltool: {}", e)); - } - Ok(output) if !output.status.success() => sess.fatal(&format!( - "Dlltool could not create import library: {}\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )), - _ => {} - } - } else { - // we've checked for \0 characters in the library name already - let dll_name_z = CString::new(lib_name).unwrap(); - - let output_path_z = rustc_fs_util::path_to_c_string(&output_path); - - tracing::trace!("invoking LLVMRustWriteImportLibrary"); - tracing::trace!(" dll_name {:#?}", dll_name_z); - tracing::trace!(" output_path {}", output_path.display()); - tracing::trace!( - " import names: {}", - dll_imports - .iter() - .map(|import| import.name.to_string()) - .collect::>() - .join(", "), - ); - - // All import names are Rust identifiers and therefore cannot contain \0 characters. - // FIXME: when support for #[link_name] is implemented, ensure that the import names - // still don't contain any \0 characters. Also need to check that the names don't - // contain substrings like " @" or "NONAME" that are keywords or otherwise reserved - // in definition files. - let cstring_import_name_and_ordinal_vector: Vec<(CString, Option)> = - import_name_and_ordinal_vector - .into_iter() - .map(|(name, ordinal)| (CString::new(name).unwrap(), ordinal)) - .collect(); - - let ffi_exports: Vec = cstring_import_name_and_ordinal_vector - .iter() - .map(|(name_z, ordinal)| LLVMRustCOFFShortExport::new(name_z.as_ptr(), *ordinal)) - .collect(); - let result = unsafe { - crate::llvm::LLVMRustWriteImportLibrary( - dll_name_z.as_ptr(), - output_path_z.as_ptr(), - ffi_exports.as_ptr(), - ffi_exports.len(), - llvm_machine_type(&sess.target.arch) as u16, - !sess.target.is_like_msvc, - ) - }; - - if result == crate::llvm::LLVMRustResult::Failure { - sess.fatal(&format!( - "Error creating import library for {}: {}", - lib_name, - llvm::last_error().unwrap_or("unknown LLVM error".to_string()) - )); - } - }; - - output_path - } -} - -impl<'a> LlvmArchiveBuilder<'a> { - fn build_with_llvm(&mut self, output: &Path) -> io::Result { - let kind = &*self.sess.target.archive_format; - let kind = kind.parse::().map_err(|_| kind).unwrap_or_else(|kind| { - self.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) - }); - - let mut additions = mem::take(&mut self.additions); - let mut strings = Vec::new(); - let mut members = Vec::new(); - - let dst = CString::new(output.to_str().unwrap())?; - - unsafe { - for addition in &mut additions { - match addition { - Addition::File { path, name_in_archive } => { - let path = CString::new(path.to_str().unwrap())?; - let name = CString::new(name_in_archive.clone())?; - members.push(llvm::LLVMRustArchiveMemberNew( - path.as_ptr(), - name.as_ptr(), - None, - )); - strings.push(path); - strings.push(name); - } - Addition::Archive { archive, skip, .. } => { - for child in archive.iter() { - let child = child.map_err(string_to_io_error)?; - if !is_relevant_child(&child) { - continue; - } - let child_name = child.name().unwrap(); - if skip(child_name) { - continue; - } - - // It appears that LLVM's archive writer is a little - // buggy if the name we pass down isn't just the - // filename component, so chop that off here and - // pass it in. - // - // See LLVM bug 25877 for more info. - let child_name = - Path::new(child_name).file_name().unwrap().to_str().unwrap(); - let name = CString::new(child_name)?; - let m = llvm::LLVMRustArchiveMemberNew( - ptr::null(), - name.as_ptr(), - Some(child.raw), - ); - members.push(m); - strings.push(name); - } - } - } - } - - let r = llvm::LLVMRustWriteArchive( - dst.as_ptr(), - members.len() as libc::size_t, - members.as_ptr() as *const &_, - true, - kind, - ); - let ret = if r.into_result().is_err() { - let err = llvm::LLVMRustGetLastError(); - let msg = if err.is_null() { - "failed to write archive".into() - } else { - String::from_utf8_lossy(CStr::from_ptr(err).to_bytes()) - }; - Err(io::Error::new(io::ErrorKind::Other, msg)) - } else { - Ok(!members.is_empty()) - }; - for member in members { - llvm::LLVMRustArchiveMemberFree(member); - } - ret - } - } - - fn i686_decorated_name(import: &DllImport, mingw: bool) -> String { - let name = import.name; - let prefix = if mingw { "" } else { "_" }; - - match import.calling_convention { - DllCallingConvention::C => format!("{}{}", prefix, name), - DllCallingConvention::Stdcall(arg_list_size) => { - format!("{}{}@{}", prefix, name, arg_list_size) - } - DllCallingConvention::Fastcall(arg_list_size) => format!("@{}@{}", name, arg_list_size), - DllCallingConvention::Vectorcall(arg_list_size) => { - format!("{}@@{}", name, arg_list_size) - } - } - } -} - -fn string_to_io_error(s: String) -> io::Error { - io::Error::new(io::ErrorKind::Other, format!("bad archive: {}", s)) -} - -fn find_binutils_dlltool(sess: &Session) -> OsString { - assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); - if let Some(dlltool_path) = &sess.opts.unstable_opts.dlltool { - return dlltool_path.clone().into_os_string(); - } - - let mut tool_name: OsString = if sess.host.arch != sess.target.arch { - // We are cross-compiling, so we need the tool with the prefix matching our target - if sess.target.arch == "x86" { - "i686-w64-mingw32-dlltool" - } else { - "x86_64-w64-mingw32-dlltool" - } - } else { - // We are not cross-compiling, so we just want `dlltool` - "dlltool" - } - .into(); - - if sess.host.options.is_like_windows { - // If we're compiling on Windows, add the .exe suffix - tool_name.push(".exe"); - } - - // NOTE: it's not clear how useful it is to explicitly search PATH. - for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { - let full_path = dir.join(&tool_name); - if full_path.is_file() { - return full_path.into_os_string(); - } - } - - // The user didn't specify the location of the dlltool binary, and we weren't able - // to find the appropriate one on the PATH. Just return the name of the tool - // and let the invocation fail with a hopefully useful error message. - tool_name -} diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs deleted file mode 100644 index 3731c6bcfe7..00000000000 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ /dev/null @@ -1,936 +0,0 @@ -use crate::back::write::{ - self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers, -}; -use crate::llvm::archive_ro::ArchiveRO; -use crate::llvm::{self, build_string, False, True}; -use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; -use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; -use rustc_codegen_ssa::back::symbol_export; -use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig}; -use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; -use rustc_data_structures::fx::FxHashMap; -use rustc_errors::{FatalError, Handler}; -use rustc_hir::def_id::LOCAL_CRATE; -use rustc_middle::bug; -use rustc_middle::dep_graph::WorkProduct; -use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; -use rustc_session::cgu_reuse_tracker::CguReuse; -use rustc_session::config::{self, CrateType, Lto}; -use tracing::{debug, info}; - -use std::ffi::{CStr, CString}; -use std::fs::File; -use std::io; -use std::iter; -use std::path::Path; -use std::ptr; -use std::slice; -use std::sync::Arc; - -/// We keep track of the computed LTO cache keys from the previous -/// session to determine which CGUs we can reuse. -pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; - -pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { - match crate_type { - CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => true, - CrateType::Dylib | CrateType::Rlib | CrateType::ProcMacro => false, - } -} - -fn prepare_lto( - cgcx: &CodegenContext, - diag_handler: &Handler, -) -> Result<(Vec, Vec<(SerializedModule, CString)>), FatalError> { - let export_threshold = match cgcx.lto { - // We're just doing LTO for our one crate - Lto::ThinLocal => SymbolExportLevel::Rust, - - // We're doing LTO for the entire crate graph - Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types), - - Lto::No => panic!("didn't request LTO but we're doing LTO"), - }; - - let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| { - if info.level.is_below_threshold(export_threshold) || info.used { - Some(CString::new(name.as_str()).unwrap()) - } else { - None - } - }; - let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); - let mut symbols_below_threshold = { - let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); - exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::>() - }; - info!("{} symbols to preserve in this crate", symbols_below_threshold.len()); - - // If we're performing LTO for the entire crate graph, then for each of our - // upstream dependencies, find the corresponding rlib and load the bitcode - // from the archive. - // - // We save off all the bytecode and LLVM module ids for later processing - // with either fat or thin LTO - let mut upstream_modules = Vec::new(); - if cgcx.lto != Lto::ThinLocal { - if cgcx.opts.cg.prefer_dynamic { - diag_handler - .struct_err("cannot prefer dynamic linking when performing LTO") - .note( - "only 'staticlib', 'bin', and 'cdylib' outputs are \ - supported with LTO", - ) - .emit(); - return Err(FatalError); - } - - // Make sure we actually can run LTO - for crate_type in cgcx.crate_types.iter() { - if !crate_type_allows_lto(*crate_type) { - let e = diag_handler.fatal( - "lto can only be run for executables, cdylibs and \ - static library outputs", - ); - return Err(e); - } - } - - for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() { - let exported_symbols = - cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); - { - let _timer = - cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); - symbols_below_threshold - .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); - } - - let archive = ArchiveRO::open(path).expect("wanted an rlib"); - let obj_files = archive - .iter() - .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c)))) - .filter(|&(name, _)| looks_like_rust_object_file(name)); - for (name, child) in obj_files { - info!("adding bitcode from {}", name); - match get_bitcode_slice_from_object_data(child.data()) { - Ok(data) => { - let module = SerializedModule::FromRlib(data.to_vec()); - upstream_modules.push((module, CString::new(name).unwrap())); - } - Err(msg) => return Err(diag_handler.fatal(&msg)), - } - } - } - } - - Ok((symbols_below_threshold, upstream_modules)) -} - -fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> { - let mut len = 0; - let data = - unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) }; - if !data.is_null() { - assert!(len != 0); - let bc = unsafe { slice::from_raw_parts(data, len) }; - - // `bc` must be a sub-slice of `obj`. - assert!(obj.as_ptr() <= bc.as_ptr()); - assert!(bc[bc.len()..bc.len()].as_ptr() <= obj[obj.len()..obj.len()].as_ptr()); - - Ok(bc) - } else { - assert!(len == 0); - let msg = llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()); - Err(format!("failed to get bitcode from object file for LTO ({})", msg)) - } -} - -/// Performs fat LTO by merging all modules into a single one and returning it -/// for further optimization. -pub(crate) fn run_fat( - cgcx: &CodegenContext, - modules: Vec>, - cached_modules: Vec<(SerializedModule, WorkProduct)>, -) -> Result, FatalError> { - let diag_handler = cgcx.create_diag_handler(); - let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?; - let symbols_below_threshold = - symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); - fat_lto( - cgcx, - &diag_handler, - modules, - cached_modules, - upstream_modules, - &symbols_below_threshold, - ) -} - -/// Performs thin LTO by performing necessary global analysis and returning two -/// lists, one of the modules that need optimization and another for modules that -/// can simply be copied over from the incr. comp. cache. -pub(crate) fn run_thin( - cgcx: &CodegenContext, - modules: Vec<(String, ThinBuffer)>, - cached_modules: Vec<(SerializedModule, WorkProduct)>, -) -> Result<(Vec>, Vec), FatalError> { - let diag_handler = cgcx.create_diag_handler(); - let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?; - let symbols_below_threshold = - symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); - if cgcx.opts.cg.linker_plugin_lto.enabled() { - unreachable!( - "We should never reach this case if the LTO step \ - is deferred to the linker" - ); - } - thin_lto( - cgcx, - &diag_handler, - modules, - upstream_modules, - cached_modules, - &symbols_below_threshold, - ) -} - -pub(crate) fn prepare_thin(module: ModuleCodegen) -> (String, ThinBuffer) { - let name = module.name.clone(); - let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); - (name, buffer) -} - -fn fat_lto( - cgcx: &CodegenContext, - diag_handler: &Handler, - modules: Vec>, - cached_modules: Vec<(SerializedModule, WorkProduct)>, - mut serialized_modules: Vec<(SerializedModule, CString)>, - symbols_below_threshold: &[*const libc::c_char], -) -> Result, FatalError> { - let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module"); - info!("going for a fat lto"); - - // Sort out all our lists of incoming modules into two lists. - // - // * `serialized_modules` (also and argument to this function) contains all - // modules that are serialized in-memory. - // * `in_memory` contains modules which are already parsed and in-memory, - // such as from multi-CGU builds. - // - // All of `cached_modules` (cached from previous incremental builds) can - // immediately go onto the `serialized_modules` modules list and then we can - // split the `modules` array into these two lists. - let mut in_memory = Vec::new(); - serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| { - info!("pushing cached module {:?}", wp.cgu_name); - (buffer, CString::new(wp.cgu_name).unwrap()) - })); - for module in modules { - match module { - FatLTOInput::InMemory(m) => in_memory.push(m), - FatLTOInput::Serialized { name, buffer } => { - info!("pushing serialized module {:?}", name); - let buffer = SerializedModule::Local(buffer); - serialized_modules.push((buffer, CString::new(name).unwrap())); - } - } - } - - // Find the "costliest" module and merge everything into that codegen unit. - // All the other modules will be serialized and reparsed into the new - // context, so this hopefully avoids serializing and parsing the largest - // codegen unit. - // - // Additionally use a regular module as the base here to ensure that various - // file copy operations in the backend work correctly. The only other kind - // of module here should be an allocator one, and if your crate is smaller - // than the allocator module then the size doesn't really matter anyway. - let costliest_module = in_memory - .iter() - .enumerate() - .filter(|&(_, module)| module.kind == ModuleKind::Regular) - .map(|(i, module)| { - let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) }; - (cost, i) - }) - .max(); - - // If we found a costliest module, we're good to go. Otherwise all our - // inputs were serialized which could happen in the case, for example, that - // all our inputs were incrementally reread from the cache and we're just - // re-executing the LTO passes. If that's the case deserialize the first - // module and create a linker with it. - let module: ModuleCodegen = match costliest_module { - Some((_cost, i)) => in_memory.remove(i), - None => { - assert!(!serialized_modules.is_empty(), "must have at least one serialized module"); - let (buffer, name) = serialized_modules.remove(0); - info!("no in-memory regular modules to choose from, parsing {:?}", name); - ModuleCodegen { - module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), diag_handler)?, - name: name.into_string().unwrap(), - kind: ModuleKind::Regular, - } - } - }; - let mut serialized_bitcode = Vec::new(); - { - let (llcx, llmod) = { - let llvm = &module.module_llvm; - (&llvm.llcx, llvm.llmod()) - }; - info!("using {:?} as a base module", module.name); - - // The linking steps below may produce errors and diagnostics within LLVM - // which we'd like to handle and print, so set up our diagnostic handlers - // (which get unregistered when they go out of scope below). - let _handler = DiagnosticHandlers::new(cgcx, diag_handler, llcx); - - // For all other modules we codegened we'll need to link them into our own - // bitcode. All modules were codegened in their own LLVM context, however, - // and we want to move everything to the same LLVM context. Currently the - // way we know of to do that is to serialize them to a string and them parse - // them later. Not great but hey, that's why it's "fat" LTO, right? - for module in in_memory { - let buffer = ModuleBuffer::new(module.module_llvm.llmod()); - let llmod_id = CString::new(&module.name[..]).unwrap(); - serialized_modules.push((SerializedModule::Local(buffer), llmod_id)); - } - // Sort the modules to ensure we produce deterministic results. - serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1)); - - // For all serialized bitcode files we parse them and link them in as we did - // above, this is all mostly handled in C++. Like above, though, we don't - // know much about the memory management here so we err on the side of being - // save and persist everything with the original module. - let mut linker = Linker::new(llmod); - for (bc_decoded, name) in serialized_modules { - let _timer = cgcx - .prof - .generic_activity_with_arg_recorder("LLVM_fat_lto_link_module", |recorder| { - recorder.record_arg(format!("{:?}", name)) - }); - info!("linking {:?}", name); - let data = bc_decoded.data(); - linker.add(data).map_err(|()| { - let msg = format!("failed to load bitcode of module {:?}", name); - write::llvm_err(diag_handler, &msg) - })?; - serialized_bitcode.push(bc_decoded); - } - drop(linker); - save_temp_bitcode(cgcx, &module, "lto.input"); - - // Internalize everything below threshold to help strip out more modules and such. - unsafe { - let ptr = symbols_below_threshold.as_ptr(); - llvm::LLVMRustRunRestrictionPass( - llmod, - ptr as *const *const libc::c_char, - symbols_below_threshold.len() as libc::size_t, - ); - save_temp_bitcode(cgcx, &module, "lto.after-restriction"); - } - } - - Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode }) -} - -pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); - -impl<'a> Linker<'a> { - pub(crate) fn new(llmod: &'a llvm::Module) -> Self { - unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) } - } - - pub(crate) fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { - unsafe { - if llvm::LLVMRustLinkerAdd( - self.0, - bytecode.as_ptr() as *const libc::c_char, - bytecode.len(), - ) { - Ok(()) - } else { - Err(()) - } - } - } -} - -impl Drop for Linker<'_> { - fn drop(&mut self) { - unsafe { - llvm::LLVMRustLinkerFree(&mut *(self.0 as *mut _)); - } - } -} - -/// Prepare "thin" LTO to get run on these modules. -/// -/// The general structure of ThinLTO is quite different from the structure of -/// "fat" LTO above. With "fat" LTO all LLVM modules in question are merged into -/// one giant LLVM module, and then we run more optimization passes over this -/// big module after internalizing most symbols. Thin LTO, on the other hand, -/// avoid this large bottleneck through more targeted optimization. -/// -/// At a high level Thin LTO looks like: -/// -/// 1. Prepare a "summary" of each LLVM module in question which describes -/// the values inside, cost of the values, etc. -/// 2. Merge the summaries of all modules in question into one "index" -/// 3. Perform some global analysis on this index -/// 4. For each module, use the index and analysis calculated previously to -/// perform local transformations on the module, for example inlining -/// small functions from other modules. -/// 5. Run thin-specific optimization passes over each module, and then code -/// generate everything at the end. -/// -/// The summary for each module is intended to be quite cheap, and the global -/// index is relatively quite cheap to create as well. As a result, the goal of -/// ThinLTO is to reduce the bottleneck on LTO and enable LTO to be used in more -/// situations. For example one cheap optimization is that we can parallelize -/// all codegen modules, easily making use of all the cores on a machine. -/// -/// With all that in mind, the function here is designed at specifically just -/// calculating the *index* for ThinLTO. This index will then be shared amongst -/// all of the `LtoModuleCodegen` units returned below and destroyed once -/// they all go out of scope. -fn thin_lto( - cgcx: &CodegenContext, - diag_handler: &Handler, - modules: Vec<(String, ThinBuffer)>, - serialized_modules: Vec<(SerializedModule, CString)>, - cached_modules: Vec<(SerializedModule, WorkProduct)>, - symbols_below_threshold: &[*const libc::c_char], -) -> Result<(Vec>, Vec), FatalError> { - let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis"); - unsafe { - info!("going for that thin, thin LTO"); - - let green_modules: FxHashMap<_, _> = - cached_modules.iter().map(|&(_, ref wp)| (wp.cgu_name.clone(), wp.clone())).collect(); - - let full_scope_len = modules.len() + serialized_modules.len() + cached_modules.len(); - let mut thin_buffers = Vec::with_capacity(modules.len()); - let mut module_names = Vec::with_capacity(full_scope_len); - let mut thin_modules = Vec::with_capacity(full_scope_len); - - for (i, (name, buffer)) in modules.into_iter().enumerate() { - info!("local module: {} - {}", i, name); - let cname = CString::new(name.clone()).unwrap(); - thin_modules.push(llvm::ThinLTOModule { - identifier: cname.as_ptr(), - data: buffer.data().as_ptr(), - len: buffer.data().len(), - }); - thin_buffers.push(buffer); - module_names.push(cname); - } - - // FIXME: All upstream crates are deserialized internally in the - // function below to extract their summary and modules. Note that - // unlike the loop above we *must* decode and/or read something - // here as these are all just serialized files on disk. An - // improvement, however, to make here would be to store the - // module summary separately from the actual module itself. Right - // now this is store in one large bitcode file, and the entire - // file is deflate-compressed. We could try to bypass some of the - // decompression by storing the index uncompressed and only - // lazily decompressing the bytecode if necessary. - // - // Note that truly taking advantage of this optimization will - // likely be further down the road. We'd have to implement - // incremental ThinLTO first where we could actually avoid - // looking at upstream modules entirely sometimes (the contents, - // we must always unconditionally look at the index). - let mut serialized = Vec::with_capacity(serialized_modules.len() + cached_modules.len()); - - let cached_modules = - cached_modules.into_iter().map(|(sm, wp)| (sm, CString::new(wp.cgu_name).unwrap())); - - for (module, name) in serialized_modules.into_iter().chain(cached_modules) { - info!("upstream or cached module {:?}", name); - thin_modules.push(llvm::ThinLTOModule { - identifier: name.as_ptr(), - data: module.data().as_ptr(), - len: module.data().len(), - }); - serialized.push(module); - module_names.push(name); - } - - // Sanity check - assert_eq!(thin_modules.len(), module_names.len()); - - // Delegate to the C++ bindings to create some data here. Once this is a - // tried-and-true interface we may wish to try to upstream some of this - // to LLVM itself, right now we reimplement a lot of what they do - // upstream... - let data = llvm::LLVMRustCreateThinLTOData( - thin_modules.as_ptr(), - thin_modules.len() as u32, - symbols_below_threshold.as_ptr(), - symbols_below_threshold.len() as u32, - ) - .ok_or_else(|| write::llvm_err(diag_handler, "failed to prepare thin LTO context"))?; - - let data = ThinData(data); - - info!("thin LTO data created"); - - let (key_map_path, prev_key_map, curr_key_map) = if let Some(ref incr_comp_session_dir) = - cgcx.incr_comp_session_dir - { - let path = incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME); - // If the previous file was deleted, or we get an IO error - // reading the file, then we'll just use `None` as the - // prev_key_map, which will force the code to be recompiled. - let prev = - if path.exists() { ThinLTOKeysMap::load_from_file(&path).ok() } else { None }; - let curr = ThinLTOKeysMap::from_thin_lto_modules(&data, &thin_modules, &module_names); - (Some(path), prev, curr) - } else { - // If we don't compile incrementally, we don't need to load the - // import data from LLVM. - assert!(green_modules.is_empty()); - let curr = ThinLTOKeysMap::default(); - (None, None, curr) - }; - info!("thin LTO cache key map loaded"); - info!("prev_key_map: {:#?}", prev_key_map); - info!("curr_key_map: {:#?}", curr_key_map); - - // Throw our data in an `Arc` as we'll be sharing it across threads. We - // also put all memory referenced by the C++ data (buffers, ids, etc) - // into the arc as well. After this we'll create a thin module - // codegen per module in this data. - let shared = Arc::new(ThinShared { - data, - thin_buffers, - serialized_modules: serialized, - module_names, - }); - - let mut copy_jobs = vec![]; - let mut opt_jobs = vec![]; - - info!("checking which modules can be-reused and which have to be re-optimized."); - for (module_index, module_name) in shared.module_names.iter().enumerate() { - let module_name = module_name_to_str(module_name); - if let (Some(prev_key_map), true) = - (prev_key_map.as_ref(), green_modules.contains_key(module_name)) - { - assert!(cgcx.incr_comp_session_dir.is_some()); - - // If a module exists in both the current and the previous session, - // and has the same LTO cache key in both sessions, then we can re-use it - if prev_key_map.keys.get(module_name) == curr_key_map.keys.get(module_name) { - let work_product = green_modules[module_name].clone(); - copy_jobs.push(work_product); - info!(" - {}: re-used", module_name); - assert!(cgcx.incr_comp_session_dir.is_some()); - cgcx.cgu_reuse_tracker.set_actual_reuse(module_name, CguReuse::PostLto); - continue; - } - } - - info!(" - {}: re-compiled", module_name); - opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { - shared: shared.clone(), - idx: module_index, - })); - } - - // Save the current ThinLTO import information for the next compilation - // session, overwriting the previous serialized data (if any). - if let Some(path) = key_map_path { - if let Err(err) = curr_key_map.save_to_file(&path) { - let msg = format!("Error while writing ThinLTO key data: {}", err); - return Err(write::llvm_err(diag_handler, &msg)); - } - } - - Ok((opt_jobs, copy_jobs)) - } -} - -pub(crate) fn run_pass_manager( - cgcx: &CodegenContext, - diag_handler: &Handler, - module: &mut ModuleCodegen, - thin: bool, -) -> Result<(), FatalError> { - let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &*module.name); - let config = cgcx.config(module.kind); - - // Now we have one massive module inside of llmod. Time to run the - // LTO-specific optimization passes that LLVM provides. - // - // This code is based off the code found in llvm's LTO code generator: - // llvm/lib/LTO/LTOCodeGenerator.cpp - debug!("running the pass manager"); - unsafe { - if !llvm::LLVMRustHasModuleFlag( - module.module_llvm.llmod(), - "LTOPostLink".as_ptr().cast(), - 11, - ) { - llvm::LLVMRustAddModuleFlag( - module.module_llvm.llmod(), - llvm::LLVMModFlagBehavior::Error, - "LTOPostLink\0".as_ptr().cast(), - 1, - ); - } - if llvm_util::should_use_new_llvm_pass_manager( - &config.new_llvm_pass_manager, - &cgcx.target_arch, - ) { - let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; - let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); - write::optimize_with_new_llvm_pass_manager( - cgcx, - diag_handler, - module, - config, - opt_level, - opt_stage, - )?; - debug!("lto done"); - return Ok(()); - } - - let pm = llvm::LLVMCreatePassManager(); - llvm::LLVMAddAnalysisPasses(module.module_llvm.tm, pm); - - if config.verify_llvm_ir { - let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); - llvm::LLVMRustAddPass(pm, pass.unwrap()); - } - - let opt_level = config - .opt_level - .map(|x| to_llvm_opt_settings(x).0) - .unwrap_or(llvm::CodeGenOptLevel::None); - with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| { - if thin { - llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm); - } else { - llvm::LLVMRustPassManagerBuilderPopulateLTOPassManager( - b, pm, /* Internalize = */ False, /* RunInliner = */ True, - ); - } - }); - - // We always generate bitcode through ThinLTOBuffers, - // which do not support anonymous globals - if config.bitcode_needed() { - let pass = llvm::LLVMRustFindAndCreatePass("name-anon-globals\0".as_ptr().cast()); - llvm::LLVMRustAddPass(pm, pass.unwrap()); - } - - if config.verify_llvm_ir { - let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); - llvm::LLVMRustAddPass(pm, pass.unwrap()); - } - - llvm::LLVMRunPassManager(pm, module.module_llvm.llmod()); - - llvm::LLVMDisposePassManager(pm); - } - debug!("lto done"); - Ok(()) -} - -pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); - -unsafe impl Send for ModuleBuffer {} -unsafe impl Sync for ModuleBuffer {} - -impl ModuleBuffer { - pub fn new(m: &llvm::Module) -> ModuleBuffer { - ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) }) - } -} - -impl ModuleBufferMethods for ModuleBuffer { - fn data(&self) -> &[u8] { - unsafe { - let ptr = llvm::LLVMRustModuleBufferPtr(self.0); - let len = llvm::LLVMRustModuleBufferLen(self.0); - slice::from_raw_parts(ptr, len) - } - } -} - -impl Drop for ModuleBuffer { - fn drop(&mut self) { - unsafe { - llvm::LLVMRustModuleBufferFree(&mut *(self.0 as *mut _)); - } - } -} - -pub struct ThinData(&'static mut llvm::ThinLTOData); - -unsafe impl Send for ThinData {} -unsafe impl Sync for ThinData {} - -impl Drop for ThinData { - fn drop(&mut self) { - unsafe { - llvm::LLVMRustFreeThinLTOData(&mut *(self.0 as *mut _)); - } - } -} - -pub struct ThinBuffer(&'static mut llvm::ThinLTOBuffer); - -unsafe impl Send for ThinBuffer {} -unsafe impl Sync for ThinBuffer {} - -impl ThinBuffer { - pub fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { - unsafe { - let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); - ThinBuffer(buffer) - } - } -} - -impl ThinBufferMethods for ThinBuffer { - fn data(&self) -> &[u8] { - unsafe { - let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; - let len = llvm::LLVMRustThinLTOBufferLen(self.0); - slice::from_raw_parts(ptr, len) - } - } -} - -impl Drop for ThinBuffer { - fn drop(&mut self) { - unsafe { - llvm::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _)); - } - } -} - -pub unsafe fn optimize_thin_module( - thin_module: ThinModule, - cgcx: &CodegenContext, -) -> Result, FatalError> { - let diag_handler = cgcx.create_diag_handler(); - - let module_name = &thin_module.shared.module_names[thin_module.idx]; - let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap()); - let tm = - (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, &e))?; - - // Right now the implementation we've got only works over serialized - // modules, so we create a fresh new LLVM context and parse the module - // into that context. One day, however, we may do this for upstream - // crates but for locally codegened modules we may be able to reuse - // that LLVM Context and Module. - let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _; - let mut module = ModuleCodegen { - module_llvm: ModuleLlvm { llmod_raw, llcx, tm }, - name: thin_module.name().to_string(), - kind: ModuleKind::Regular, - }; - { - let target = &*module.module_llvm.tm; - let llmod = module.module_llvm.llmod(); - save_temp_bitcode(cgcx, &module, "thin-lto-input"); - - // Before we do much else find the "main" `DICompileUnit` that we'll be - // using below. If we find more than one though then rustc has changed - // in a way we're not ready for, so generate an ICE by returning - // an error. - let mut cu1 = ptr::null_mut(); - let mut cu2 = ptr::null_mut(); - llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); - if !cu2.is_null() { - let msg = "multiple source DICompileUnits found"; - return Err(write::llvm_err(&diag_handler, msg)); - } - - // Up next comes the per-module local analyses that we do for Thin LTO. - // Each of these functions is basically copied from the LLVM - // implementation and then tailored to suit this implementation. Ideally - // each of these would be supported by upstream LLVM but that's perhaps - // a patch for another day! - // - // You can find some more comments about these functions in the LLVM - // bindings we've got (currently `PassWrapper.cpp`) - { - let _timer = - cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name()); - if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); - } - save_temp_bitcode(cgcx, &module, "thin-lto-after-rename"); - } - - { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name()); - if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); - } - save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); - } - - { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name()); - if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); - } - save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); - } - - { - let _timer = - cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name()); - if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); - } - save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); - } - - // Ok now this is a bit unfortunate. This is also something you won't - // find upstream in LLVM's ThinLTO passes! This is a hack for now to - // work around bugs in LLVM. - // - // First discovered in #45511 it was found that as part of ThinLTO - // importing passes LLVM will import `DICompileUnit` metadata - // information across modules. This means that we'll be working with one - // LLVM module that has multiple `DICompileUnit` instances in it (a - // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of - // bugs in LLVM's backend which generates invalid DWARF in a situation - // like this: - // - // https://bugs.llvm.org/show_bug.cgi?id=35212 - // https://bugs.llvm.org/show_bug.cgi?id=35562 - // - // While the first bug there is fixed the second ended up causing #46346 - // which was basically a resurgence of #45511 after LLVM's bug 35212 was - // fixed. - // - // This function below is a huge hack around this problem. The function - // below is defined in `PassWrapper.cpp` and will basically "merge" - // all `DICompileUnit` instances in a module. Basically it'll take all - // the objects, rewrite all pointers of `DISubprogram` to point to the - // first `DICompileUnit`, and then delete all the other units. - // - // This is probably mangling to the debug info slightly (but hopefully - // not too much) but for now at least gets LLVM to emit valid DWARF (or - // so it appears). Hopefully we can remove this once upstream bugs are - // fixed in LLVM. - { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); - llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); - save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); - } - - // Alright now that we've done everything related to the ThinLTO - // analysis it's time to run some optimizations! Here we use the same - // `run_pass_manager` as the "fat" LTO above except that we tell it to - // populate a thin-specific pass manager, which presumably LLVM treats a - // little differently. - { - info!("running thin lto passes over {}", module.name); - run_pass_manager(cgcx, &diag_handler, &mut module, true)?; - save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); - } - } - Ok(module) -} - -/// Maps LLVM module identifiers to their corresponding LLVM LTO cache keys -#[derive(Debug, Default)] -pub struct ThinLTOKeysMap { - // key = llvm name of importing module, value = LLVM cache key - keys: FxHashMap, -} - -impl ThinLTOKeysMap { - fn save_to_file(&self, path: &Path) -> io::Result<()> { - use std::io::Write; - let file = File::create(path)?; - let mut writer = io::BufWriter::new(file); - for (module, key) in &self.keys { - writeln!(writer, "{} {}", module, key)?; - } - Ok(()) - } - - fn load_from_file(path: &Path) -> io::Result { - use std::io::BufRead; - let mut keys = FxHashMap::default(); - let file = File::open(path)?; - for line in io::BufReader::new(file).lines() { - let line = line?; - let mut split = line.split(' '); - let module = split.next().unwrap(); - let key = split.next().unwrap(); - assert_eq!(split.next(), None, "Expected two space-separated values, found {:?}", line); - keys.insert(module.to_string(), key.to_string()); - } - Ok(Self { keys }) - } - - fn from_thin_lto_modules( - data: &ThinData, - modules: &[llvm::ThinLTOModule], - names: &[CString], - ) -> Self { - let keys = iter::zip(modules, names) - .map(|(module, name)| { - let key = build_string(|rust_str| unsafe { - llvm::LLVMRustComputeLTOCacheKey(rust_str, module.identifier, data.0); - }) - .expect("Invalid ThinLTO module key"); - (name.clone().into_string().unwrap(), key) - }) - .collect(); - Self { keys } - } -} - -fn module_name_to_str(c_str: &CStr) -> &str { - c_str.to_str().unwrap_or_else(|e| { - bug!("Encountered non-utf8 LLVM module name `{}`: {}", c_str.to_string_lossy(), e) - }) -} - -pub fn parse_module<'a>( - cx: &'a llvm::Context, - name: &CStr, - data: &[u8], - diag_handler: &Handler, -) -> Result<&'a llvm::Module, FatalError> { - unsafe { - llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else( - || { - let msg = "failed to parse bitcode for LTO module"; - write::llvm_err(diag_handler, msg) - }, - ) - } -} diff --git a/compiler/rustc_codegen_llvm/src/back/profiling.rs b/compiler/rustc_codegen_llvm/src/back/profiling.rs deleted file mode 100644 index 2741f7d848e..00000000000 --- a/compiler/rustc_codegen_llvm/src/back/profiling.rs +++ /dev/null @@ -1,58 +0,0 @@ -use measureme::{event_id::SEPARATOR_BYTE, EventId, StringComponent, StringId}; -use rustc_data_structures::profiling::{SelfProfiler, TimingGuard}; -use std::ffi::{c_void, CStr}; -use std::os::raw::c_char; -use std::sync::Arc; - -fn llvm_args_to_string_id(profiler: &SelfProfiler, pass_name: &str, ir_name: &str) -> EventId { - let pass_name = profiler.get_or_alloc_cached_string(pass_name); - let mut components = vec![StringComponent::Ref(pass_name)]; - // handle that LazyCallGraph::SCC is a comma separated list within parentheses - let parentheses: &[_] = &['(', ')']; - let trimmed = ir_name.trim_matches(parentheses); - for part in trimmed.split(", ") { - let demangled_ir_name = rustc_demangle::demangle(part).to_string(); - let ir_name = profiler.get_or_alloc_cached_string(demangled_ir_name); - components.push(StringComponent::Value(SEPARATOR_BYTE)); - components.push(StringComponent::Ref(ir_name)); - } - EventId::from_label(profiler.alloc_string(components.as_slice())) -} - -pub struct LlvmSelfProfiler<'a> { - profiler: Arc, - stack: Vec>, - llvm_pass_event_kind: StringId, -} - -impl<'a> LlvmSelfProfiler<'a> { - pub fn new(profiler: Arc) -> Self { - let llvm_pass_event_kind = profiler.alloc_string("LLVM Pass"); - Self { profiler, stack: Vec::default(), llvm_pass_event_kind } - } - - fn before_pass_callback(&'a mut self, pass_name: &str, ir_name: &str) { - let event_id = llvm_args_to_string_id(&self.profiler, pass_name, ir_name); - - self.stack.push(TimingGuard::start(&self.profiler, self.llvm_pass_event_kind, event_id)); - } - fn after_pass_callback(&mut self) { - self.stack.pop(); - } -} - -pub unsafe extern "C" fn selfprofile_before_pass_callback( - llvm_self_profiler: *mut c_void, - pass_name: *const c_char, - ir_name: *const c_char, -) { - let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>); - let pass_name = CStr::from_ptr(pass_name).to_str().expect("valid UTF-8"); - let ir_name = CStr::from_ptr(ir_name).to_str().expect("valid UTF-8"); - llvm_self_profiler.before_pass_callback(pass_name, ir_name); -} - -pub unsafe extern "C" fn selfprofile_after_pass_callback(llvm_self_profiler: *mut c_void) { - let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>); - llvm_self_profiler.after_pass_callback(); -} diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs deleted file mode 100644 index 534d32e8a90..00000000000 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ /dev/null @@ -1,1212 +0,0 @@ -use crate::back::lto::ThinBuffer; -use crate::back::profiling::{ - selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler, -}; -use crate::base; -use crate::common; -use crate::consts; -use crate::llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; -use crate::llvm_util; -use crate::type_::Type; -use crate::LlvmCodegenBackend; -use crate::ModuleLlvm; -use rustc_codegen_ssa::back::link::ensure_removed; -use rustc_codegen_ssa::back::write::{ - BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig, - TargetMachineFactoryFn, -}; -use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; -use rustc_data_structures::profiling::SelfProfilerRef; -use rustc_data_structures::small_c_str::SmallCStr; -use rustc_errors::{FatalError, Handler, Level}; -use rustc_fs_util::{link_or_copy, path_to_c_string}; -use rustc_middle::bug; -use rustc_middle::ty::TyCtxt; -use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath}; -use rustc_session::Session; -use rustc_span::symbol::sym; -use rustc_span::InnerSpan; -use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; -use tracing::debug; - -use libc::{c_char, c_int, c_uint, c_void, size_t}; -use std::ffi::CString; -use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; -use std::slice; -use std::str; -use std::sync::Arc; - -pub fn llvm_err(handler: &rustc_errors::Handler, msg: &str) -> FatalError { - match llvm::last_error() { - Some(err) => handler.fatal(&format!("{}: {}", msg, err)), - None => handler.fatal(msg), - } -} - -pub fn write_output_file<'ll>( - handler: &rustc_errors::Handler, - target: &'ll llvm::TargetMachine, - pm: &llvm::PassManager<'ll>, - m: &'ll llvm::Module, - output: &Path, - dwo_output: Option<&Path>, - file_type: llvm::FileType, - self_profiler_ref: &SelfProfilerRef, -) -> Result<(), FatalError> { - debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); - unsafe { - let output_c = path_to_c_string(output); - let dwo_output_c; - let dwo_output_ptr = if let Some(dwo_output) = dwo_output { - dwo_output_c = path_to_c_string(dwo_output); - dwo_output_c.as_ptr() - } else { - std::ptr::null() - }; - let result = llvm::LLVMRustWriteOutputFile( - target, - pm, - m, - output_c.as_ptr(), - dwo_output_ptr, - file_type, - ); - - // Record artifact sizes for self-profiling - if result == llvm::LLVMRustResult::Success { - let artifact_kind = match file_type { - llvm::FileType::ObjectFile => "object_file", - llvm::FileType::AssemblyFile => "assembly_file", - }; - record_artifact_size(self_profiler_ref, artifact_kind, output); - if let Some(dwo_file) = dwo_output { - record_artifact_size(self_profiler_ref, "dwo_file", dwo_file); - } - } - - result.into_result().map_err(|()| { - let msg = format!("could not write output to {}", output.display()); - llvm_err(handler, &msg) - }) - } -} - -pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm::TargetMachine { - let config = TargetMachineFactoryConfig { split_dwarf_file: None }; - // Can't use query system here quite yet because this function is invoked before the query - // system/tcx is set up. - let features = llvm_util::global_llvm_features(sess, false); - target_machine_factory(sess, config::OptLevel::No, &features)(config) - .unwrap_or_else(|err| llvm_err(sess.diagnostic(), &err).raise()) -} - -pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { - let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { - tcx.output_filenames(()).split_dwarf_path( - tcx.sess.split_debuginfo(), - tcx.sess.opts.unstable_opts.split_dwarf_kind, - Some(mod_name), - ) - } else { - None - }; - let config = TargetMachineFactoryConfig { split_dwarf_file }; - target_machine_factory( - &tcx.sess, - tcx.backend_optimization_level(()), - tcx.global_backend_features(()), - )(config) - .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) -} - -pub fn to_llvm_opt_settings( - cfg: config::OptLevel, -) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { - use self::config::OptLevel::*; - match cfg { - No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone), - Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone), - Default => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone), - Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone), - Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault), - SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive), - } -} - -fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel { - use config::OptLevel::*; - match cfg { - No => llvm::PassBuilderOptLevel::O0, - Less => llvm::PassBuilderOptLevel::O1, - Default => llvm::PassBuilderOptLevel::O2, - Aggressive => llvm::PassBuilderOptLevel::O3, - Size => llvm::PassBuilderOptLevel::Os, - SizeMin => llvm::PassBuilderOptLevel::Oz, - } -} - -fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel { - match relocation_model { - RelocModel::Static => llvm::RelocModel::Static, - // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra attribute. - RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC, - RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic, - RelocModel::Ropi => llvm::RelocModel::ROPI, - RelocModel::Rwpi => llvm::RelocModel::RWPI, - RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI, - } -} - -pub(crate) fn to_llvm_code_model(code_model: Option) -> llvm::CodeModel { - match code_model { - Some(CodeModel::Tiny) => llvm::CodeModel::Tiny, - Some(CodeModel::Small) => llvm::CodeModel::Small, - Some(CodeModel::Kernel) => llvm::CodeModel::Kernel, - Some(CodeModel::Medium) => llvm::CodeModel::Medium, - Some(CodeModel::Large) => llvm::CodeModel::Large, - None => llvm::CodeModel::None, - } -} - -pub fn target_machine_factory( - sess: &Session, - optlvl: config::OptLevel, - target_features: &[String], -) -> TargetMachineFactoryFn { - let reloc_model = to_llvm_relocation_model(sess.relocation_model()); - - let (opt_level, _) = to_llvm_opt_settings(optlvl); - let use_softfp = sess.opts.cg.soft_float; - - let ffunction_sections = - sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections); - let fdata_sections = ffunction_sections; - let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names; - - let code_model = to_llvm_code_model(sess.code_model()); - - let mut singlethread = sess.target.singlethread; - - // On the wasm target once the `atomics` feature is enabled that means that - // we're no longer single-threaded, or otherwise we don't want LLVM to - // lower atomic operations to single-threaded operations. - if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) { - singlethread = false; - } - - let triple = SmallCStr::new(&sess.target.llvm_target); - let cpu = SmallCStr::new(llvm_util::target_cpu(sess)); - let features = CString::new(target_features.join(",")).unwrap(); - let abi = SmallCStr::new(&sess.target.llvm_abiname); - let trap_unreachable = - sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); - let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; - - let asm_comments = sess.asm_comments(); - let relax_elf_relocations = - sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); - - let use_init_array = - !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section); - - let path_mapping = sess.source_map().path_mapping().clone(); - - Arc::new(move |config: TargetMachineFactoryConfig| { - let split_dwarf_file = - path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; - let split_dwarf_file = CString::new(split_dwarf_file.to_str().unwrap()).unwrap(); - - let tm = unsafe { - llvm::LLVMRustCreateTargetMachine( - triple.as_ptr(), - cpu.as_ptr(), - features.as_ptr(), - abi.as_ptr(), - code_model, - reloc_model, - opt_level, - use_softfp, - ffunction_sections, - fdata_sections, - funique_section_names, - trap_unreachable, - singlethread, - asm_comments, - emit_stack_size_section, - relax_elf_relocations, - use_init_array, - split_dwarf_file.as_ptr(), - ) - }; - - tm.ok_or_else(|| { - format!("Could not create LLVM TargetMachine for triple: {}", triple.to_str().unwrap()) - }) - }) -} - -pub(crate) fn save_temp_bitcode( - cgcx: &CodegenContext, - module: &ModuleCodegen, - name: &str, -) { - if !cgcx.save_temps { - return; - } - unsafe { - let ext = format!("{}.bc", name); - let cgu = Some(&module.name[..]); - let path = cgcx.output_filenames.temp_path_ext(&ext, cgu); - let cstr = path_to_c_string(&path); - let llmod = module.module_llvm.llmod(); - llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); - } -} - -pub struct DiagnosticHandlers<'a> { - data: *mut (&'a CodegenContext, &'a Handler), - llcx: &'a llvm::Context, - old_handler: Option<&'a llvm::DiagnosticHandler>, -} - -impl<'a> DiagnosticHandlers<'a> { - pub fn new( - cgcx: &'a CodegenContext, - handler: &'a Handler, - llcx: &'a llvm::Context, - ) -> Self { - let remark_passes_all: bool; - let remark_passes: Vec; - match &cgcx.remark { - Passes::All => { - remark_passes_all = true; - remark_passes = Vec::new(); - } - Passes::Some(passes) => { - remark_passes_all = false; - remark_passes = - passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect(); - } - }; - let remark_passes: Vec<*const c_char> = - remark_passes.iter().map(|name: &CString| name.as_ptr()).collect(); - let data = Box::into_raw(Box::new((cgcx, handler))); - unsafe { - let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx); - llvm::LLVMRustContextConfigureDiagnosticHandler( - llcx, - diagnostic_handler, - data.cast(), - remark_passes_all, - remark_passes.as_ptr(), - remark_passes.len(), - ); - llvm::LLVMRustSetInlineAsmDiagnosticHandler(llcx, inline_asm_handler, data.cast()); - DiagnosticHandlers { data, llcx, old_handler } - } - } -} - -impl<'a> Drop for DiagnosticHandlers<'a> { - fn drop(&mut self) { - use std::ptr::null_mut; - unsafe { - llvm::LLVMRustSetInlineAsmDiagnosticHandler(self.llcx, inline_asm_handler, null_mut()); - llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler); - drop(Box::from_raw(self.data)); - } - } -} - -fn report_inline_asm( - cgcx: &CodegenContext, - msg: String, - level: llvm::DiagnosticLevel, - mut cookie: c_uint, - source: Option<(String, Vec)>, -) { - // In LTO build we may get srcloc values from other crates which are invalid - // since they use a different source map. To be safe we just suppress these - // in LTO builds. - if matches!(cgcx.lto, Lto::Fat | Lto::Thin) { - cookie = 0; - } - let level = match level { - llvm::DiagnosticLevel::Error => Level::Error { lint: false }, - llvm::DiagnosticLevel::Warning => Level::Warning(None), - llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, - }; - cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); -} - -unsafe extern "C" fn inline_asm_handler(diag: &SMDiagnostic, user: *const c_void, cookie: c_uint) { - if user.is_null() { - return; - } - let (cgcx, _) = *(user as *const (&CodegenContext, &Handler)); - - let smdiag = llvm::diagnostic::SrcMgrDiagnostic::unpack(diag); - report_inline_asm(cgcx, smdiag.message, smdiag.level, cookie, smdiag.source); -} - -unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { - if user.is_null() { - return; - } - let (cgcx, diag_handler) = *(user as *const (&CodegenContext, &Handler)); - - match llvm::diagnostic::Diagnostic::unpack(info) { - llvm::diagnostic::InlineAsm(inline) => { - report_inline_asm(cgcx, inline.message, inline.level, inline.cookie, inline.source); - } - - llvm::diagnostic::Optimization(opt) => { - let enabled = match cgcx.remark { - Passes::All => true, - Passes::Some(ref v) => v.iter().any(|s| *s == opt.pass_name), - }; - - if enabled { - diag_handler.note_without_error(&format!( - "{}:{}:{}: {}: {}", - opt.filename, opt.line, opt.column, opt.pass_name, opt.message, - )); - } - } - llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => { - let msg = llvm::build_string(|s| { - llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) - }) - .expect("non-UTF8 diagnostic"); - diag_handler.warn(&msg); - } - llvm::diagnostic::Unsupported(diagnostic_ref) => { - let msg = llvm::build_string(|s| { - llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) - }) - .expect("non-UTF8 diagnostic"); - diag_handler.err(&msg); - } - llvm::diagnostic::UnknownDiagnostic(..) => {} - } -} - -fn get_pgo_gen_path(config: &ModuleConfig) -> Option { - match config.pgo_gen { - SwitchWithOptPath::Enabled(ref opt_dir_path) => { - let path = if let Some(dir_path) = opt_dir_path { - dir_path.join("default_%m.profraw") - } else { - PathBuf::from("default_%m.profraw") - }; - - Some(CString::new(format!("{}", path.display())).unwrap()) - } - SwitchWithOptPath::Disabled => None, - } -} - -fn get_pgo_use_path(config: &ModuleConfig) -> Option { - config - .pgo_use - .as_ref() - .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) -} - -fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option { - config - .pgo_sample_use - .as_ref() - .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) -} - -pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( - cgcx: &CodegenContext, - diag_handler: &Handler, - module: &ModuleCodegen, - config: &ModuleConfig, - opt_level: config::OptLevel, - opt_stage: llvm::OptStage, -) -> Result<(), FatalError> { - let unroll_loops = - opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin; - let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed(); - let pgo_gen_path = get_pgo_gen_path(config); - let pgo_use_path = get_pgo_use_path(config); - let pgo_sample_use_path = get_pgo_sample_use_path(config); - let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; - // Sanitizer instrumentation is only inserted during the pre-link optimization stage. - let sanitizer_options = if !is_lto { - Some(llvm::SanitizerOptions { - sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS), - sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS), - sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY), - sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY), - sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int, - sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD), - sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS), - sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS), - }) - } else { - None - }; - - let mut llvm_profiler = if cgcx.prof.llvm_recording_enabled() { - Some(LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())) - } else { - None - }; - - let llvm_selfprofiler = - llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); - - let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() }; - - let llvm_plugins = config.llvm_plugins.join(","); - - // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. - // We would have to add upstream support for this first, before we can support - // config.inline_threshold and our more aggressive default thresholds. - let result = llvm::LLVMRustOptimizeWithNewPassManager( - module.module_llvm.llmod(), - &*module.module_llvm.tm, - to_pass_builder_opt_level(opt_level), - opt_stage, - config.no_prepopulate_passes, - config.verify_llvm_ir, - using_thin_buffers, - config.merge_functions, - unroll_loops, - config.vectorize_slp, - config.vectorize_loop, - config.no_builtins, - config.emit_lifetime_markers, - sanitizer_options.as_ref(), - pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), - pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), - config.instrument_coverage, - config.instrument_gcov, - pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), - config.debug_info_for_profiling, - llvm_selfprofiler, - selfprofile_before_pass_callback, - selfprofile_after_pass_callback, - extra_passes.as_ptr().cast(), - extra_passes.len(), - llvm_plugins.as_ptr().cast(), - llvm_plugins.len(), - ); - result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes")) -} - -// Unsafe due to LLVM calls. -pub(crate) unsafe fn optimize( - cgcx: &CodegenContext, - diag_handler: &Handler, - module: &ModuleCodegen, - config: &ModuleConfig, -) -> Result<(), FatalError> { - let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); - - let llmod = module.module_llvm.llmod(); - let llcx = &*module.module_llvm.llcx; - let tm = &*module.module_llvm.tm; - let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); - - let module_name = module.name.clone(); - let module_name = Some(&module_name[..]); - - if let Some(false) = config.new_llvm_pass_manager && llvm_util::get_version() >= (15, 0, 0) { - diag_handler.warn( - "ignoring `-Z new-llvm-pass-manager=no`, which is no longer supported with LLVM 15", - ); - } - - if config.emit_no_opt_bc { - let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); - let out = path_to_c_string(&out); - llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); - } - - if let Some(opt_level) = config.opt_level { - if llvm_util::should_use_new_llvm_pass_manager( - &config.new_llvm_pass_manager, - &cgcx.target_arch, - ) { - let opt_stage = match cgcx.lto { - Lto::Fat => llvm::OptStage::PreLinkFatLTO, - Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, - _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, - _ => llvm::OptStage::PreLinkNoLTO, - }; - return optimize_with_new_llvm_pass_manager( - cgcx, - diag_handler, - module, - config, - opt_level, - opt_stage, - ); - } - - if cgcx.prof.llvm_recording_enabled() { - diag_handler - .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`"); - } - - // Create the two optimizing pass managers. These mirror what clang - // does, and are by populated by LLVM's default PassManagerBuilder. - // Each manager has a different set of passes, but they also share - // some common passes. - let fpm = llvm::LLVMCreateFunctionPassManagerForModule(llmod); - let mpm = llvm::LLVMCreatePassManager(); - - { - let find_pass = |pass_name: &str| { - let pass_name = SmallCStr::new(pass_name); - llvm::LLVMRustFindAndCreatePass(pass_name.as_ptr()) - }; - - if config.verify_llvm_ir { - // Verification should run as the very first pass. - llvm::LLVMRustAddPass(fpm, find_pass("verify").unwrap()); - } - - let mut extra_passes = Vec::new(); - let mut have_name_anon_globals_pass = false; - - for pass_name in &config.passes { - if pass_name == "lint" { - // Linting should also be performed early, directly on the generated IR. - llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap()); - continue; - } - - if let Some(pass) = find_pass(pass_name) { - extra_passes.push(pass); - } else { - diag_handler.warn(&format!("unknown pass `{}`, ignoring", pass_name)); - } - - if pass_name == "name-anon-globals" { - have_name_anon_globals_pass = true; - } - } - - // Instrumentation must be inserted before optimization, - // otherwise LLVM may optimize some functions away which - // breaks llvm-cov. - // - // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp. - if config.instrument_gcov { - llvm::LLVMRustAddPass(mpm, find_pass("insert-gcov-profiling").unwrap()); - } - if config.instrument_coverage { - llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap()); - } - if config.debug_info_for_profiling { - llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap()); - } - - add_sanitizer_passes(config, &mut extra_passes); - - // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need - // to make sure we run LLVM's NameAnonGlobals pass when emitting bitcode; otherwise - // we'll get errors in LLVM. - let using_thin_buffers = config.bitcode_needed(); - if !config.no_prepopulate_passes { - llvm::LLVMAddAnalysisPasses(tm, fpm); - llvm::LLVMAddAnalysisPasses(tm, mpm); - let opt_level = to_llvm_opt_settings(opt_level).0; - let prepare_for_thin_lto = cgcx.lto == Lto::Thin - || cgcx.lto == Lto::ThinLocal - || (cgcx.lto != Lto::Fat && cgcx.opts.cg.linker_plugin_lto.enabled()); - with_llvm_pmb(llmod, config, opt_level, prepare_for_thin_lto, &mut |b| { - llvm::LLVMRustAddLastExtensionPasses( - b, - extra_passes.as_ptr(), - extra_passes.len() as size_t, - ); - llvm::LLVMRustPassManagerBuilderPopulateFunctionPassManager(b, fpm); - llvm::LLVMRustPassManagerBuilderPopulateModulePassManager(b, mpm); - }); - - have_name_anon_globals_pass = have_name_anon_globals_pass || prepare_for_thin_lto; - if using_thin_buffers && !prepare_for_thin_lto { - llvm::LLVMRustAddPass(mpm, find_pass("name-anon-globals").unwrap()); - have_name_anon_globals_pass = true; - } - } else { - // If we don't use the standard pipeline, directly populate the MPM - // with the extra passes. - for pass in extra_passes { - llvm::LLVMRustAddPass(mpm, pass); - } - } - - if using_thin_buffers && !have_name_anon_globals_pass { - // As described above, this will probably cause an error in LLVM - if config.no_prepopulate_passes { - diag_handler.err( - "The current compilation is going to use thin LTO buffers \ - without running LLVM's NameAnonGlobals pass. \ - This will likely cause errors in LLVM. Consider adding \ - -C passes=name-anon-globals to the compiler command line.", - ); - } else { - bug!( - "We are using thin LTO buffers without running the NameAnonGlobals pass. \ - This will likely cause errors in LLVM and should never happen." - ); - } - } - } - - diag_handler.abort_if_errors(); - - // Finally, run the actual optimization passes - { - let _timer = cgcx.prof.extra_verbose_generic_activity( - "LLVM_module_optimize_function_passes", - &*module.name, - ); - llvm::LLVMRustRunFunctionPassManager(fpm, llmod); - } - { - let _timer = cgcx.prof.extra_verbose_generic_activity( - "LLVM_module_optimize_module_passes", - &*module.name, - ); - llvm::LLVMRunPassManager(mpm, llmod); - } - - // Deallocate managers that we're now done with - llvm::LLVMDisposePassManager(fpm); - llvm::LLVMDisposePassManager(mpm); - } - Ok(()) -} - -unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static mut llvm::Pass>) { - if config.sanitizer.contains(SanitizerSet::ADDRESS) { - let recover = config.sanitizer_recover.contains(SanitizerSet::ADDRESS); - passes.push(llvm::LLVMRustCreateAddressSanitizerFunctionPass(recover)); - passes.push(llvm::LLVMRustCreateModuleAddressSanitizerPass(recover)); - } - if config.sanitizer.contains(SanitizerSet::MEMORY) { - let track_origins = config.sanitizer_memory_track_origins as c_int; - let recover = config.sanitizer_recover.contains(SanitizerSet::MEMORY); - passes.push(llvm::LLVMRustCreateMemorySanitizerPass(track_origins, recover)); - } - if config.sanitizer.contains(SanitizerSet::THREAD) { - passes.push(llvm::LLVMRustCreateThreadSanitizerPass()); - } - if config.sanitizer.contains(SanitizerSet::HWADDRESS) { - let recover = config.sanitizer_recover.contains(SanitizerSet::HWADDRESS); - passes.push(llvm::LLVMRustCreateHWAddressSanitizerPass(recover)); - } -} - -pub(crate) fn link( - cgcx: &CodegenContext, - diag_handler: &Handler, - mut modules: Vec>, -) -> Result, FatalError> { - use super::lto::{Linker, ModuleBuffer}; - // Sort the modules by name to ensure to ensure deterministic behavior. - modules.sort_by(|a, b| a.name.cmp(&b.name)); - let (first, elements) = - modules.split_first().expect("Bug! modules must contain at least one module."); - - let mut linker = Linker::new(first.module_llvm.llmod()); - for module in elements { - let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); - let buffer = ModuleBuffer::new(module.module_llvm.llmod()); - linker.add(buffer.data()).map_err(|()| { - let msg = format!("failed to serialize module {:?}", module.name); - llvm_err(diag_handler, &msg) - })?; - } - drop(linker); - Ok(modules.remove(0)) -} - -pub(crate) unsafe fn codegen( - cgcx: &CodegenContext, - diag_handler: &Handler, - module: ModuleCodegen, - config: &ModuleConfig, -) -> Result { - let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name); - { - let llmod = module.module_llvm.llmod(); - let llcx = &*module.module_llvm.llcx; - let tm = &*module.module_llvm.tm; - let module_name = module.name.clone(); - let module_name = Some(&module_name[..]); - let handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); - - if cgcx.msvc_imps_needed { - create_msvc_imps(cgcx, llcx, llmod); - } - - // A codegen-specific pass manager is used to generate object - // files for an LLVM module. - // - // Apparently each of these pass managers is a one-shot kind of - // thing, so we create a new one for each type of output. The - // pass manager passed to the closure should be ensured to not - // escape the closure itself, and the manager should only be - // used once. - unsafe fn with_codegen<'ll, F, R>( - tm: &'ll llvm::TargetMachine, - llmod: &'ll llvm::Module, - no_builtins: bool, - f: F, - ) -> R - where - F: FnOnce(&'ll mut PassManager<'ll>) -> R, - { - let cpm = llvm::LLVMCreatePassManager(); - llvm::LLVMAddAnalysisPasses(tm, cpm); - llvm::LLVMRustAddLibraryInfo(cpm, llmod, no_builtins); - f(cpm) - } - - // Two things to note: - // - If object files are just LLVM bitcode we write bitcode, copy it to - // the .o file, and delete the bitcode if it wasn't otherwise - // requested. - // - If we don't have the integrated assembler then we need to emit - // asm from LLVM and use `gcc` to create the object file. - - let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name); - let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, module_name); - - if config.bitcode_needed() { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name); - let thin = ThinBuffer::new(llmod, config.emit_thin_lto); - let data = thin.data(); - - if let Some(bitcode_filename) = bc_out.file_name() { - cgcx.prof.artifact_size( - "llvm_bitcode", - bitcode_filename.to_string_lossy(), - data.len() as u64, - ); - } - - if config.emit_bc || config.emit_obj == EmitObj::Bitcode { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name); - if let Err(e) = fs::write(&bc_out, data) { - let msg = format!("failed to write bytecode to {}: {}", bc_out.display(), e); - diag_handler.err(&msg); - } - } - - if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name); - embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data); - } - } - - if config.emit_ir { - let _timer = - cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name); - let out = cgcx.output_filenames.temp_path(OutputType::LlvmAssembly, module_name); - let out_c = path_to_c_string(&out); - - extern "C" fn demangle_callback( - input_ptr: *const c_char, - input_len: size_t, - output_ptr: *mut c_char, - output_len: size_t, - ) -> size_t { - let input = - unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) }; - - let Ok(input) = str::from_utf8(input) else { return 0 }; - - let output = unsafe { - slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize) - }; - let mut cursor = io::Cursor::new(output); - - let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 }; - - if write!(cursor, "{:#}", demangled).is_err() { - // Possible only if provided buffer is not big enough - return 0; - } - - cursor.position() as size_t - } - - let result = llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback); - - if result == llvm::LLVMRustResult::Success { - record_artifact_size(&cgcx.prof, "llvm_ir", &out); - } - - result.into_result().map_err(|()| { - let msg = format!("failed to write LLVM IR to {}", out.display()); - llvm_err(diag_handler, &msg) - })?; - } - - if config.emit_asm { - let _timer = - cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name); - let path = cgcx.output_filenames.temp_path(OutputType::Assembly, module_name); - - // We can't use the same module for asm and object code output, - // because that triggers various errors like invalid IR or broken - // binaries. So we must clone the module to produce the asm output - // if we are also producing object code. - let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj { - llvm::LLVMCloneModule(llmod) - } else { - llmod - }; - with_codegen(tm, llmod, config.no_builtins, |cpm| { - write_output_file( - diag_handler, - tm, - cpm, - llmod, - &path, - None, - llvm::FileType::AssemblyFile, - &cgcx.prof, - ) - })?; - } - - match config.emit_obj { - EmitObj::ObjectCode(_) => { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name); - - let dwo_out = cgcx.output_filenames.temp_path_dwo(module_name); - let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) { - // Don't change how DWARF is emitted when disabled. - (SplitDebuginfo::Off, _) => None, - // Don't provide a DWARF object path if split debuginfo is enabled but this is - // a platform that doesn't support Split DWARF. - _ if !cgcx.target_can_use_split_dwarf => None, - // Don't provide a DWARF object path in single mode, sections will be written - // into the object as normal but ignored by linker. - (_, SplitDwarfKind::Single) => None, - // Emit (a subset of the) DWARF into a separate dwarf object file in split - // mode. - (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()), - }; - - with_codegen(tm, llmod, config.no_builtins, |cpm| { - write_output_file( - diag_handler, - tm, - cpm, - llmod, - &obj_out, - dwo_out, - llvm::FileType::ObjectFile, - &cgcx.prof, - ) - })?; - } - - EmitObj::Bitcode => { - debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out); - if let Err(e) = link_or_copy(&bc_out, &obj_out) { - diag_handler.err(&format!("failed to copy bitcode to object file: {}", e)); - } - - if !config.emit_bc { - debug!("removing_bitcode {:?}", bc_out); - ensure_removed(diag_handler, &bc_out); - } - } - - EmitObj::None => {} - } - - drop(handlers); - } - - Ok(module.into_compiled_module( - config.emit_obj != EmitObj::None, - cgcx.target_can_use_split_dwarf - && cgcx.split_debuginfo != SplitDebuginfo::Off - && cgcx.split_dwarf_kind == SplitDwarfKind::Split, - config.emit_bc, - &cgcx.output_filenames, - )) -} - -fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec { - let mut asm = format!(".section {},\"{}\"\n", section_name, section_flags).into_bytes(); - asm.extend_from_slice(b".ascii \""); - asm.reserve(data.len()); - for &byte in data { - if byte == b'\\' || byte == b'"' { - asm.push(b'\\'); - asm.push(byte); - } else if byte < 0x20 || byte >= 0x80 { - // Avoid non UTF-8 inline assembly. Use octal escape sequence, because it is fixed - // width, while hex escapes will consume following characters. - asm.push(b'\\'); - asm.push(b'0' + ((byte >> 6) & 0x7)); - asm.push(b'0' + ((byte >> 3) & 0x7)); - asm.push(b'0' + ((byte >> 0) & 0x7)); - } else { - asm.push(byte); - } - } - asm.extend_from_slice(b"\"\n"); - asm -} - -/// Embed the bitcode of an LLVM module in the LLVM module itself. -/// -/// This is done primarily for iOS where it appears to be standard to compile C -/// code at least with `-fembed-bitcode` which creates two sections in the -/// executable: -/// -/// * __LLVM,__bitcode -/// * __LLVM,__cmdline -/// -/// It appears *both* of these sections are necessary to get the linker to -/// recognize what's going on. A suitable cmdline value is taken from the -/// target spec. -/// -/// Furthermore debug/O1 builds don't actually embed bitcode but rather just -/// embed an empty section. -/// -/// Basically all of this is us attempting to follow in the footsteps of clang -/// on iOS. See #35968 for lots more info. -unsafe fn embed_bitcode( - cgcx: &CodegenContext, - llcx: &llvm::Context, - llmod: &llvm::Module, - cmdline: &str, - bitcode: &[u8], -) { - // We're adding custom sections to the output object file, but we definitely - // do not want these custom sections to make their way into the final linked - // executable. The purpose of these custom sections is for tooling - // surrounding object files to work with the LLVM IR, if necessary. For - // example rustc's own LTO will look for LLVM IR inside of the object file - // in these sections by default. - // - // To handle this is a bit different depending on the object file format - // used by the backend, broken down into a few different categories: - // - // * Mach-O - this is for macOS. Inspecting the source code for the native - // linker here shows that the `.llvmbc` and `.llvmcmd` sections are - // automatically skipped by the linker. In that case there's nothing extra - // that we need to do here. - // - // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and - // `.llvmcmd` sections, so there's nothing extra we need to do. - // - // * COFF - if we don't do anything the linker will by default copy all - // these sections to the output artifact, not what we want! To subvert - // this we want to flag the sections we inserted here as - // `IMAGE_SCN_LNK_REMOVE`. - // - // * ELF - this is very similar to COFF above. One difference is that these - // sections are removed from the output linked artifact when - // `--gc-sections` is passed, which we pass by default. If that flag isn't - // passed though then these sections will show up in the final output. - // Additionally the flag that we need to set here is `SHF_EXCLUDE`. - // - // Unfortunately, LLVM provides no way to set custom section flags. For ELF - // and COFF we emit the sections using module level inline assembly for that - // reason (see issue #90326 for historical background). - let is_apple = cgcx.opts.target_triple.triple().contains("-ios") - || cgcx.opts.target_triple.triple().contains("-darwin") - || cgcx.opts.target_triple.triple().contains("-tvos") - || cgcx.opts.target_triple.triple().contains("-watchos"); - if is_apple - || cgcx.opts.target_triple.triple().starts_with("wasm") - || cgcx.opts.target_triple.triple().starts_with("asmjs") - { - // We don't need custom section flags, create LLVM globals. - let llconst = common::bytes_in_context(llcx, bitcode); - let llglobal = llvm::LLVMAddGlobal( - llmod, - common::val_ty(llconst), - "rustc.embedded.module\0".as_ptr().cast(), - ); - llvm::LLVMSetInitializer(llglobal, llconst); - - let section = if is_apple { "__LLVM,__bitcode\0" } else { ".llvmbc\0" }; - llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); - llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); - llvm::LLVMSetGlobalConstant(llglobal, llvm::True); - - let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); - let llglobal = llvm::LLVMAddGlobal( - llmod, - common::val_ty(llconst), - "rustc.embedded.cmdline\0".as_ptr().cast(), - ); - llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { "__LLVM,__cmdline\0" } else { ".llvmcmd\0" }; - llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); - llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); - } else { - // We need custom section flags, so emit module-level inline assembly. - let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; - let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); - let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); - } -} - -pub unsafe fn with_llvm_pmb( - llmod: &llvm::Module, - config: &ModuleConfig, - opt_level: llvm::CodeGenOptLevel, - prepare_for_thin_lto: bool, - f: &mut dyn FnMut(&llvm::PassManagerBuilder), -) { - use std::ptr; - - // Create the PassManagerBuilder for LLVM. We configure it with - // reasonable defaults and prepare it to actually populate the pass - // manager. - let builder = llvm::LLVMRustPassManagerBuilderCreate(); - let opt_size = config.opt_size.map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); - let inline_threshold = config.inline_threshold; - let pgo_gen_path = get_pgo_gen_path(config); - let pgo_use_path = get_pgo_use_path(config); - let pgo_sample_use_path = get_pgo_sample_use_path(config); - - llvm::LLVMRustConfigurePassManagerBuilder( - builder, - opt_level, - config.merge_functions, - config.vectorize_slp, - config.vectorize_loop, - prepare_for_thin_lto, - pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - opt_size as c_int, - ); - - llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins); - - // Here we match what clang does (kinda). For O0 we only inline - // always-inline functions (but don't add lifetime intrinsics), at O1 we - // inline with lifetime intrinsics, and O2+ we add an inliner with a - // thresholds copied from clang. - match (opt_level, opt_size, inline_threshold) { - (.., Some(t)) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, t); - } - (llvm::CodeGenOptLevel::Aggressive, ..) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 275); - } - (_, llvm::CodeGenOptSizeDefault, _) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 75); - } - (_, llvm::CodeGenOptSizeAggressive, _) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 25); - } - (llvm::CodeGenOptLevel::None, ..) => { - llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); - } - (llvm::CodeGenOptLevel::Less, ..) => { - llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); - } - (llvm::CodeGenOptLevel::Default, ..) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 225); - } - } - - f(builder); - llvm::LLVMRustPassManagerBuilderDispose(builder); -} - -// Create a `__imp_ = &symbol` global for every public static `symbol`. -// This is required to satisfy `dllimport` references to static data in .rlibs -// when using MSVC linker. We do this only for data, as linker can fix up -// code references on its own. -// See #26591, #27438 -fn create_msvc_imps( - cgcx: &CodegenContext, - llcx: &llvm::Context, - llmod: &llvm::Module, -) { - if !cgcx.msvc_imps_needed { - return; - } - // The x86 ABI seems to require that leading underscores are added to symbol - // names, so we need an extra underscore on x86. There's also a leading - // '\x01' here which disables LLVM's symbol mangling (e.g., no extra - // underscores added in front). - let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" }; - - unsafe { - let i8p_ty = Type::i8p_llcx(llcx); - let globals = base::iter_globals(llmod) - .filter(|&val| { - llvm::LLVMRustGetLinkage(val) == llvm::Linkage::ExternalLinkage - && llvm::LLVMIsDeclaration(val) == 0 - }) - .filter_map(|val| { - // Exclude some symbols that we know are not Rust symbols. - let name = llvm::get_value_name(val); - if ignored(name) { None } else { Some((val, name)) } - }) - .map(move |(val, name)| { - let mut imp_name = prefix.as_bytes().to_vec(); - imp_name.extend(name); - let imp_name = CString::new(imp_name).unwrap(); - (imp_name, val) - }) - .collect::>(); - - for (imp_name, val) in globals { - let imp = llvm::LLVMAddGlobal(llmod, i8p_ty, imp_name.as_ptr().cast()); - llvm::LLVMSetInitializer(imp, consts::ptrcast(val, i8p_ty)); - llvm::LLVMRustSetLinkage(imp, llvm::Linkage::ExternalLinkage); - } - } - - // Use this function to exclude certain symbols from `__imp` generation. - fn ignored(symbol_name: &[u8]) -> bool { - // These are symbols generated by LLVM's profiling instrumentation - symbol_name.starts_with(b"__llvm_profile_") - } -} - -fn record_artifact_size( - self_profiler_ref: &SelfProfilerRef, - artifact_kind: &'static str, - path: &Path, -) { - // Don't stat the file if we are not going to record its size. - if !self_profiler_ref.enabled() { - return; - } - - if let Some(artifact_name) = path.file_name() { - let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); - self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size); - } -} diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs deleted file mode 100644 index 86f92dc0239..00000000000 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ /dev/null @@ -1,173 +0,0 @@ -//! Codegen the MIR to the LLVM IR. -//! -//! Hopefully useful general knowledge about codegen: -//! -//! * There's no way to find out the [`Ty`] type of a [`Value`]. Doing so -//! would be "trying to get the eggs out of an omelette" (credit: -//! pcwalton). You can, instead, find out its [`llvm::Type`] by calling [`val_ty`], -//! but one [`llvm::Type`] corresponds to many [`Ty`]s; for instance, `tup(int, int, -//! int)` and `rec(x=int, y=int, z=int)` will have the same [`llvm::Type`]. -//! -//! [`Ty`]: rustc_middle::ty::Ty -//! [`val_ty`]: crate::common::val_ty - -use super::ModuleLlvm; - -use crate::attributes; -use crate::builder::Builder; -use crate::context::CodegenCx; -use crate::llvm; -use crate::value::Value; - -use rustc_codegen_ssa::base::maybe_create_entry_wrapper; -use rustc_codegen_ssa::mono_item::MonoItemExt; -use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::{ModuleCodegen, ModuleKind}; -use rustc_data_structures::small_c_str::SmallCStr; -use rustc_middle::dep_graph; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; -use rustc_middle::mir::mono::{Linkage, Visibility}; -use rustc_middle::ty::TyCtxt; -use rustc_session::config::DebugInfo; -use rustc_span::symbol::Symbol; -use rustc_target::spec::SanitizerSet; - -use std::time::Instant; - -pub struct ValueIter<'ll> { - cur: Option<&'ll Value>, - step: unsafe extern "C" fn(&'ll Value) -> Option<&'ll Value>, -} - -impl<'ll> Iterator for ValueIter<'ll> { - type Item = &'ll Value; - - fn next(&mut self) -> Option<&'ll Value> { - let old = self.cur; - if let Some(old) = old { - self.cur = unsafe { (self.step)(old) }; - } - old - } -} - -pub fn iter_globals(llmod: &llvm::Module) -> ValueIter<'_> { - unsafe { ValueIter { cur: llvm::LLVMGetFirstGlobal(llmod), step: llvm::LLVMGetNextGlobal } } -} - -pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen, u64) { - let start_time = Instant::now(); - - let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx); - let (module, _) = tcx.dep_graph.with_task( - dep_node, - tcx, - cgu_name, - module_codegen, - Some(dep_graph::hash_result), - ); - let time_to_codegen = start_time.elapsed(); - - // We assume that the cost to run LLVM on a CGU is proportional to - // the time we needed for codegenning it. - let cost = time_to_codegen.as_nanos() as u64; - - fn module_codegen(tcx: TyCtxt<'_>, cgu_name: Symbol) -> ModuleCodegen { - let cgu = tcx.codegen_unit(cgu_name); - let _prof_timer = - tcx.prof.generic_activity_with_arg_recorder("codegen_module", |recorder| { - recorder.record_arg(cgu_name.to_string()); - recorder.record_arg(cgu.size_estimate().to_string()); - }); - // Instantiate monomorphizations without filling out definitions yet... - let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str()); - { - let cx = CodegenCx::new(tcx, cgu, &llvm_module); - let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); - for &(mono_item, (linkage, visibility)) in &mono_items { - mono_item.predefine::>(&cx, linkage, visibility); - } - - // ... and now that we have everything pre-defined, fill out those definitions. - for &(mono_item, _) in &mono_items { - mono_item.define::>(&cx); - } - - // If this codegen unit contains the main function, also create the - // wrapper here - if let Some(entry) = maybe_create_entry_wrapper::>(&cx) { - let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty()); - attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); - } - - // Finalize code coverage by injecting the coverage map. Note, the coverage map will - // also be added to the `llvm.compiler.used` variable, created next. - if cx.sess().instrument_coverage() { - cx.coverageinfo_finalize(); - } - - // Create the llvm.used and llvm.compiler.used variables. - if !cx.used_statics().borrow().is_empty() { - cx.create_used_variable() - } - if !cx.compiler_used_statics().borrow().is_empty() { - cx.create_compiler_used_variable() - } - - // Run replace-all-uses-with for statics that need it. This must - // happen after the llvm.used variables are created. - for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { - unsafe { - let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); - llvm::LLVMReplaceAllUsesWith(old_g, bitcast); - llvm::LLVMDeleteGlobal(old_g); - } - } - - // Finalize debuginfo - if cx.sess().opts.debuginfo != DebugInfo::None { - cx.debuginfo_finalize(); - } - } - - ModuleCodegen { - name: cgu_name.to_string(), - module_llvm: llvm_module, - kind: ModuleKind::Regular, - } - } - - (module, cost) -} - -pub fn set_link_section(llval: &Value, attrs: &CodegenFnAttrs) { - let Some(sect) = attrs.link_section else { return }; - unsafe { - let buf = SmallCStr::new(sect.as_str()); - llvm::LLVMSetSection(llval, buf.as_ptr()); - } -} - -pub fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage { - match linkage { - Linkage::External => llvm::Linkage::ExternalLinkage, - Linkage::AvailableExternally => llvm::Linkage::AvailableExternallyLinkage, - Linkage::LinkOnceAny => llvm::Linkage::LinkOnceAnyLinkage, - Linkage::LinkOnceODR => llvm::Linkage::LinkOnceODRLinkage, - Linkage::WeakAny => llvm::Linkage::WeakAnyLinkage, - Linkage::WeakODR => llvm::Linkage::WeakODRLinkage, - Linkage::Appending => llvm::Linkage::AppendingLinkage, - Linkage::Internal => llvm::Linkage::InternalLinkage, - Linkage::Private => llvm::Linkage::PrivateLinkage, - Linkage::ExternalWeak => llvm::Linkage::ExternalWeakLinkage, - Linkage::Common => llvm::Linkage::CommonLinkage, - } -} - -pub fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { - match linkage { - Visibility::Default => llvm::Visibility::Default, - Visibility::Hidden => llvm::Visibility::Hidden, - Visibility::Protected => llvm::Visibility::Protected, - } -} diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs deleted file mode 100644 index d3096c73a8a..00000000000 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ /dev/null @@ -1,1508 +0,0 @@ -use crate::attributes; -use crate::common::Funclet; -use crate::context::CodegenCx; -use crate::llvm::{self, BasicBlock, False}; -use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope}; -use crate::llvm_util; -use crate::type_::Type; -use crate::type_of::LayoutLlvmExt; -use crate::value::Value; -use cstr::cstr; -use libc::{c_char, c_uint}; -use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, TypeKind}; -use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; -use rustc_codegen_ssa::mir::place::PlaceRef; -use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::MemFlags; -use rustc_data_structures::small_c_str::SmallCStr; -use rustc_hir::def_id::DefId; -use rustc_middle::ty::layout::{ - FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, -}; -use rustc_middle::ty::{self, Ty, TyCtxt}; -use rustc_span::Span; -use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; -use rustc_target::spec::{HasTargetSpec, Target}; -use std::borrow::Cow; -use std::ffi::CStr; -use std::iter; -use std::ops::Deref; -use std::ptr; -use tracing::{debug, instrument}; - -// All Builders must have an llfn associated with them -#[must_use] -pub struct Builder<'a, 'll, 'tcx> { - pub llbuilder: &'ll mut llvm::Builder<'ll>, - pub cx: &'a CodegenCx<'ll, 'tcx>, -} - -impl Drop for Builder<'_, '_, '_> { - fn drop(&mut self) { - unsafe { - llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _)); - } - } -} - -// FIXME(eddyb) use a checked constructor when they become `const fn`. -const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") }; - -/// Empty string, to be used where LLVM expects an instruction name, indicating -/// that the instruction is to be left unnamed (i.e. numbered, in textual IR). -// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer. -const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr(); - -impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> { - type Value = as BackendTypes>::Value; - type Function = as BackendTypes>::Function; - type BasicBlock = as BackendTypes>::BasicBlock; - type Type = as BackendTypes>::Type; - type Funclet = as BackendTypes>::Funclet; - - type DIScope = as BackendTypes>::DIScope; - type DILocation = as BackendTypes>::DILocation; - type DIVariable = as BackendTypes>::DIVariable; -} - -impl abi::HasDataLayout for Builder<'_, '_, '_> { - fn data_layout(&self) -> &abi::TargetDataLayout { - self.cx.data_layout() - } -} - -impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> { - #[inline] - fn tcx(&self) -> TyCtxt<'tcx> { - self.cx.tcx - } -} - -impl<'tcx> ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> { - fn param_env(&self) -> ty::ParamEnv<'tcx> { - self.cx.param_env() - } -} - -impl HasTargetSpec for Builder<'_, '_, '_> { - #[inline] - fn target_spec(&self) -> &Target { - self.cx.target_spec() - } -} - -impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { - type LayoutOfResult = TyAndLayout<'tcx>; - - #[inline] - fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - self.cx.handle_layout_err(err, span, ty) - } -} - -impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { - type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; - - #[inline] - fn handle_fn_abi_err( - &self, - err: FnAbiError<'tcx>, - span: Span, - fn_abi_request: FnAbiRequest<'tcx>, - ) -> ! { - self.cx.handle_fn_abi_err(err, span, fn_abi_request) - } -} - -impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> { - type Target = CodegenCx<'ll, 'tcx>; - - #[inline] - fn deref(&self) -> &Self::Target { - self.cx - } -} - -impl<'ll, 'tcx> HasCodegen<'tcx> for Builder<'_, 'll, 'tcx> { - type CodegenCx = CodegenCx<'ll, 'tcx>; -} - -macro_rules! builder_methods_for_value_instructions { - ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => { - $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value { - unsafe { - llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED) - } - })+ - } -} - -impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { - fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self { - let bx = Builder::with_cx(cx); - unsafe { - llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb); - } - bx - } - - fn cx(&self) -> &CodegenCx<'ll, 'tcx> { - self.cx - } - - fn llbb(&self) -> &'ll BasicBlock { - unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) } - } - - fn set_span(&mut self, _span: Span) {} - - fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock { - unsafe { - let name = SmallCStr::new(name); - llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr()) - } - } - - fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock { - Self::append_block(self.cx, self.llfn(), name) - } - - fn switch_to_block(&mut self, llbb: Self::BasicBlock) { - *self = Self::build(self.cx, llbb) - } - - fn ret_void(&mut self) { - unsafe { - llvm::LLVMBuildRetVoid(self.llbuilder); - } - } - - fn ret(&mut self, v: &'ll Value) { - unsafe { - llvm::LLVMBuildRet(self.llbuilder, v); - } - } - - fn br(&mut self, dest: &'ll BasicBlock) { - unsafe { - llvm::LLVMBuildBr(self.llbuilder, dest); - } - } - - fn cond_br( - &mut self, - cond: &'ll Value, - then_llbb: &'ll BasicBlock, - else_llbb: &'ll BasicBlock, - ) { - unsafe { - llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb); - } - } - - fn switch( - &mut self, - v: &'ll Value, - else_llbb: &'ll BasicBlock, - cases: impl ExactSizeIterator, - ) { - let switch = - unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) }; - for (on_val, dest) in cases { - let on_val = self.const_uint_big(self.val_ty(v), on_val); - unsafe { llvm::LLVMAddCase(switch, on_val, dest) } - } - } - - fn invoke( - &mut self, - llty: &'ll Type, - llfn: &'ll Value, - args: &[&'ll Value], - then: &'ll BasicBlock, - catch: &'ll BasicBlock, - funclet: Option<&Funclet<'ll>>, - ) -> &'ll Value { - debug!("invoke {:?} with args ({:?})", llfn, args); - - let args = self.check_call("invoke", llty, llfn, args); - let bundle = funclet.map(|funclet| funclet.bundle()); - let bundle = bundle.as_ref().map(|b| &*b.raw); - - unsafe { - llvm::LLVMRustBuildInvoke( - self.llbuilder, - llty, - llfn, - args.as_ptr(), - args.len() as c_uint, - then, - catch, - bundle, - UNNAMED, - ) - } - } - - fn unreachable(&mut self) { - unsafe { - llvm::LLVMBuildUnreachable(self.llbuilder); - } - } - - builder_methods_for_value_instructions! { - add(a, b) => LLVMBuildAdd, - fadd(a, b) => LLVMBuildFAdd, - sub(a, b) => LLVMBuildSub, - fsub(a, b) => LLVMBuildFSub, - mul(a, b) => LLVMBuildMul, - fmul(a, b) => LLVMBuildFMul, - udiv(a, b) => LLVMBuildUDiv, - exactudiv(a, b) => LLVMBuildExactUDiv, - sdiv(a, b) => LLVMBuildSDiv, - exactsdiv(a, b) => LLVMBuildExactSDiv, - fdiv(a, b) => LLVMBuildFDiv, - urem(a, b) => LLVMBuildURem, - srem(a, b) => LLVMBuildSRem, - frem(a, b) => LLVMBuildFRem, - shl(a, b) => LLVMBuildShl, - lshr(a, b) => LLVMBuildLShr, - ashr(a, b) => LLVMBuildAShr, - and(a, b) => LLVMBuildAnd, - or(a, b) => LLVMBuildOr, - xor(a, b) => LLVMBuildXor, - neg(x) => LLVMBuildNeg, - fneg(x) => LLVMBuildFNeg, - not(x) => LLVMBuildNot, - unchecked_sadd(x, y) => LLVMBuildNSWAdd, - unchecked_uadd(x, y) => LLVMBuildNUWAdd, - unchecked_ssub(x, y) => LLVMBuildNSWSub, - unchecked_usub(x, y) => LLVMBuildNUWSub, - unchecked_smul(x, y) => LLVMBuildNSWMul, - unchecked_umul(x, y) => LLVMBuildNUWMul, - } - - fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - - fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - - fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - - fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - - fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - - fn checked_binop( - &mut self, - oop: OverflowOp, - ty: Ty<'_>, - lhs: Self::Value, - rhs: Self::Value, - ) -> (Self::Value, Self::Value) { - use rustc_middle::ty::{Int, Uint}; - use rustc_middle::ty::{IntTy::*, UintTy::*}; - - let new_kind = match ty.kind() { - Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)), - Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)), - t @ (Uint(_) | Int(_)) => t.clone(), - _ => panic!("tried to get overflow intrinsic for op applied to non-int type"), - }; - - let name = match oop { - OverflowOp::Add => match new_kind { - Int(I8) => "llvm.sadd.with.overflow.i8", - Int(I16) => "llvm.sadd.with.overflow.i16", - Int(I32) => "llvm.sadd.with.overflow.i32", - Int(I64) => "llvm.sadd.with.overflow.i64", - Int(I128) => "llvm.sadd.with.overflow.i128", - - Uint(U8) => "llvm.uadd.with.overflow.i8", - Uint(U16) => "llvm.uadd.with.overflow.i16", - Uint(U32) => "llvm.uadd.with.overflow.i32", - Uint(U64) => "llvm.uadd.with.overflow.i64", - Uint(U128) => "llvm.uadd.with.overflow.i128", - - _ => unreachable!(), - }, - OverflowOp::Sub => match new_kind { - Int(I8) => "llvm.ssub.with.overflow.i8", - Int(I16) => "llvm.ssub.with.overflow.i16", - Int(I32) => "llvm.ssub.with.overflow.i32", - Int(I64) => "llvm.ssub.with.overflow.i64", - Int(I128) => "llvm.ssub.with.overflow.i128", - - Uint(U8) => "llvm.usub.with.overflow.i8", - Uint(U16) => "llvm.usub.with.overflow.i16", - Uint(U32) => "llvm.usub.with.overflow.i32", - Uint(U64) => "llvm.usub.with.overflow.i64", - Uint(U128) => "llvm.usub.with.overflow.i128", - - _ => unreachable!(), - }, - OverflowOp::Mul => match new_kind { - Int(I8) => "llvm.smul.with.overflow.i8", - Int(I16) => "llvm.smul.with.overflow.i16", - Int(I32) => "llvm.smul.with.overflow.i32", - Int(I64) => "llvm.smul.with.overflow.i64", - Int(I128) => "llvm.smul.with.overflow.i128", - - Uint(U8) => "llvm.umul.with.overflow.i8", - Uint(U16) => "llvm.umul.with.overflow.i16", - Uint(U32) => "llvm.umul.with.overflow.i32", - Uint(U64) => "llvm.umul.with.overflow.i64", - Uint(U128) => "llvm.umul.with.overflow.i128", - - _ => unreachable!(), - }, - }; - - let res = self.call_intrinsic(name, &[lhs, rhs]); - (self.extract_value(res, 0), self.extract_value(res, 1)) - } - - fn from_immediate(&mut self, val: Self::Value) -> Self::Value { - if self.cx().val_ty(val) == self.cx().type_i1() { - self.zext(val, self.cx().type_i8()) - } else { - val - } - } - fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value { - if scalar.is_bool() { - return self.trunc(val, self.cx().type_i1()); - } - val - } - - fn alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { - let mut bx = Builder::with_cx(self.cx); - bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) }); - bx.dynamic_alloca(ty, align) - } - - fn dynamic_alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { - unsafe { - let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED); - llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); - alloca - } - } - - fn array_alloca(&mut self, ty: &'ll Type, len: &'ll Value, align: Align) -> &'ll Value { - unsafe { - let alloca = llvm::LLVMBuildArrayAlloca(self.llbuilder, ty, len, UNNAMED); - llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); - alloca - } - } - - fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value { - unsafe { - let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); - llvm::LLVMSetAlignment(load, align.bytes() as c_uint); - load - } - } - - fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { - unsafe { - let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); - llvm::LLVMSetVolatile(load, llvm::True); - load - } - } - - fn atomic_load( - &mut self, - ty: &'ll Type, - ptr: &'ll Value, - order: rustc_codegen_ssa::common::AtomicOrdering, - size: Size, - ) -> &'ll Value { - unsafe { - let load = llvm::LLVMRustBuildAtomicLoad( - self.llbuilder, - ty, - ptr, - UNNAMED, - AtomicOrdering::from_generic(order), - ); - // LLVM requires the alignment of atomic loads to be at least the size of the type. - llvm::LLVMSetAlignment(load, size.bytes() as c_uint); - load - } - } - - #[instrument(level = "trace", skip(self))] - fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> { - assert_eq!(place.llextra.is_some(), place.layout.is_unsized()); - - if place.layout.is_zst() { - return OperandRef::new_zst(self, place.layout); - } - - #[instrument(level = "trace", skip(bx))] - fn scalar_load_metadata<'a, 'll, 'tcx>( - bx: &mut Builder<'a, 'll, 'tcx>, - load: &'ll Value, - scalar: abi::Scalar, - layout: TyAndLayout<'tcx>, - offset: Size, - ) { - if !scalar.is_always_valid(bx) { - bx.noundef_metadata(load); - } - - match scalar.primitive() { - abi::Int(..) => { - if !scalar.is_always_valid(bx) { - bx.range_metadata(load, scalar.valid_range(bx)); - } - } - abi::Pointer => { - if !scalar.valid_range(bx).contains(0) { - bx.nonnull_metadata(load); - } - - if let Some(pointee) = layout.pointee_info_at(bx, offset) { - if let Some(_) = pointee.safe { - bx.align_metadata(load, pointee.align); - } - } - } - abi::F32 | abi::F64 => {} - } - } - - let val = if let Some(llextra) = place.llextra { - OperandValue::Ref(place.llval, Some(llextra), place.align) - } else if place.layout.is_llvm_immediate() { - let mut const_llval = None; - let llty = place.layout.llvm_type(self); - unsafe { - if let Some(global) = llvm::LLVMIsAGlobalVariable(place.llval) { - if llvm::LLVMIsGlobalConstant(global) == llvm::True { - if let Some(init) = llvm::LLVMGetInitializer(global) { - if self.val_ty(init) == llty { - const_llval = Some(init); - } - } - } - } - } - let llval = const_llval.unwrap_or_else(|| { - let load = self.load(llty, place.llval, place.align); - if let abi::Abi::Scalar(scalar) = place.layout.abi { - scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO); - } - load - }); - OperandValue::Immediate(self.to_immediate(llval, place.layout)) - } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { - let b_offset = a.size(self).align_to(b.align(self).abi); - let pair_ty = place.layout.llvm_type(self); - - let mut load = |i, scalar: abi::Scalar, layout, align, offset| { - let llptr = self.struct_gep(pair_ty, place.llval, i as u64); - let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); - let load = self.load(llty, llptr, align); - scalar_load_metadata(self, load, scalar, layout, offset); - self.to_immediate_scalar(load, scalar) - }; - - OperandValue::Pair( - load(0, a, place.layout, place.align, Size::ZERO), - load(1, b, place.layout, place.align.restrict_for_offset(b_offset), b_offset), - ) - } else { - OperandValue::Ref(place.llval, None, place.align) - }; - - OperandRef { val, layout: place.layout } - } - - fn write_operand_repeatedly( - mut self, - cg_elem: OperandRef<'tcx, &'ll Value>, - count: u64, - dest: PlaceRef<'tcx, &'ll Value>, - ) -> Self { - let zero = self.const_usize(0); - let count = self.const_usize(count); - let start = dest.project_index(&mut self, zero).llval; - let end = dest.project_index(&mut self, count).llval; - - let header_bb = self.append_sibling_block("repeat_loop_header"); - let body_bb = self.append_sibling_block("repeat_loop_body"); - let next_bb = self.append_sibling_block("repeat_loop_next"); - - self.br(header_bb); - - let mut header_bx = Self::build(self.cx, header_bb); - let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]); - - let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end); - header_bx.cond_br(keep_going, body_bb, next_bb); - - let mut body_bx = Self::build(self.cx, body_bb); - let align = dest.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size); - cg_elem - .val - .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align)); - - let next = body_bx.inbounds_gep( - self.backend_type(cg_elem.layout), - current, - &[self.const_usize(1)], - ); - body_bx.br(header_bb); - header_bx.add_incoming_to_phi(current, next, body_bb); - - Self::build(self.cx, next_bb) - } - - fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) { - if self.sess().target.arch == "amdgpu" { - // amdgpu/LLVM does something weird and thinks an i64 value is - // split into a v2i32, halving the bitwidth LLVM expects, - // tripping an assertion. So, for now, just disable this - // optimization. - return; - } - - unsafe { - let llty = self.cx.val_ty(load); - let v = [ - self.cx.const_uint_big(llty, range.start), - self.cx.const_uint_big(llty, range.end.wrapping_add(1)), - ]; - - llvm::LLVMSetMetadata( - load, - llvm::MD_range as c_uint, - llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint), - ); - } - } - - fn nonnull_metadata(&mut self, load: &'ll Value) { - unsafe { - llvm::LLVMSetMetadata( - load, - llvm::MD_nonnull as c_uint, - llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), - ); - } - } - - fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { - self.store_with_flags(val, ptr, align, MemFlags::empty()) - } - - fn store_with_flags( - &mut self, - val: &'ll Value, - ptr: &'ll Value, - align: Align, - flags: MemFlags, - ) -> &'ll Value { - debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags); - let ptr = self.check_store(val, ptr); - unsafe { - let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr); - let align = - if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint }; - llvm::LLVMSetAlignment(store, align); - if flags.contains(MemFlags::VOLATILE) { - llvm::LLVMSetVolatile(store, llvm::True); - } - if flags.contains(MemFlags::NONTEMPORAL) { - // According to LLVM [1] building a nontemporal store must - // *always* point to a metadata value of the integer 1. - // - // [1]: https://llvm.org/docs/LangRef.html#store-instruction - let one = self.cx.const_i32(1); - let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1); - llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node); - } - store - } - } - - fn atomic_store( - &mut self, - val: &'ll Value, - ptr: &'ll Value, - order: rustc_codegen_ssa::common::AtomicOrdering, - size: Size, - ) { - debug!("Store {:?} -> {:?}", val, ptr); - let ptr = self.check_store(val, ptr); - unsafe { - let store = llvm::LLVMRustBuildAtomicStore( - self.llbuilder, - val, - ptr, - AtomicOrdering::from_generic(order), - ); - // LLVM requires the alignment of atomic stores to be at least the size of the type. - llvm::LLVMSetAlignment(store, size.bytes() as c_uint); - } - } - - fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value { - unsafe { - llvm::LLVMBuildGEP2( - self.llbuilder, - ty, - ptr, - indices.as_ptr(), - indices.len() as c_uint, - UNNAMED, - ) - } - } - - fn inbounds_gep( - &mut self, - ty: &'ll Type, - ptr: &'ll Value, - indices: &[&'ll Value], - ) -> &'ll Value { - unsafe { - llvm::LLVMBuildInBoundsGEP2( - self.llbuilder, - ty, - ptr, - indices.as_ptr(), - indices.len() as c_uint, - UNNAMED, - ) - } - } - - fn struct_gep(&mut self, ty: &'ll Type, ptr: &'ll Value, idx: u64) -> &'ll Value { - assert_eq!(idx as c_uint as u64, idx); - unsafe { llvm::LLVMBuildStructGEP2(self.llbuilder, ty, ptr, idx as c_uint, UNNAMED) } - } - - /* Casts */ - fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { - self.fptoint_sat(false, val, dest_ty) - } - - fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { - self.fptoint_sat(true, val, dest_ty) - } - - fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - // On WebAssembly the `fptoui` and `fptosi` instructions currently have - // poor codegen. The reason for this is that the corresponding wasm - // instructions, `i32.trunc_f32_s` for example, will trap when the float - // is out-of-bounds, infinity, or nan. This means that LLVM - // automatically inserts control flow around `fptoui` and `fptosi` - // because the LLVM instruction `fptoui` is defined as producing a - // poison value, not having UB on out-of-bounds values. - // - // This method, however, is only used with non-saturating casts that - // have UB on out-of-bounds values. This means that it's ok if we use - // the raw wasm instruction since out-of-bounds values can do whatever - // we like. To ensure that LLVM picks the right instruction we choose - // the raw wasm intrinsic functions which avoid LLVM inserting all the - // other control flow automatically. - if self.sess().target.is_like_wasm { - let src_ty = self.cx.val_ty(val); - if self.cx.type_kind(src_ty) != TypeKind::Vector { - let float_width = self.cx.float_width(src_ty); - let int_width = self.cx.int_width(dest_ty); - let name = match (int_width, float_width) { - (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"), - (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"), - (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"), - (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"), - _ => None, - }; - if let Some(name) = name { - return self.call_intrinsic(name, &[val]); - } - } - } - unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - // see `fptoui` above for why wasm is different here - if self.sess().target.is_like_wasm { - let src_ty = self.cx.val_ty(val); - if self.cx.type_kind(src_ty) != TypeKind::Vector { - let float_width = self.cx.float_width(src_ty); - let int_width = self.cx.int_width(dest_ty); - let name = match (int_width, float_width) { - (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"), - (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"), - (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"), - (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"), - _ => None, - }; - if let Some(name) = name { - return self.call_intrinsic(name, &[val]); - } - } - } - unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { - unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) } - } - - fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) } - } - - /* Comparisons */ - fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - let op = llvm::IntPredicate::from_generic(op); - unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) } - } - - fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - let op = llvm::RealPredicate::from_generic(op); - unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) } - } - - /* Miscellaneous instructions */ - fn memcpy( - &mut self, - dst: &'ll Value, - dst_align: Align, - src: &'ll Value, - src_align: Align, - size: &'ll Value, - flags: MemFlags, - ) { - assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); - let size = self.intcast(size, self.type_isize(), false); - let is_volatile = flags.contains(MemFlags::VOLATILE); - let dst = self.pointercast(dst, self.type_i8p()); - let src = self.pointercast(src, self.type_i8p()); - unsafe { - llvm::LLVMRustBuildMemCpy( - self.llbuilder, - dst, - dst_align.bytes() as c_uint, - src, - src_align.bytes() as c_uint, - size, - is_volatile, - ); - } - } - - fn memmove( - &mut self, - dst: &'ll Value, - dst_align: Align, - src: &'ll Value, - src_align: Align, - size: &'ll Value, - flags: MemFlags, - ) { - assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported"); - let size = self.intcast(size, self.type_isize(), false); - let is_volatile = flags.contains(MemFlags::VOLATILE); - let dst = self.pointercast(dst, self.type_i8p()); - let src = self.pointercast(src, self.type_i8p()); - unsafe { - llvm::LLVMRustBuildMemMove( - self.llbuilder, - dst, - dst_align.bytes() as c_uint, - src, - src_align.bytes() as c_uint, - size, - is_volatile, - ); - } - } - - fn memset( - &mut self, - ptr: &'ll Value, - fill_byte: &'ll Value, - size: &'ll Value, - align: Align, - flags: MemFlags, - ) { - let is_volatile = flags.contains(MemFlags::VOLATILE); - let ptr = self.pointercast(ptr, self.type_i8p()); - unsafe { - llvm::LLVMRustBuildMemSet( - self.llbuilder, - ptr, - align.bytes() as c_uint, - fill_byte, - size, - is_volatile, - ); - } - } - - fn select( - &mut self, - cond: &'ll Value, - then_val: &'ll Value, - else_val: &'ll Value, - ) -> &'ll Value { - unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) } - } - - fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } - } - - fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) } - } - - fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value { - unsafe { - let elt_ty = self.cx.val_ty(elt); - let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64)); - let vec = self.insert_element(undef, elt, self.cx.const_i32(0)); - let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64); - self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty)) - } - } - - fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value { - assert_eq!(idx as c_uint as u64, idx); - unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) } - } - - fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value { - assert_eq!(idx as c_uint as u64, idx); - unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) } - } - - fn set_personality_fn(&mut self, personality: &'ll Value) { - unsafe { - llvm::LLVMSetPersonalityFn(self.llfn(), personality); - } - } - - fn cleanup_landing_pad(&mut self, ty: &'ll Type, pers_fn: &'ll Value) -> &'ll Value { - let landing_pad = self.landing_pad(ty, pers_fn, 1 /* FIXME should this be 0? */); - unsafe { - llvm::LLVMSetCleanup(landing_pad, llvm::True); - } - landing_pad - } - - fn resume(&mut self, exn: &'ll Value) { - unsafe { - llvm::LLVMBuildResume(self.llbuilder, exn); - } - } - - fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { - let name = cstr!("cleanuppad"); - let ret = unsafe { - llvm::LLVMRustBuildCleanupPad( - self.llbuilder, - parent, - args.len() as c_uint, - args.as_ptr(), - name.as_ptr(), - ) - }; - Funclet::new(ret.expect("LLVM does not have support for cleanuppad")) - } - - fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) { - unsafe { - llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) - .expect("LLVM does not have support for cleanupret"); - } - } - - fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { - let name = cstr!("catchpad"); - let ret = unsafe { - llvm::LLVMRustBuildCatchPad( - self.llbuilder, - parent, - args.len() as c_uint, - args.as_ptr(), - name.as_ptr(), - ) - }; - Funclet::new(ret.expect("LLVM does not have support for catchpad")) - } - - fn catch_switch( - &mut self, - parent: Option<&'ll Value>, - unwind: Option<&'ll BasicBlock>, - handlers: &[&'ll BasicBlock], - ) -> &'ll Value { - let name = cstr!("catchswitch"); - let ret = unsafe { - llvm::LLVMRustBuildCatchSwitch( - self.llbuilder, - parent, - unwind, - handlers.len() as c_uint, - name.as_ptr(), - ) - }; - let ret = ret.expect("LLVM does not have support for catchswitch"); - for handler in handlers { - unsafe { - llvm::LLVMRustAddHandler(ret, handler); - } - } - ret - } - - // Atomic Operations - fn atomic_cmpxchg( - &mut self, - dst: &'ll Value, - cmp: &'ll Value, - src: &'ll Value, - mut order: rustc_codegen_ssa::common::AtomicOrdering, - failure_order: rustc_codegen_ssa::common::AtomicOrdering, - weak: bool, - ) -> &'ll Value { - let weak = if weak { llvm::True } else { llvm::False }; - if llvm_util::get_version() < (13, 0, 0) { - use rustc_codegen_ssa::common::AtomicOrdering::*; - // Older llvm has the pre-C++17 restriction on - // success and failure memory ordering, - // requiring the former to be at least as strong as the latter. - // So, for llvm 12, we upgrade the success ordering to a stronger - // one if necessary. - match (order, failure_order) { - (Relaxed, Acquire) => order = Acquire, - (Release, Acquire) => order = AcquireRelease, - (_, SequentiallyConsistent) => order = SequentiallyConsistent, - _ => {} - } - } - unsafe { - llvm::LLVMRustBuildAtomicCmpXchg( - self.llbuilder, - dst, - cmp, - src, - AtomicOrdering::from_generic(order), - AtomicOrdering::from_generic(failure_order), - weak, - ) - } - } - fn atomic_rmw( - &mut self, - op: rustc_codegen_ssa::common::AtomicRmwBinOp, - dst: &'ll Value, - src: &'ll Value, - order: rustc_codegen_ssa::common::AtomicOrdering, - ) -> &'ll Value { - unsafe { - llvm::LLVMBuildAtomicRMW( - self.llbuilder, - AtomicRmwBinOp::from_generic(op), - dst, - src, - AtomicOrdering::from_generic(order), - False, - ) - } - } - - fn atomic_fence( - &mut self, - order: rustc_codegen_ssa::common::AtomicOrdering, - scope: rustc_codegen_ssa::common::SynchronizationScope, - ) { - unsafe { - llvm::LLVMRustBuildAtomicFence( - self.llbuilder, - AtomicOrdering::from_generic(order), - SynchronizationScope::from_generic(scope), - ); - } - } - - fn set_invariant_load(&mut self, load: &'ll Value) { - unsafe { - llvm::LLVMSetMetadata( - load, - llvm::MD_invariant_load as c_uint, - llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), - ); - } - } - - fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) { - self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size); - } - - fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) { - self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size); - } - - fn instrprof_increment( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - num_counters: &'ll Value, - index: &'ll Value, - ) { - debug!( - "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})", - fn_name, hash, num_counters, index - ); - - let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_i8p(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], - self.cx.type_void(), - ); - let args = &[fn_name, hash, num_counters, index]; - let args = self.check_call("call", llty, llfn, args); - - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - None, - ); - } - } - - fn call( - &mut self, - llty: &'ll Type, - llfn: &'ll Value, - args: &[&'ll Value], - funclet: Option<&Funclet<'ll>>, - ) -> &'ll Value { - debug!("call {:?} with args ({:?})", llfn, args); - - let args = self.check_call("call", llty, llfn, args); - let bundle = funclet.map(|funclet| funclet.bundle()); - let bundle = bundle.as_ref().map(|b| &*b.raw); - - unsafe { - llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - bundle, - ) - } - } - - fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } - } - - fn do_not_inline(&mut self, llret: &'ll Value) { - let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); - attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); - } -} - -impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> { - fn get_static(&mut self, def_id: DefId) -> &'ll Value { - // Forward to the `get_static` method of `CodegenCx` - self.cx().get_static(def_id) - } -} - -impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { - fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self { - // Create a fresh builder from the crate context. - let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) }; - Builder { llbuilder, cx } - } - - pub fn llfn(&self) -> &'ll Value { - unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) } - } - - fn position_at_start(&mut self, llbb: &'ll BasicBlock) { - unsafe { - llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb); - } - } - - fn align_metadata(&mut self, load: &'ll Value, align: Align) { - unsafe { - let v = [self.cx.const_u64(align.bytes())]; - - llvm::LLVMSetMetadata( - load, - llvm::MD_align as c_uint, - llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint), - ); - } - } - - fn noundef_metadata(&mut self, load: &'ll Value) { - unsafe { - llvm::LLVMSetMetadata( - load, - llvm::MD_noundef as c_uint, - llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), - ); - } - } - - pub fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) } - } - - pub fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) } - } - - pub fn insert_element( - &mut self, - vec: &'ll Value, - elt: &'ll Value, - idx: &'ll Value, - ) -> &'ll Value { - unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) } - } - - pub fn shuffle_vector( - &mut self, - v1: &'ll Value, - v2: &'ll Value, - mask: &'ll Value, - ) -> &'ll Value { - unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) } - } - - pub fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) } - } - pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } - } - pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - pub fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) } - } - pub fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) } - } - pub fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) } - } - pub fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) } - } - pub fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) } - } - pub fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false) - } - } - pub fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) - } - } - pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } - } - pub fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) } - } - - pub fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) { - unsafe { - llvm::LLVMAddClause(landing_pad, clause); - } - } - - pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value { - let ret = - unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; - ret.expect("LLVM does not have support for catchret") - } - - fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value { - let dest_ptr_ty = self.cx.val_ty(ptr); - let stored_ty = self.cx.val_ty(val); - let stored_ptr_ty = self.cx.type_ptr_to(stored_ty); - - assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer); - - if dest_ptr_ty == stored_ptr_ty { - ptr - } else { - debug!( - "type mismatch in store. \ - Expected {:?}, got {:?}; inserting bitcast", - dest_ptr_ty, stored_ptr_ty - ); - self.bitcast(ptr, stored_ptr_ty) - } - } - - fn check_call<'b>( - &mut self, - typ: &str, - fn_ty: &'ll Type, - llfn: &'ll Value, - args: &'b [&'ll Value], - ) -> Cow<'b, [&'ll Value]> { - assert!( - self.cx.type_kind(fn_ty) == TypeKind::Function, - "builder::{} not passed a function, but {:?}", - typ, - fn_ty - ); - - let param_tys = self.cx.func_params_types(fn_ty); - - let all_args_match = iter::zip(¶m_tys, args.iter().map(|&v| self.val_ty(v))) - .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty); - - if all_args_match { - return Cow::Borrowed(args); - } - - let casted_args: Vec<_> = iter::zip(param_tys, args) - .enumerate() - .map(|(i, (expected_ty, &actual_val))| { - let actual_ty = self.val_ty(actual_val); - if expected_ty != actual_ty { - debug!( - "type mismatch in function call of {:?}. \ - Expected {:?} for param {}, got {:?}; injecting bitcast", - llfn, expected_ty, i, actual_ty - ); - self.bitcast(actual_val, expected_ty) - } else { - actual_val - } - }) - .collect(); - - Cow::Owned(casted_args) - } - - pub fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } - } - - pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { - let (ty, f) = self.cx.get_intrinsic(intrinsic); - self.call(ty, f, args, None) - } - - fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { - let size = size.bytes(); - if size == 0 { - return; - } - - if !self.cx().sess().emit_lifetime_markers() { - return; - } - - let ptr = self.pointercast(ptr, self.cx.type_i8p()); - self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]); - } - - pub(crate) fn phi( - &mut self, - ty: &'ll Type, - vals: &[&'ll Value], - bbs: &[&'ll BasicBlock], - ) -> &'ll Value { - assert_eq!(vals.len(), bbs.len()); - let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) }; - unsafe { - llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint); - phi - } - } - - fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) { - unsafe { - llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint); - } - } - - fn fptoint_sat_broken_in_llvm(&self) -> bool { - match self.tcx.sess.target.arch.as_ref() { - // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083 - "riscv64" => llvm_util::get_version() < (13, 0, 0), - _ => false, - } - } - - fn fptoint_sat( - &mut self, - signed: bool, - val: &'ll Value, - dest_ty: &'ll Type, - ) -> Option<&'ll Value> { - if !self.fptoint_sat_broken_in_llvm() { - let src_ty = self.cx.val_ty(val); - let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector - { - assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); - ( - self.cx.element_type(src_ty), - self.cx.element_type(dest_ty), - Some(self.cx.vector_length(src_ty)), - ) - } else { - (src_ty, dest_ty, None) - }; - let float_width = self.cx.float_width(float_ty); - let int_width = self.cx.int_width(int_ty); - - let instr = if signed { "fptosi" } else { "fptoui" }; - let name = if let Some(vector_length) = vector_length { - format!( - "llvm.{}.sat.v{}i{}.v{}f{}", - instr, vector_length, int_width, vector_length, float_width - ) - } else { - format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) - }; - let f = - self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); - Some(self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None)) - } else { - None - } - } - - pub(crate) fn landing_pad( - &mut self, - ty: &'ll Type, - pers_fn: &'ll Value, - num_clauses: usize, - ) -> &'ll Value { - // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while, - // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The - // personality lives on the parent function anyway. - self.set_personality_fn(pers_fn); - unsafe { - llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs deleted file mode 100644 index 72155d874a2..00000000000 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ /dev/null @@ -1,194 +0,0 @@ -//! Handles codegen of callees as well as other call-related -//! things. Callees are a superset of normal rust values and sometimes -//! have different representations. In particular, top-level fn items -//! and methods are represented as just a fn ptr and not a full -//! closure. - -use crate::abi::FnAbiLlvmExt; -use crate::attributes; -use crate::context::CodegenCx; -use crate::llvm; -use crate::value::Value; -use rustc_codegen_ssa::traits::*; -use tracing::debug; - -use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; -use rustc_middle::ty::{self, Instance, TypeVisitable}; - -/// Codegens a reference to a fn/method item, monomorphizing and -/// inlining as it goes. -/// -/// # Parameters -/// -/// - `cx`: the crate context -/// - `instance`: the instance to be instantiated -pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value { - let tcx = cx.tcx(); - - debug!("get_fn(instance={:?})", instance); - - assert!(!instance.substs.needs_infer()); - assert!(!instance.substs.has_escaping_bound_vars()); - - if let Some(&llfn) = cx.instances.borrow().get(&instance) { - return llfn; - } - - let sym = tcx.symbol_name(instance).name; - debug!( - "get_fn({:?}: {:?}) => {}", - instance, - instance.ty(cx.tcx(), ty::ParamEnv::reveal_all()), - sym - ); - - let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); - - let llfn = if let Some(llfn) = cx.get_declared_value(sym) { - // Create a fn pointer with the new signature. - let llptrty = fn_abi.ptr_to_llvm_type(cx); - - // This is subtle and surprising, but sometimes we have to bitcast - // the resulting fn pointer. The reason has to do with external - // functions. If you have two crates that both bind the same C - // library, they may not use precisely the same types: for - // example, they will probably each declare their own structs, - // which are distinct types from LLVM's point of view (nominal - // types). - // - // Now, if those two crates are linked into an application, and - // they contain inlined code, you can wind up with a situation - // where both of those functions wind up being loaded into this - // application simultaneously. In that case, the same function - // (from LLVM's point of view) requires two types. But of course - // LLVM won't allow one function to have two types. - // - // What we currently do, therefore, is declare the function with - // one of the two types (whichever happens to come first) and then - // bitcast as needed when the function is referenced to make sure - // it has the type we expect. - // - // This can occur on either a crate-local or crate-external - // reference. It also occurs when testing libcore and in some - // other weird situations. Annoying. - if cx.val_ty(llfn) != llptrty { - debug!("get_fn: casting {:?} to {:?}", llfn, llptrty); - cx.const_ptrcast(llfn, llptrty) - } else { - debug!("get_fn: not casting pointer!"); - llfn - } - } else { - let llfn = cx.declare_fn(sym, fn_abi); - debug!("get_fn: not casting pointer!"); - - attributes::from_fn_attrs(cx, llfn, instance); - - let instance_def_id = instance.def_id(); - - // Apply an appropriate linkage/visibility value to our item that we - // just declared. - // - // This is sort of subtle. Inside our codegen unit we started off - // compilation by predefining all our own `MonoItem` instances. That - // is, everything we're codegenning ourselves is already defined. That - // means that anything we're actually codegenning in this codegen unit - // will have hit the above branch in `get_declared_value`. As a result, - // we're guaranteed here that we're declaring a symbol that won't get - // defined, or in other words we're referencing a value from another - // codegen unit or even another crate. - // - // So because this is a foreign value we blanket apply an external - // linkage directive because it's coming from a different object file. - // The visibility here is where it gets tricky. This symbol could be - // referencing some foreign crate or foreign library (an `extern` - // block) in which case we want to leave the default visibility. We may - // also, though, have multiple codegen units. It could be a - // monomorphization, in which case its expected visibility depends on - // whether we are sharing generics or not. The important thing here is - // that the visibility we apply to the declaration is the same one that - // has been applied to the definition (wherever that definition may be). - unsafe { - llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::ExternalLinkage); - - let is_generic = instance.substs.non_erasable_generics().next().is_some(); - - if is_generic { - // This is a monomorphization. Its expected visibility depends - // on whether we are in share-generics mode. - - if cx.tcx.sess.opts.share_generics() { - // We are in share_generics mode. - - if let Some(instance_def_id) = instance_def_id.as_local() { - // This is a definition from the current crate. If the - // definition is unreachable for downstream crates or - // the current crate does not re-export generics, the - // definition of the instance will have been declared - // as `hidden`. - if cx.tcx.is_unreachable_local_definition(instance_def_id) - || !cx.tcx.local_crate_exports_generics() - { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } - } else { - // This is a monomorphization of a generic function - // defined in an upstream crate. - if instance.upstream_monomorphization(tcx).is_some() { - // This is instantiated in another crate. It cannot - // be `hidden`. - } else { - // This is a local instantiation of an upstream definition. - // If the current crate does not re-export it - // (because it is a C library or an executable), it - // will have been declared `hidden`. - if !cx.tcx.local_crate_exports_generics() { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } - } - } - } else { - // When not sharing generics, all instances are in the same - // crate and have hidden visibility - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } - } else { - // This is a non-generic function - if cx.tcx.is_codegened_item(instance_def_id) { - // This is a function that is instantiated in the local crate - - if instance_def_id.is_local() { - // This is function that is defined in the local crate. - // If it is not reachable, it is hidden. - if !cx.tcx.is_reachable_non_generic(instance_def_id) { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } - } else { - // This is a function from an upstream crate that has - // been instantiated here. These are always hidden. - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } - } - } - - // MinGW: For backward compatibility we rely on the linker to decide whether it - // should use dllimport for functions. - if cx.use_dll_storage_attrs - && tcx.is_dllimport_foreign_item(instance_def_id) - && !matches!(tcx.sess.target.env.as_ref(), "gnu" | "uclibc") - { - llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); - } - - if cx.should_assume_dso_local(llfn, true) { - llvm::LLVMRustSetDSOLocal(llfn, true); - } - } - - llfn - }; - - cx.instances.borrow_mut().insert(instance, llfn); - - llfn -} diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs deleted file mode 100644 index fb4da9a5f33..00000000000 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ /dev/null @@ -1,359 +0,0 @@ -//! Code that is useful in various codegen modules. - -use crate::consts::{self, const_alloc_to_llvm}; -pub use crate::context::CodegenCx; -use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, OperandBundleDef, True}; -use crate::type_::Type; -use crate::type_of::LayoutLlvmExt; -use crate::value::Value; - -use rustc_ast::Mutability; -use rustc_codegen_ssa::mir::place::PlaceRef; -use rustc_codegen_ssa::traits::*; -use rustc_middle::bug; -use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; - -use libc::{c_char, c_uint}; -use tracing::debug; - -/* -* A note on nomenclature of linking: "extern", "foreign", and "upcall". -* -* An "extern" is an LLVM symbol we wind up emitting an undefined external -* reference to. This means "we don't have the thing in this compilation unit, -* please make sure you link it in at runtime". This could be a reference to -* C code found in a C library, or rust code found in a rust crate. -* -* Most "externs" are implicitly declared (automatically) as a result of a -* user declaring an extern _module_ dependency; this causes the rust driver -* to locate an extern crate, scan its compilation metadata, and emit extern -* declarations for any symbols used by the declaring crate. -* -* A "foreign" is an extern that references C (or other non-rust ABI) code. -* There is no metadata to scan for extern references so in these cases either -* a header-digester like bindgen, or manual function prototypes, have to -* serve as declarators. So these are usually given explicitly as prototype -* declarations, in rust code, with ABI attributes on them noting which ABI to -* link via. -* -* An "upcall" is a foreign call generated by the compiler (not corresponding -* to any user-written call in the code) into the runtime library, to perform -* some helper task such as bringing a task to life, allocating memory, etc. -* -*/ - -/// A structure representing an active landing pad for the duration of a basic -/// block. -/// -/// Each `Block` may contain an instance of this, indicating whether the block -/// is part of a landing pad or not. This is used to make decision about whether -/// to emit `invoke` instructions (e.g., in a landing pad we don't continue to -/// use `invoke`) and also about various function call metadata. -/// -/// For GNU exceptions (`landingpad` + `resume` instructions) this structure is -/// just a bunch of `None` instances (not too interesting), but for MSVC -/// exceptions (`cleanuppad` + `cleanupret` instructions) this contains data. -/// When inside of a landing pad, each function call in LLVM IR needs to be -/// annotated with which landing pad it's a part of. This is accomplished via -/// the `OperandBundleDef` value created for MSVC landing pads. -pub struct Funclet<'ll> { - cleanuppad: &'ll Value, - operand: OperandBundleDef<'ll>, -} - -impl<'ll> Funclet<'ll> { - pub fn new(cleanuppad: &'ll Value) -> Self { - Funclet { cleanuppad, operand: OperandBundleDef::new("funclet", &[cleanuppad]) } - } - - pub fn cleanuppad(&self) -> &'ll Value { - self.cleanuppad - } - - pub fn bundle(&self) -> &OperandBundleDef<'ll> { - &self.operand - } -} - -impl<'ll> BackendTypes for CodegenCx<'ll, '_> { - type Value = &'ll Value; - // FIXME(eddyb) replace this with a `Function` "subclass" of `Value`. - type Function = &'ll Value; - - type BasicBlock = &'ll BasicBlock; - type Type = &'ll Type; - type Funclet = Funclet<'ll>; - - type DIScope = &'ll llvm::debuginfo::DIScope; - type DILocation = &'ll llvm::debuginfo::DILocation; - type DIVariable = &'ll llvm::debuginfo::DIVariable; -} - -impl<'ll> CodegenCx<'ll, '_> { - pub fn const_array(&self, ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { - unsafe { llvm::LLVMConstArray(ty, elts.as_ptr(), elts.len() as c_uint) } - } - - pub fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { - unsafe { llvm::LLVMConstVector(elts.as_ptr(), elts.len() as c_uint) } - } - - pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { - bytes_in_context(self.llcx, bytes) - } - - pub fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { - unsafe { - assert_eq!(idx as c_uint as u64, idx); - let r = llvm::LLVMGetAggregateElement(v, idx as c_uint).unwrap(); - - debug!("const_get_elt(v={:?}, idx={}, r={:?})", v, idx, r); - - r - } - } -} - -impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn const_null(&self, t: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMConstNull(t) } - } - - fn const_undef(&self, t: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMGetUndef(t) } - } - - fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value { - unsafe { llvm::LLVMConstInt(t, i as u64, True) } - } - - fn const_uint(&self, t: &'ll Type, i: u64) -> &'ll Value { - unsafe { llvm::LLVMConstInt(t, i, False) } - } - - fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value { - unsafe { - let words = [u as u64, (u >> 64) as u64]; - llvm::LLVMConstIntOfArbitraryPrecision(t, 2, words.as_ptr()) - } - } - - fn const_bool(&self, val: bool) -> &'ll Value { - self.const_uint(self.type_i1(), val as u64) - } - - fn const_i16(&self, i: i16) -> &'ll Value { - self.const_int(self.type_i16(), i as i64) - } - - fn const_i32(&self, i: i32) -> &'ll Value { - self.const_int(self.type_i32(), i as i64) - } - - fn const_u32(&self, i: u32) -> &'ll Value { - self.const_uint(self.type_i32(), i as u64) - } - - fn const_u64(&self, i: u64) -> &'ll Value { - self.const_uint(self.type_i64(), i) - } - - fn const_usize(&self, i: u64) -> &'ll Value { - let bit_size = self.data_layout().pointer_size.bits(); - if bit_size < 64 { - // make sure it doesn't overflow - assert!(i < (1 << bit_size)); - } - - self.const_uint(self.isize_ty, i) - } - - fn const_u8(&self, i: u8) -> &'ll Value { - self.const_uint(self.type_i8(), i as u64) - } - - fn const_real(&self, t: &'ll Type, val: f64) -> &'ll Value { - unsafe { llvm::LLVMConstReal(t, val) } - } - - fn const_str(&self, s: &str) -> (&'ll Value, &'ll Value) { - let str_global = *self - .const_str_cache - .borrow_mut() - .raw_entry_mut() - .from_key(s) - .or_insert_with(|| { - let sc = self.const_bytes(s.as_bytes()); - let sym = self.generate_local_symbol_name("str"); - let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { - bug!("symbol `{}` is already defined", sym); - }); - unsafe { - llvm::LLVMSetInitializer(g, sc); - llvm::LLVMSetGlobalConstant(g, True); - llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); - } - (s.to_owned(), g) - }) - .1; - let len = s.len(); - let cs = consts::ptrcast( - str_global, - self.type_ptr_to(self.layout_of(self.tcx.types.str_).llvm_type(self)), - ); - (cs, self.const_usize(len as u64)) - } - - fn const_struct(&self, elts: &[&'ll Value], packed: bool) -> &'ll Value { - struct_in_context(self.llcx, elts, packed) - } - - fn const_to_opt_uint(&self, v: &'ll Value) -> Option { - try_as_const_integral(v).map(|v| unsafe { llvm::LLVMConstIntGetZExtValue(v) }) - } - - fn const_to_opt_u128(&self, v: &'ll Value, sign_ext: bool) -> Option { - try_as_const_integral(v).and_then(|v| unsafe { - let (mut lo, mut hi) = (0u64, 0u64); - let success = llvm::LLVMRustConstInt128Get(v, sign_ext, &mut hi, &mut lo); - success.then_some(hi_lo_to_u128(lo, hi)) - }) - } - - fn zst_to_backend(&self, _llty: &'ll Type) -> &'ll Value { - self.const_undef(self.type_ix(0)) - } - - fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { - let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() }; - match cv { - Scalar::Int(int) => { - let data = int.assert_bits(layout.size(self)); - let llval = self.const_uint_big(self.type_ix(bitsize), data); - if layout.primitive() == Pointer { - unsafe { llvm::LLVMConstIntToPtr(llval, llty) } - } else { - self.const_bitcast(llval, llty) - } - } - Scalar::Ptr(ptr, _size) => { - let (alloc_id, offset) = ptr.into_parts(); - let (base_addr, base_addr_space) = match self.tcx.global_alloc(alloc_id) { - GlobalAlloc::Memory(alloc) => { - let init = const_alloc_to_llvm(self, alloc); - let alloc = alloc.inner(); - let value = match alloc.mutability { - Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), - _ => self.static_addr_of(init, alloc.align, None), - }; - if !self.sess().fewer_names() { - llvm::set_value_name(value, format!("{:?}", alloc_id).as_bytes()); - } - (value, AddressSpace::DATA) - } - GlobalAlloc::Function(fn_instance) => ( - self.get_fn_addr(fn_instance.polymorphize(self.tcx)), - self.data_layout().instruction_address_space, - ), - GlobalAlloc::VTable(ty, trait_ref) => { - let alloc = self - .tcx - .global_alloc(self.tcx.vtable_allocation((ty, trait_ref))) - .unwrap_memory(); - let init = const_alloc_to_llvm(self, alloc); - let value = self.static_addr_of(init, alloc.inner().align, None); - (value, AddressSpace::DATA) - } - GlobalAlloc::Static(def_id) => { - assert!(self.tcx.is_static(def_id)); - assert!(!self.tcx.is_thread_local_static(def_id)); - (self.get_static(def_id), AddressSpace::DATA) - } - }; - let llval = unsafe { - llvm::LLVMRustConstInBoundsGEP2( - self.type_i8(), - self.const_bitcast(base_addr, self.type_i8p_ext(base_addr_space)), - &self.const_usize(offset.bytes()), - 1, - ) - }; - if layout.primitive() != Pointer { - unsafe { llvm::LLVMConstPtrToInt(llval, llty) } - } else { - self.const_bitcast(llval, llty) - } - } - } - } - - fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value { - const_alloc_to_llvm(self, alloc) - } - - fn from_const_alloc( - &self, - layout: TyAndLayout<'tcx>, - alloc: ConstAllocation<'tcx>, - offset: Size, - ) -> PlaceRef<'tcx, &'ll Value> { - let alloc_align = alloc.inner().align; - assert_eq!(alloc_align, layout.align.abi); - let llty = self.type_ptr_to(layout.llvm_type(self)); - let llval = if layout.size == Size::ZERO { - let llval = self.const_usize(alloc_align.bytes()); - unsafe { llvm::LLVMConstIntToPtr(llval, llty) } - } else { - let init = const_alloc_to_llvm(self, alloc); - let base_addr = self.static_addr_of(init, alloc_align, None); - - let llval = unsafe { - llvm::LLVMRustConstInBoundsGEP2( - self.type_i8(), - self.const_bitcast(base_addr, self.type_i8p()), - &self.const_usize(offset.bytes()), - 1, - ) - }; - self.const_bitcast(llval, llty) - }; - PlaceRef::new_sized(llval, layout) - } - - fn const_ptrcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { - consts::ptrcast(val, ty) - } -} - -/// Get the [LLVM type][Type] of a [`Value`]. -pub fn val_ty(v: &Value) -> &Type { - unsafe { llvm::LLVMTypeOf(v) } -} - -pub fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { - unsafe { - let ptr = bytes.as_ptr() as *const c_char; - llvm::LLVMConstStringInContext(llcx, ptr, bytes.len() as c_uint, True) - } -} - -pub fn struct_in_context<'ll>( - llcx: &'ll llvm::Context, - elts: &[&'ll Value], - packed: bool, -) -> &'ll Value { - unsafe { - llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), elts.len() as c_uint, packed as Bool) - } -} - -#[inline] -fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 { - ((hi as u128) << 64) | (lo as u128) -} - -fn try_as_const_integral(v: &Value) -> Option<&ConstantInt> { - unsafe { llvm::LLVMIsAConstantInt(v) } -} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs deleted file mode 100644 index 18467e37082..00000000000 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ /dev/null @@ -1,577 +0,0 @@ -use crate::base; -use crate::common::CodegenCx; -use crate::debuginfo; -use crate::llvm::{self, True}; -use crate::llvm_util; -use crate::type_::Type; -use crate::type_of::LayoutLlvmExt; -use crate::value::Value; -use cstr::cstr; -use libc::c_uint; -use rustc_codegen_ssa::traits::*; -use rustc_hir::def_id::DefId; -use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; -use rustc_middle::mir::interpret::{ - read_target_uint, Allocation, ConstAllocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer, - Scalar as InterpScalar, -}; -use rustc_middle::mir::mono::MonoItem; -use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::{self, Instance, Ty}; -use rustc_middle::{bug, span_bug}; -use rustc_target::abi::{ - AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange, -}; -use std::ops::Range; -use tracing::debug; - -pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value { - let alloc = alloc.inner(); - let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1); - let dl = cx.data_layout(); - let pointer_size = dl.pointer_size.bytes() as usize; - - // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, - // so `range` must be within the bounds of `alloc` and not contain or overlap a relocation. - fn append_chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>( - llvals: &mut Vec<&'ll Value>, - cx: &'a CodegenCx<'ll, 'b>, - alloc: &'a Allocation, - range: Range, - ) { - let chunks = alloc - .init_mask() - .range_as_init_chunks(Size::from_bytes(range.start), Size::from_bytes(range.end)); - - let chunk_to_llval = move |chunk| match chunk { - InitChunk::Init(range) => { - let range = (range.start.bytes() as usize)..(range.end.bytes() as usize); - let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); - cx.const_bytes(bytes) - } - InitChunk::Uninit(range) => { - let len = range.end.bytes() - range.start.bytes(); - cx.const_undef(cx.type_array(cx.type_i8(), len)) - } - }; - - // Generating partially-uninit consts is limited to small numbers of chunks, - // to avoid the cost of generating large complex const expressions. - // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element, - // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`. - let max = if llvm_util::get_version() < (14, 0, 0) { - // Generating partially-uninit consts inhibits optimizations in LLVM < 14. - // See https://github.com/rust-lang/rust/issues/84565. - 1 - } else { - cx.sess().opts.unstable_opts.uninit_const_chunk_threshold - }; - let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; - - if allow_uninit_chunks { - llvals.extend(chunks.map(chunk_to_llval)); - } else { - // If this allocation contains any uninit bytes, codegen as if it was initialized - // (using some arbitrary value for uninit bytes). - let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); - llvals.push(cx.const_bytes(bytes)); - } - } - - let mut next_offset = 0; - for &(offset, alloc_id) in alloc.relocations().iter() { - let offset = offset.bytes(); - assert_eq!(offset as usize as u64, offset); - let offset = offset as usize; - if offset > next_offset { - // This `inspect` is okay since we have checked that it is not within a relocation, it - // is within the bounds of the allocation, and it doesn't affect interpreter execution - // (we inspect the result after interpreter execution). - append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); - } - let ptr_offset = read_target_uint( - dl.endian, - // This `inspect` is okay since it is within the bounds of the allocation, it doesn't - // affect interpreter execution (we inspect the result after interpreter execution), - // and we properly interpret the relocation as a relocation pointer offset. - alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), - ) - .expect("const_alloc_to_llvm: could not read relocation pointer") - as u64; - - let address_space = match cx.tcx.global_alloc(alloc_id) { - GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space, - GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) | GlobalAlloc::VTable(..) => { - AddressSpace::DATA - } - }; - - llvals.push(cx.scalar_to_backend( - InterpScalar::from_pointer( - Pointer::new(alloc_id, Size::from_bytes(ptr_offset)), - &cx.tcx, - ), - Scalar::Initialized { - value: Primitive::Pointer, - valid_range: WrappingRange::full(dl.pointer_size), - }, - cx.type_i8p_ext(address_space), - )); - next_offset = offset + pointer_size; - } - if alloc.len() >= next_offset { - let range = next_offset..alloc.len(); - // This `inspect` is okay since we have check that it is after all relocations, it is - // within the bounds of the allocation, and it doesn't affect interpreter execution (we - // inspect the result after interpreter execution). - append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); - } - - cx.const_struct(&llvals, true) -} - -pub fn codegen_static_initializer<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - def_id: DefId, -) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { - let alloc = cx.tcx.eval_static_initializer(def_id)?; - Ok((const_alloc_to_llvm(cx, alloc), alloc)) -} - -fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) { - // The target may require greater alignment for globals than the type does. - // Note: GCC and Clang also allow `__attribute__((aligned))` on variables, - // which can force it to be smaller. Rust doesn't support this yet. - if let Some(min) = cx.sess().target.min_global_align { - match Align::from_bits(min) { - Ok(min) => align = align.max(min), - Err(err) => { - cx.sess().err(&format!("invalid minimum global alignment: {}", err)); - } - } - } - unsafe { - llvm::LLVMSetAlignment(gv, align.bytes() as u32); - } -} - -fn check_and_apply_linkage<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - attrs: &CodegenFnAttrs, - ty: Ty<'tcx>, - sym: &str, - span_def_id: DefId, -) -> &'ll Value { - let llty = cx.layout_of(ty).llvm_type(cx); - if let Some(linkage) = attrs.linkage { - debug!("get_static: sym={} linkage={:?}", sym, linkage); - - // If this is a static with a linkage specified, then we need to handle - // it a little specially. The typesystem prevents things like &T and - // extern "C" fn() from being non-null, so we can't just declare a - // static and call it a day. Some linkages (like weak) will make it such - // that the static actually has a null value. - let llty2 = if let ty::RawPtr(ref mt) = ty.kind() { - cx.layout_of(mt.ty).llvm_type(cx) - } else { - cx.sess().span_fatal( - cx.tcx.def_span(span_def_id), - "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", - ) - }; - unsafe { - // Declare a symbol `foo` with the desired linkage. - let g1 = cx.declare_global(sym, llty2); - llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage)); - - // Declare an internal global `extern_with_linkage_foo` which - // is initialized with the address of `foo`. If `foo` is - // discarded during linking (for example, if `foo` has weak - // linkage and there are no definitions), then - // `extern_with_linkage_foo` will instead be initialized to - // zero. - let mut real_name = "_rust_extern_with_linkage_".to_string(); - real_name.push_str(sym); - let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { - cx.sess().span_fatal( - cx.tcx.def_span(span_def_id), - &format!("symbol `{}` is already defined", &sym), - ) - }); - llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage); - llvm::LLVMSetInitializer(g2, g1); - g2 - } - } else { - // Generate an external declaration. - // FIXME(nagisa): investigate whether it can be changed into define_global - cx.declare_global(sym, llty) - } -} - -pub fn ptrcast<'ll>(val: &'ll Value, ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMConstPointerCast(val, ty) } -} - -impl<'ll> CodegenCx<'ll, '_> { - pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMConstBitCast(val, ty) } - } - - pub(crate) fn static_addr_of_mut( - &self, - cv: &'ll Value, - align: Align, - kind: Option<&str>, - ) -> &'ll Value { - unsafe { - let gv = match kind { - Some(kind) if !self.tcx.sess.fewer_names() => { - let name = self.generate_local_symbol_name(kind); - let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { - bug!("symbol `{}` is already defined", name); - }); - llvm::LLVMRustSetLinkage(gv, llvm::Linkage::PrivateLinkage); - gv - } - _ => self.define_private_global(self.val_ty(cv)), - }; - llvm::LLVMSetInitializer(gv, cv); - set_global_alignment(self, gv, align); - llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); - gv - } - } - - pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { - let instance = Instance::mono(self.tcx, def_id); - if let Some(&g) = self.instances.borrow().get(&instance) { - return g; - } - - let defined_in_current_codegen_unit = - self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); - assert!( - !defined_in_current_codegen_unit, - "consts::get_static() should always hit the cache for \ - statics defined in the same CGU, but did not for `{:?}`", - def_id - ); - - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); - let sym = self.tcx.symbol_name(instance).name; - let fn_attrs = self.tcx.codegen_fn_attrs(def_id); - - debug!("get_static: sym={} instance={:?} fn_attrs={:?}", sym, instance, fn_attrs); - - let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) { - let llty = self.layout_of(ty).llvm_type(self); - if let Some(g) = self.get_declared_value(sym) { - if self.val_ty(g) != self.type_ptr_to(llty) { - span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); - } - } - - let g = self.declare_global(sym, llty); - - if !self.tcx.is_reachable_non_generic(def_id) { - unsafe { - llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden); - } - } - - g - } else { - check_and_apply_linkage(self, fn_attrs, ty, sym, def_id) - }; - - // Thread-local statics in some other crate need to *always* be linked - // against in a thread-local fashion, so we need to be sure to apply the - // thread-local attribute locally if it was present remotely. If we - // don't do this then linker errors can be generated where the linker - // complains that one object files has a thread local version of the - // symbol and another one doesn't. - if fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { - llvm::set_thread_local_mode(g, self.tls_model); - } - - if !def_id.is_local() { - let needs_dll_storage_attr = self.use_dll_storage_attrs && !self.tcx.is_foreign_item(def_id) && - // ThinLTO can't handle this workaround in all cases, so we don't - // emit the attrs. Instead we make them unnecessary by disallowing - // dynamic linking when linker plugin based LTO is enabled. - !self.tcx.sess.opts.cg.linker_plugin_lto.enabled(); - - // If this assertion triggers, there's something wrong with commandline - // argument validation. - debug_assert!( - !(self.tcx.sess.opts.cg.linker_plugin_lto.enabled() - && self.tcx.sess.target.is_like_windows - && self.tcx.sess.opts.cg.prefer_dynamic) - ); - - if needs_dll_storage_attr { - // This item is external but not foreign, i.e., it originates from an external Rust - // crate. Since we don't know whether this crate will be linked dynamically or - // statically in the final application, we always mark such symbols as 'dllimport'. - // If final linkage happens to be static, we rely on compiler-emitted __imp_ stubs - // to make things work. - // - // However, in some scenarios we defer emission of statics to downstream - // crates, so there are cases where a static with an upstream DefId - // is actually present in the current crate. We can find out via the - // is_codegened_item query. - if !self.tcx.is_codegened_item(def_id) { - unsafe { - llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport); - } - } - } - } - - if self.use_dll_storage_attrs && self.tcx.is_dllimport_foreign_item(def_id) { - // For foreign (native) libs we know the exact storage type to use. - unsafe { - llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport); - } - } - - unsafe { - if self.should_assume_dso_local(g, true) { - llvm::LLVMRustSetDSOLocal(g, true); - } - } - - self.instances.borrow_mut().insert(instance, g); - g - } -} - -impl<'ll> StaticMethods for CodegenCx<'ll, '_> { - fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { - if let Some(&gv) = self.const_globals.borrow().get(&cv) { - unsafe { - // Upgrade the alignment in cases where the same constant is used with different - // alignment requirements - let llalign = align.bytes() as u32; - if llalign > llvm::LLVMGetAlignment(gv) { - llvm::LLVMSetAlignment(gv, llalign); - } - } - return gv; - } - let gv = self.static_addr_of_mut(cv, align, kind); - unsafe { - llvm::LLVMSetGlobalConstant(gv, True); - } - self.const_globals.borrow_mut().insert(cv, gv); - gv - } - - fn codegen_static(&self, def_id: DefId, is_mutable: bool) { - unsafe { - let attrs = self.tcx.codegen_fn_attrs(def_id); - - let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else { - // Error has already been reported - return; - }; - let alloc = alloc.inner(); - - let g = self.get_static(def_id); - - // boolean SSA values are i1, but they have to be stored in i8 slots, - // otherwise some LLVM optimization passes don't work as expected - let mut val_llty = self.val_ty(v); - let v = if val_llty == self.type_i1() { - val_llty = self.type_i8(); - llvm::LLVMConstZExt(v, val_llty) - } else { - v - }; - - let instance = Instance::mono(self.tcx, def_id); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); - let llty = self.layout_of(ty).llvm_type(self); - let g = if val_llty == llty { - g - } else { - // If we created the global with the wrong type, - // correct the type. - let name = llvm::get_value_name(g).to_vec(); - llvm::set_value_name(g, b""); - - let linkage = llvm::LLVMRustGetLinkage(g); - let visibility = llvm::LLVMRustGetVisibility(g); - - let new_g = llvm::LLVMRustGetOrInsertGlobal( - self.llmod, - name.as_ptr().cast(), - name.len(), - val_llty, - ); - - llvm::LLVMRustSetLinkage(new_g, linkage); - llvm::LLVMRustSetVisibility(new_g, visibility); - - // The old global has had its name removed but is returned by - // get_static since it is in the instance cache. Provide an - // alternative lookup that points to the new global so that - // global_asm! can compute the correct mangled symbol name - // for the global. - self.renamed_statics.borrow_mut().insert(def_id, new_g); - - // To avoid breaking any invariants, we leave around the old - // global for the moment; we'll replace all references to it - // with the new global later. (See base::codegen_backend.) - self.statics_to_rauw.borrow_mut().push((g, new_g)); - new_g - }; - set_global_alignment(self, g, self.align_of(ty)); - llvm::LLVMSetInitializer(g, v); - - if self.should_assume_dso_local(g, true) { - llvm::LLVMRustSetDSOLocal(g, true); - } - - // As an optimization, all shared statics which do not have interior - // mutability are placed into read-only memory. - if !is_mutable && self.type_is_freeze(ty) { - llvm::LLVMSetGlobalConstant(g, llvm::True); - } - - debuginfo::build_global_var_di_node(self, def_id, g); - - if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { - llvm::set_thread_local_mode(g, self.tls_model); - - // Do not allow LLVM to change the alignment of a TLS on macOS. - // - // By default a global's alignment can be freely increased. - // This allows LLVM to generate more performant instructions - // e.g., using load-aligned into a SIMD register. - // - // However, on macOS 10.10 or below, the dynamic linker does not - // respect any alignment given on the TLS (radar 24221680). - // This will violate the alignment assumption, and causing segfault at runtime. - // - // This bug is very easy to trigger. In `println!` and `panic!`, - // the `LOCAL_STDOUT`/`LOCAL_STDERR` handles are stored in a TLS, - // which the values would be `mem::replace`d on initialization. - // The implementation of `mem::replace` will use SIMD - // whenever the size is 32 bytes or higher. LLVM notices SIMD is used - // and tries to align `LOCAL_STDOUT`/`LOCAL_STDERR` to a 32-byte boundary, - // which macOS's dyld disregarded and causing crashes - // (see issues #51794, #51758, #50867, #48866 and #44056). - // - // To workaround the bug, we trick LLVM into not increasing - // the global's alignment by explicitly assigning a section to it - // (equivalent to automatically generating a `#[link_section]` attribute). - // See the comment in the `GlobalValue::canIncreaseAlignment()` function - // of `lib/IR/Globals.cpp` for why this works. - // - // When the alignment is not increased, the optimized `mem::replace` - // will use load-unaligned instructions instead, and thus avoiding the crash. - // - // We could remove this hack whenever we decide to drop macOS 10.10 support. - if self.tcx.sess.target.is_like_osx { - // The `inspect` method is okay here because we checked relocations, and - // because we are doing this access to inspect the final interpreter state - // (not as part of the interpreter execution). - // - // FIXME: This check requires that the (arbitrary) value of undefined bytes - // happens to be zero. Instead, we should only check the value of defined bytes - // and set all undefined bytes to zero if this allocation is headed for the - // BSS. - let all_bytes_are_zero = alloc.relocations().is_empty() - && alloc - .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()) - .iter() - .all(|&byte| byte == 0); - - let sect_name = if all_bytes_are_zero { - cstr!("__DATA,__thread_bss") - } else { - cstr!("__DATA,__thread_data") - }; - llvm::LLVMSetSection(g, sect_name.as_ptr()); - } - } - - // Wasm statics with custom link sections get special treatment as they - // go into custom sections of the wasm executable. - if self.tcx.sess.target.is_like_wasm { - if let Some(section) = attrs.link_section { - let section = llvm::LLVMMDStringInContext( - self.llcx, - section.as_str().as_ptr().cast(), - section.as_str().len() as c_uint, - ); - assert!(alloc.relocations().is_empty()); - - // The `inspect` method is okay here because we checked relocations, and - // because we are doing this access to inspect the final interpreter state (not - // as part of the interpreter execution). - let bytes = - alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); - let alloc = llvm::LLVMMDStringInContext( - self.llcx, - bytes.as_ptr().cast(), - bytes.len() as c_uint, - ); - let data = [section, alloc]; - let meta = llvm::LLVMMDNodeInContext(self.llcx, data.as_ptr(), 2); - llvm::LLVMAddNamedMetadataOperand( - self.llmod, - "wasm.custom_sections\0".as_ptr().cast(), - meta, - ); - } - } else { - base::set_link_section(g, attrs); - } - - if attrs.flags.contains(CodegenFnAttrFlags::USED) { - // `USED` and `USED_LINKER` can't be used together. - assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)); - - // The semantics of #[used] in Rust only require the symbol to make it into the - // object file. It is explicitly allowed for the linker to strip the symbol if it - // is dead, which means we are allowed use `llvm.compiler.used` instead of - // `llvm.used` here. - // - // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique - // sections with SHF_GNU_RETAIN flag for llvm.used symbols, which may trigger bugs - // in the handling of `.init_array` (the static constructor list) in versions of - // the gold linker (prior to the one released with binutils 2.36). - // - // That said, we only ever emit these when compiling for ELF targets, unless - // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage - // on other targets, in particular MachO targets have *their* static constructor - // lists broken if `llvm.compiler.used` is emitted rather than llvm.used. However, - // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_typeck`, - // so we don't need to take care of it here. - self.add_compiler_used_global(g); - } - if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { - // `USED` and `USED_LINKER` can't be used together. - assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED)); - - self.add_used_global(g); - } - } - } - - /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*. - fn add_used_global(&self, global: &'ll Value) { - let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; - self.used_statics.borrow_mut().push(cast); - } - - /// Add a global value to a list to be stored in the `llvm.compiler.used` variable, - /// an array of i8*. - fn add_compiler_used_global(&self, global: &'ll Value) { - let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; - self.compiler_used_statics.borrow_mut().push(cast); - } -} diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs deleted file mode 100644 index 5857b83f6c9..00000000000 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ /dev/null @@ -1,1014 +0,0 @@ -use crate::attributes; -use crate::back::write::to_llvm_code_model; -use crate::callee::get_fn; -use crate::coverageinfo; -use crate::debuginfo; -use crate::llvm; -use crate::llvm_util; -use crate::type_::Type; -use crate::value::Value; - -use cstr::cstr; -use rustc_codegen_ssa::base::wants_msvc_seh; -use rustc_codegen_ssa::traits::*; -use rustc_data_structures::base_n; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::small_c_str::SmallCStr; -use rustc_hir::def_id::DefId; -use rustc_middle::mir::mono::CodegenUnit; -use rustc_middle::ty::layout::{ - FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, - TyAndLayout, -}; -use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; -use rustc_middle::{bug, span_bug}; -use rustc_session::config::{BranchProtection, CFGuard, CFProtection}; -use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; -use rustc_session::Session; -use rustc_span::source_map::Span; -use rustc_target::abi::{ - call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, -}; -use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; -use smallvec::SmallVec; - -use std::cell::{Cell, RefCell}; -use std::ffi::CStr; -use std::str; - -/// There is one `CodegenCx` per compilation unit. Each one has its own LLVM -/// `llvm::Context` so that several compilation units may be optimized in parallel. -/// All other LLVM data structures in the `CodegenCx` are tied to that `llvm::Context`. -pub struct CodegenCx<'ll, 'tcx> { - pub tcx: TyCtxt<'tcx>, - pub check_overflow: bool, - pub use_dll_storage_attrs: bool, - pub tls_model: llvm::ThreadLocalMode, - - pub llmod: &'ll llvm::Module, - pub llcx: &'ll llvm::Context, - pub codegen_unit: &'tcx CodegenUnit<'tcx>, - - /// Cache instances of monomorphic and polymorphic items - pub instances: RefCell, &'ll Value>>, - /// Cache generated vtables - pub vtables: - RefCell, Option>), &'ll Value>>, - /// Cache of constant strings, - pub const_str_cache: RefCell>, - - /// Reverse-direction for const ptrs cast from globals. - /// - /// Key is a Value holding a `*T`, - /// Val is a Value holding a `*[T]`. - /// - /// Needed because LLVM loses pointer->pointee association - /// when we ptrcast, and we have to ptrcast during codegen - /// of a `[T]` const because we form a slice, a `(*T,usize)` pair, not - /// a pointer to an LLVM array type. Similar for trait objects. - pub const_unsized: RefCell>, - - /// Cache of emitted const globals (value -> global) - pub const_globals: RefCell>, - - /// List of globals for static variables which need to be passed to the - /// LLVM function ReplaceAllUsesWith (RAUW) when codegen is complete. - /// (We have to make sure we don't invalidate any Values referring - /// to constants.) - pub statics_to_rauw: RefCell>, - - /// Statics that will be placed in the llvm.used variable - /// See for details - pub used_statics: RefCell>, - - /// Statics that will be placed in the llvm.compiler.used variable - /// See for details - pub compiler_used_statics: RefCell>, - - /// Mapping of non-scalar types to llvm types and field remapping if needed. - pub type_lowering: RefCell, Option), TypeLowering<'ll>>>, - - /// Mapping of scalar types to llvm types. - pub scalar_lltypes: RefCell, &'ll Type>>, - - pub pointee_infos: RefCell, Size), Option>>, - pub isize_ty: &'ll Type, - - pub coverage_cx: Option>, - pub dbg_cx: Option>, - - eh_personality: Cell>, - eh_catch_typeinfo: Cell>, - pub rust_try_fn: Cell>, - - intrinsics: RefCell>, - - /// A counter that is used for generating local symbol names - local_gen_sym_counter: Cell, - - /// `codegen_static` will sometimes create a second global variable with a - /// different type and clear the symbol name of the original global. - /// `global_asm!` needs to be able to find this new global so that it can - /// compute the correct mangled symbol name to insert into the asm. - pub renamed_statics: RefCell>, -} - -pub struct TypeLowering<'ll> { - /// Associated LLVM type - pub lltype: &'ll Type, - - /// If padding is used the slice maps fields from source order - /// to llvm order. - pub field_remapping: Option>, -} - -fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { - match tls_model { - TlsModel::GeneralDynamic => llvm::ThreadLocalMode::GeneralDynamic, - TlsModel::LocalDynamic => llvm::ThreadLocalMode::LocalDynamic, - TlsModel::InitialExec => llvm::ThreadLocalMode::InitialExec, - TlsModel::LocalExec => llvm::ThreadLocalMode::LocalExec, - } -} - -pub unsafe fn create_module<'ll>( - tcx: TyCtxt<'_>, - llcx: &'ll llvm::Context, - mod_name: &str, -) -> &'ll llvm::Module { - let sess = tcx.sess; - let mod_name = SmallCStr::new(mod_name); - let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx); - - let mut target_data_layout = sess.target.data_layout.to_string(); - let llvm_version = llvm_util::get_version(); - if llvm_version < (13, 0, 0) { - if sess.target.arch == "powerpc64" { - target_data_layout = target_data_layout.replace("-S128", ""); - } - if sess.target.arch == "wasm32" { - target_data_layout = "e-m:e-p:32:32-i64:64-n32:64-S128".to_string(); - } - if sess.target.arch == "wasm64" { - target_data_layout = "e-m:e-p:64:64-i64:64-n32:64-S128".to_string(); - } - } - if llvm_version < (14, 0, 0) { - if sess.target.llvm_target == "i686-pc-windows-msvc" - || sess.target.llvm_target == "i586-pc-windows-msvc" - { - target_data_layout = - "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32" - .to_string(); - } - if sess.target.arch == "wasm32" { - target_data_layout = target_data_layout.replace("-p10:8:8-p20:8:8", ""); - } - } - - // Ensure the data-layout values hardcoded remain the defaults. - if sess.target.is_builtin { - let tm = crate::back::write::create_informational_target_machine(tcx.sess); - llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, tm); - llvm::LLVMRustDisposeTargetMachine(tm); - - let llvm_data_layout = llvm::LLVMGetDataLayoutStr(llmod); - let llvm_data_layout = str::from_utf8(CStr::from_ptr(llvm_data_layout).to_bytes()) - .expect("got a non-UTF8 data-layout from LLVM"); - - // Unfortunately LLVM target specs change over time, and right now we - // don't have proper support to work with any more than one - // `data_layout` than the one that is in the rust-lang/rust repo. If - // this compiler is configured against a custom LLVM, we may have a - // differing data layout, even though we should update our own to use - // that one. - // - // As an interim hack, if CFG_LLVM_ROOT is not an empty string then we - // disable this check entirely as we may be configured with something - // that has a different target layout. - // - // Unsure if this will actually cause breakage when rustc is configured - // as such. - // - // FIXME(#34960) - let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or(""); - let custom_llvm_used = cfg_llvm_root.trim() != ""; - - if !custom_llvm_used && target_data_layout != llvm_data_layout { - bug!( - "data-layout for target `{rustc_target}`, `{rustc_layout}`, \ - differs from LLVM target's `{llvm_target}` default layout, `{llvm_layout}`", - rustc_target = sess.opts.target_triple, - rustc_layout = target_data_layout, - llvm_target = sess.target.llvm_target, - llvm_layout = llvm_data_layout - ); - } - } - - let data_layout = SmallCStr::new(&target_data_layout); - llvm::LLVMSetDataLayout(llmod, data_layout.as_ptr()); - - let llvm_target = SmallCStr::new(&sess.target.llvm_target); - llvm::LLVMRustSetNormalizedTarget(llmod, llvm_target.as_ptr()); - - let reloc_model = sess.relocation_model(); - if matches!(reloc_model, RelocModel::Pic | RelocModel::Pie) { - llvm::LLVMRustSetModulePICLevel(llmod); - // PIE is potentially more effective than PIC, but can only be used in executables. - // If all our outputs are executables, then we can relax PIC to PIE. - if reloc_model == RelocModel::Pie - || sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) - { - llvm::LLVMRustSetModulePIELevel(llmod); - } - } - - // Linking object files with different code models is undefined behavior - // because the compiler would have to generate additional code (to span - // longer jumps) if a larger code model is used with a smaller one. - // - // See https://reviews.llvm.org/D52322 and https://reviews.llvm.org/D52323. - llvm::LLVMRustSetModuleCodeModel(llmod, to_llvm_code_model(sess.code_model())); - - // If skipping the PLT is enabled, we need to add some module metadata - // to ensure intrinsic calls don't use it. - if !sess.needs_plt() { - let avoid_plt = "RtLibUseGOT\0".as_ptr().cast(); - llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); - } - - if sess.is_sanitizer_cfi_enabled() { - // FIXME(rcvalle): Add support for non canonical jump tables. - let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); - // FIXME(rcvalle): Add it with Override behavior flag. - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Warning, - canonical_jump_tables, - 1, - ); - } - - // Control Flow Guard is currently only supported by the MSVC linker on Windows. - if sess.target.is_like_msvc { - match sess.opts.cg.control_flow_guard { - CFGuard::Disabled => {} - CFGuard::NoChecks => { - // Set `cfguard=1` module flag to emit metadata only. - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Warning, - "cfguard\0".as_ptr() as *const _, - 1, - ) - } - CFGuard::Checks => { - // Set `cfguard=2` module flag to emit metadata and checks. - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Warning, - "cfguard\0".as_ptr() as *const _, - 2, - ) - } - } - } - - if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { - if sess.target.arch != "aarch64" { - sess.err("-Zbranch-protection is only supported on aarch64"); - } else { - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Error, - "branch-target-enforcement\0".as_ptr().cast(), - bti.into(), - ); - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Error, - "sign-return-address\0".as_ptr().cast(), - pac_ret.is_some().into(), - ); - let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Error, - "sign-return-address-all\0".as_ptr().cast(), - pac_opts.leaf.into(), - ); - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Error, - "sign-return-address-with-bkey\0".as_ptr().cast(), - u32::from(pac_opts.key == PAuthKey::B), - ); - } - } - - // Pass on the control-flow protection flags to LLVM (equivalent to `-fcf-protection` in Clang). - if let CFProtection::Branch | CFProtection::Full = sess.opts.unstable_opts.cf_protection { - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Override, - "cf-protection-branch\0".as_ptr().cast(), - 1, - ) - } - if let CFProtection::Return | CFProtection::Full = sess.opts.unstable_opts.cf_protection { - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Override, - "cf-protection-return\0".as_ptr().cast(), - 1, - ) - } - - if sess.opts.unstable_opts.virtual_function_elimination { - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Error, - "Virtual Function Elim\0".as_ptr().cast(), - 1, - ); - } - - llmod -} - -impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { - pub(crate) fn new( - tcx: TyCtxt<'tcx>, - codegen_unit: &'tcx CodegenUnit<'tcx>, - llvm_module: &'ll crate::ModuleLlvm, - ) -> Self { - // An interesting part of Windows which MSVC forces our hand on (and - // apparently MinGW didn't) is the usage of `dllimport` and `dllexport` - // attributes in LLVM IR as well as native dependencies (in C these - // correspond to `__declspec(dllimport)`). - // - // LD (BFD) in MinGW mode can often correctly guess `dllexport` but - // relying on that can result in issues like #50176. - // LLD won't support that and expects symbols with proper attributes. - // Because of that we make MinGW target emit dllexport just like MSVC. - // When it comes to dllimport we use it for constants but for functions - // rely on the linker to do the right thing. Opposed to dllexport this - // task is easy for them (both LD and LLD) and allows us to easily use - // symbols from static libraries in shared libraries. - // - // Whenever a dynamic library is built on Windows it must have its public - // interface specified by functions tagged with `dllexport` or otherwise - // they're not available to be linked against. This poses a few problems - // for the compiler, some of which are somewhat fundamental, but we use - // the `use_dll_storage_attrs` variable below to attach the `dllexport` - // attribute to all LLVM functions that are exported e.g., they're - // already tagged with external linkage). This is suboptimal for a few - // reasons: - // - // * If an object file will never be included in a dynamic library, - // there's no need to attach the dllexport attribute. Most object - // files in Rust are not destined to become part of a dll as binaries - // are statically linked by default. - // * If the compiler is emitting both an rlib and a dylib, the same - // source object file is currently used but with MSVC this may be less - // feasible. The compiler may be able to get around this, but it may - // involve some invasive changes to deal with this. - // - // The flip side of this situation is that whenever you link to a dll and - // you import a function from it, the import should be tagged with - // `dllimport`. At this time, however, the compiler does not emit - // `dllimport` for any declarations other than constants (where it is - // required), which is again suboptimal for even more reasons! - // - // * Calling a function imported from another dll without using - // `dllimport` causes the linker/compiler to have extra overhead (one - // `jmp` instruction on x86) when calling the function. - // * The same object file may be used in different circumstances, so a - // function may be imported from a dll if the object is linked into a - // dll, but it may be just linked against if linked into an rlib. - // * The compiler has no knowledge about whether native functions should - // be tagged dllimport or not. - // - // For now the compiler takes the perf hit (I do not have any numbers to - // this effect) by marking very little as `dllimport` and praying the - // linker will take care of everything. Fixing this problem will likely - // require adding a few attributes to Rust itself (feature gated at the - // start) and then strongly recommending static linkage on Windows! - let use_dll_storage_attrs = tcx.sess.target.is_like_windows; - - let check_overflow = tcx.sess.overflow_checks(); - - let tls_model = to_llvm_tls_model(tcx.sess.tls_model()); - - let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod()); - - let coverage_cx = if tcx.sess.instrument_coverage() { - let covctx = coverageinfo::CrateCoverageContext::new(); - Some(covctx) - } else { - None - }; - - let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None { - let dctx = debuginfo::CodegenUnitDebugContext::new(llmod); - debuginfo::metadata::build_compile_unit_di_node( - tcx, - codegen_unit.name().as_str(), - &dctx, - ); - Some(dctx) - } else { - None - }; - - let isize_ty = Type::ix_llcx(llcx, tcx.data_layout.pointer_size.bits()); - - CodegenCx { - tcx, - check_overflow, - use_dll_storage_attrs, - tls_model, - llmod, - llcx, - codegen_unit, - instances: Default::default(), - vtables: Default::default(), - const_str_cache: Default::default(), - const_unsized: Default::default(), - const_globals: Default::default(), - statics_to_rauw: RefCell::new(Vec::new()), - used_statics: RefCell::new(Vec::new()), - compiler_used_statics: RefCell::new(Vec::new()), - type_lowering: Default::default(), - scalar_lltypes: Default::default(), - pointee_infos: Default::default(), - isize_ty, - coverage_cx, - dbg_cx, - eh_personality: Cell::new(None), - eh_catch_typeinfo: Cell::new(None), - rust_try_fn: Cell::new(None), - intrinsics: Default::default(), - local_gen_sym_counter: Cell::new(0), - renamed_statics: Default::default(), - } - } - - pub(crate) fn statics_to_rauw(&self) -> &RefCell> { - &self.statics_to_rauw - } - - #[inline] - pub fn coverage_context(&self) -> Option<&coverageinfo::CrateCoverageContext<'ll, 'tcx>> { - self.coverage_cx.as_ref() - } - - fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { - let section = cstr!("llvm.metadata"); - let array = self.const_array(self.type_ptr_to(self.type_i8()), values); - - unsafe { - let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); - llvm::LLVMSetInitializer(g, array); - llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); - llvm::LLVMSetSection(g, section.as_ptr()); - } - } -} - -impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn vtables( - &self, - ) -> &RefCell, Option>), &'ll Value>> - { - &self.vtables - } - - fn get_fn(&self, instance: Instance<'tcx>) -> &'ll Value { - get_fn(self, instance) - } - - fn get_fn_addr(&self, instance: Instance<'tcx>) -> &'ll Value { - get_fn(self, instance) - } - - fn eh_personality(&self) -> &'ll Value { - // The exception handling personality function. - // - // If our compilation unit has the `eh_personality` lang item somewhere - // within it, then we just need to codegen that. Otherwise, we're - // building an rlib which will depend on some upstream implementation of - // this function, so we just codegen a generic reference to it. We don't - // specify any of the types for the function, we just make it a symbol - // that LLVM can later use. - // - // Note that MSVC is a little special here in that we don't use the - // `eh_personality` lang item at all. Currently LLVM has support for - // both Dwarf and SEH unwind mechanisms for MSVC targets and uses the - // *name of the personality function* to decide what kind of unwind side - // tables/landing pads to emit. It looks like Dwarf is used by default, - // injecting a dependency on the `_Unwind_Resume` symbol for resuming - // an "exception", but for MSVC we want to force SEH. This means that we - // can't actually have the personality function be our standard - // `rust_eh_personality` function, but rather we wired it up to the - // CRT's custom personality function, which forces LLVM to consider - // landing pads as "landing pads for SEH". - if let Some(llpersonality) = self.eh_personality.get() { - return llpersonality; - } - let tcx = self.tcx; - let llfn = match tcx.lang_items().eh_personality() { - Some(def_id) if !wants_msvc_seh(self.sess()) => self.get_fn_addr( - ty::Instance::resolve( - tcx, - ty::ParamEnv::reveal_all(), - def_id, - tcx.intern_substs(&[]), - ) - .unwrap() - .unwrap(), - ), - _ => { - let name = if wants_msvc_seh(self.sess()) { - "__CxxFrameHandler3" - } else { - "rust_eh_personality" - }; - if let Some(llfn) = self.get_declared_value(name) { - llfn - } else { - let fty = self.type_variadic_func(&[], self.type_i32()); - let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty); - let target_cpu = attributes::target_cpu_attr(self); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]); - llfn - } - } - }; - self.eh_personality.set(Some(llfn)); - llfn - } - - fn sess(&self) -> &Session { - self.tcx.sess - } - - fn check_overflow(&self) -> bool { - self.check_overflow - } - - fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx> { - self.codegen_unit - } - - fn used_statics(&self) -> &RefCell> { - &self.used_statics - } - - fn compiler_used_statics(&self) -> &RefCell> { - &self.compiler_used_statics - } - - fn set_frame_pointer_type(&self, llfn: &'ll Value) { - if let Some(attr) = attributes::frame_pointer_type_attr(self) { - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); - } - } - - fn apply_target_cpu_attr(&self, llfn: &'ll Value) { - let mut attrs = SmallVec::<[_; 2]>::new(); - attrs.push(attributes::target_cpu_attr(self)); - attrs.extend(attributes::tune_cpu_attr(self)); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); - } - - fn create_used_variable(&self) { - self.create_used_variable_impl(cstr!("llvm.used"), &*self.used_statics.borrow()); - } - - fn create_compiler_used_variable(&self) { - self.create_used_variable_impl( - cstr!("llvm.compiler.used"), - &*self.compiler_used_statics.borrow(), - ); - } - - fn declare_c_main(&self, fn_type: Self::Type) -> Option { - if self.get_declared_value("main").is_none() { - Some(self.declare_cfn("main", llvm::UnnamedAddr::Global, fn_type)) - } else { - // If the symbol already exists, it is an error: for example, the user wrote - // #[no_mangle] extern "C" fn main(..) {..} - // instead of #[start] - None - } - } -} - -impl<'ll> CodegenCx<'ll, '_> { - pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { - if let Some(v) = self.intrinsics.borrow().get(key).cloned() { - return v; - } - - self.declare_intrinsic(key).unwrap_or_else(|| bug!("unknown intrinsic '{}'", key)) - } - - fn insert_intrinsic( - &self, - name: &'static str, - args: Option<&[&'ll llvm::Type]>, - ret: &'ll llvm::Type, - ) -> (&'ll llvm::Type, &'ll llvm::Value) { - let fn_ty = if let Some(args) = args { - self.type_func(args, ret) - } else { - self.type_variadic_func(&[], ret) - }; - let f = self.declare_cfn(name, llvm::UnnamedAddr::No, fn_ty); - self.intrinsics.borrow_mut().insert(name, (fn_ty, f)); - (fn_ty, f) - } - - fn declare_intrinsic(&self, key: &str) -> Option<(&'ll Type, &'ll Value)> { - macro_rules! ifn { - ($name:expr, fn() -> $ret:expr) => ( - if key == $name { - return Some(self.insert_intrinsic($name, Some(&[]), $ret)); - } - ); - ($name:expr, fn(...) -> $ret:expr) => ( - if key == $name { - return Some(self.insert_intrinsic($name, None, $ret)); - } - ); - ($name:expr, fn($($arg:expr),*) -> $ret:expr) => ( - if key == $name { - return Some(self.insert_intrinsic($name, Some(&[$($arg),*]), $ret)); - } - ); - } - macro_rules! mk_struct { - ($($field_ty:expr),*) => (self.type_struct( &[$($field_ty),*], false)) - } - - let i8p = self.type_i8p(); - let void = self.type_void(); - let i1 = self.type_i1(); - let t_i8 = self.type_i8(); - let t_i16 = self.type_i16(); - let t_i32 = self.type_i32(); - let t_i64 = self.type_i64(); - let t_i128 = self.type_i128(); - let t_isize = self.type_isize(); - let t_f32 = self.type_f32(); - let t_f64 = self.type_f64(); - let t_metadata = self.type_metadata(); - - ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32); - ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32); - ifn!("llvm.wasm.trunc.unsigned.i64.f32", fn(t_f32) -> t_i64); - ifn!("llvm.wasm.trunc.unsigned.i64.f64", fn(t_f64) -> t_i64); - ifn!("llvm.wasm.trunc.signed.i32.f32", fn(t_f32) -> t_i32); - ifn!("llvm.wasm.trunc.signed.i32.f64", fn(t_f64) -> t_i32); - ifn!("llvm.wasm.trunc.signed.i64.f32", fn(t_f32) -> t_i64); - ifn!("llvm.wasm.trunc.signed.i64.f64", fn(t_f64) -> t_i64); - - ifn!("llvm.fptosi.sat.i8.f32", fn(t_f32) -> t_i8); - ifn!("llvm.fptosi.sat.i16.f32", fn(t_f32) -> t_i16); - ifn!("llvm.fptosi.sat.i32.f32", fn(t_f32) -> t_i32); - ifn!("llvm.fptosi.sat.i64.f32", fn(t_f32) -> t_i64); - ifn!("llvm.fptosi.sat.i128.f32", fn(t_f32) -> t_i128); - ifn!("llvm.fptosi.sat.i8.f64", fn(t_f64) -> t_i8); - ifn!("llvm.fptosi.sat.i16.f64", fn(t_f64) -> t_i16); - ifn!("llvm.fptosi.sat.i32.f64", fn(t_f64) -> t_i32); - ifn!("llvm.fptosi.sat.i64.f64", fn(t_f64) -> t_i64); - ifn!("llvm.fptosi.sat.i128.f64", fn(t_f64) -> t_i128); - - ifn!("llvm.fptoui.sat.i8.f32", fn(t_f32) -> t_i8); - ifn!("llvm.fptoui.sat.i16.f32", fn(t_f32) -> t_i16); - ifn!("llvm.fptoui.sat.i32.f32", fn(t_f32) -> t_i32); - ifn!("llvm.fptoui.sat.i64.f32", fn(t_f32) -> t_i64); - ifn!("llvm.fptoui.sat.i128.f32", fn(t_f32) -> t_i128); - ifn!("llvm.fptoui.sat.i8.f64", fn(t_f64) -> t_i8); - ifn!("llvm.fptoui.sat.i16.f64", fn(t_f64) -> t_i16); - ifn!("llvm.fptoui.sat.i32.f64", fn(t_f64) -> t_i32); - ifn!("llvm.fptoui.sat.i64.f64", fn(t_f64) -> t_i64); - ifn!("llvm.fptoui.sat.i128.f64", fn(t_f64) -> t_i128); - - ifn!("llvm.trap", fn() -> void); - ifn!("llvm.debugtrap", fn() -> void); - ifn!("llvm.frameaddress", fn(t_i32) -> i8p); - - ifn!("llvm.powi.f32", fn(t_f32, t_i32) -> t_f32); - ifn!("llvm.powi.f64", fn(t_f64, t_i32) -> t_f64); - - ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.pow.f64", fn(t_f64, t_f64) -> t_f64); - - ifn!("llvm.sqrt.f32", fn(t_f32) -> t_f32); - ifn!("llvm.sqrt.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.sin.f32", fn(t_f32) -> t_f32); - ifn!("llvm.sin.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.cos.f32", fn(t_f32) -> t_f32); - ifn!("llvm.cos.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.exp.f32", fn(t_f32) -> t_f32); - ifn!("llvm.exp.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.exp2.f32", fn(t_f32) -> t_f32); - ifn!("llvm.exp2.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.log.f32", fn(t_f32) -> t_f32); - ifn!("llvm.log.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.log10.f32", fn(t_f32) -> t_f32); - ifn!("llvm.log10.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.log2.f32", fn(t_f32) -> t_f32); - ifn!("llvm.log2.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.fma.f32", fn(t_f32, t_f32, t_f32) -> t_f32); - ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64); - - ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32); - ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64); - ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64); - - ifn!("llvm.floor.f32", fn(t_f32) -> t_f32); - ifn!("llvm.floor.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.ceil.f32", fn(t_f32) -> t_f32); - ifn!("llvm.ceil.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.trunc.f32", fn(t_f32) -> t_f32); - ifn!("llvm.trunc.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32); - ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64); - ifn!("llvm.round.f32", fn(t_f32) -> t_f32); - ifn!("llvm.round.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.rint.f32", fn(t_f32) -> t_f32); - ifn!("llvm.rint.f64", fn(t_f64) -> t_f64); - ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32); - ifn!("llvm.nearbyint.f64", fn(t_f64) -> t_f64); - - ifn!("llvm.ctpop.i8", fn(t_i8) -> t_i8); - ifn!("llvm.ctpop.i16", fn(t_i16) -> t_i16); - ifn!("llvm.ctpop.i32", fn(t_i32) -> t_i32); - ifn!("llvm.ctpop.i64", fn(t_i64) -> t_i64); - ifn!("llvm.ctpop.i128", fn(t_i128) -> t_i128); - - ifn!("llvm.ctlz.i8", fn(t_i8, i1) -> t_i8); - ifn!("llvm.ctlz.i16", fn(t_i16, i1) -> t_i16); - ifn!("llvm.ctlz.i32", fn(t_i32, i1) -> t_i32); - ifn!("llvm.ctlz.i64", fn(t_i64, i1) -> t_i64); - ifn!("llvm.ctlz.i128", fn(t_i128, i1) -> t_i128); - - ifn!("llvm.cttz.i8", fn(t_i8, i1) -> t_i8); - ifn!("llvm.cttz.i16", fn(t_i16, i1) -> t_i16); - ifn!("llvm.cttz.i32", fn(t_i32, i1) -> t_i32); - ifn!("llvm.cttz.i64", fn(t_i64, i1) -> t_i64); - ifn!("llvm.cttz.i128", fn(t_i128, i1) -> t_i128); - - ifn!("llvm.bswap.i16", fn(t_i16) -> t_i16); - ifn!("llvm.bswap.i32", fn(t_i32) -> t_i32); - ifn!("llvm.bswap.i64", fn(t_i64) -> t_i64); - ifn!("llvm.bswap.i128", fn(t_i128) -> t_i128); - - ifn!("llvm.bitreverse.i8", fn(t_i8) -> t_i8); - ifn!("llvm.bitreverse.i16", fn(t_i16) -> t_i16); - ifn!("llvm.bitreverse.i32", fn(t_i32) -> t_i32); - ifn!("llvm.bitreverse.i64", fn(t_i64) -> t_i64); - ifn!("llvm.bitreverse.i128", fn(t_i128) -> t_i128); - - ifn!("llvm.fshl.i8", fn(t_i8, t_i8, t_i8) -> t_i8); - ifn!("llvm.fshl.i16", fn(t_i16, t_i16, t_i16) -> t_i16); - ifn!("llvm.fshl.i32", fn(t_i32, t_i32, t_i32) -> t_i32); - ifn!("llvm.fshl.i64", fn(t_i64, t_i64, t_i64) -> t_i64); - ifn!("llvm.fshl.i128", fn(t_i128, t_i128, t_i128) -> t_i128); - - ifn!("llvm.fshr.i8", fn(t_i8, t_i8, t_i8) -> t_i8); - ifn!("llvm.fshr.i16", fn(t_i16, t_i16, t_i16) -> t_i16); - ifn!("llvm.fshr.i32", fn(t_i32, t_i32, t_i32) -> t_i32); - ifn!("llvm.fshr.i64", fn(t_i64, t_i64, t_i64) -> t_i64); - ifn!("llvm.fshr.i128", fn(t_i128, t_i128, t_i128) -> t_i128); - - ifn!("llvm.sadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.sadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.sadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.sadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.sadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.uadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.uadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.uadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.uadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.uadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.ssub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.ssub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.ssub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.ssub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.ssub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.usub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.usub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.usub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.usub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.usub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.smul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.smul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.smul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.smul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.smul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.umul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); - ifn!("llvm.umul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); - ifn!("llvm.umul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); - ifn!("llvm.umul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); - ifn!("llvm.umul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); - - ifn!("llvm.sadd.sat.i8", fn(t_i8, t_i8) -> t_i8); - ifn!("llvm.sadd.sat.i16", fn(t_i16, t_i16) -> t_i16); - ifn!("llvm.sadd.sat.i32", fn(t_i32, t_i32) -> t_i32); - ifn!("llvm.sadd.sat.i64", fn(t_i64, t_i64) -> t_i64); - ifn!("llvm.sadd.sat.i128", fn(t_i128, t_i128) -> t_i128); - - ifn!("llvm.uadd.sat.i8", fn(t_i8, t_i8) -> t_i8); - ifn!("llvm.uadd.sat.i16", fn(t_i16, t_i16) -> t_i16); - ifn!("llvm.uadd.sat.i32", fn(t_i32, t_i32) -> t_i32); - ifn!("llvm.uadd.sat.i64", fn(t_i64, t_i64) -> t_i64); - ifn!("llvm.uadd.sat.i128", fn(t_i128, t_i128) -> t_i128); - - ifn!("llvm.ssub.sat.i8", fn(t_i8, t_i8) -> t_i8); - ifn!("llvm.ssub.sat.i16", fn(t_i16, t_i16) -> t_i16); - ifn!("llvm.ssub.sat.i32", fn(t_i32, t_i32) -> t_i32); - ifn!("llvm.ssub.sat.i64", fn(t_i64, t_i64) -> t_i64); - ifn!("llvm.ssub.sat.i128", fn(t_i128, t_i128) -> t_i128); - - ifn!("llvm.usub.sat.i8", fn(t_i8, t_i8) -> t_i8); - ifn!("llvm.usub.sat.i16", fn(t_i16, t_i16) -> t_i16); - ifn!("llvm.usub.sat.i32", fn(t_i32, t_i32) -> t_i32); - ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64); - ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128); - - ifn!("llvm.lifetime.start.p0i8", fn(t_i64, i8p) -> void); - ifn!("llvm.lifetime.end.p0i8", fn(t_i64, i8p) -> void); - - ifn!("llvm.expect.i1", fn(i1, i1) -> i1); - ifn!("llvm.eh.typeid.for", fn(i8p) -> t_i32); - ifn!("llvm.localescape", fn(...) -> void); - ifn!("llvm.localrecover", fn(i8p, i8p, t_i32) -> i8p); - ifn!("llvm.x86.seh.recoverfp", fn(i8p, i8p) -> i8p); - - ifn!("llvm.assume", fn(i1) -> void); - ifn!("llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void); - - // This isn't an "LLVM intrinsic", but LLVM's optimization passes - // recognize it like one and we assume it exists in `core::slice::cmp` - match self.sess().target.arch.as_ref() { - "avr" | "msp430" => ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i16), - _ => ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i32), - } - - // variadic intrinsics - ifn!("llvm.va_start", fn(i8p) -> void); - ifn!("llvm.va_end", fn(i8p) -> void); - ifn!("llvm.va_copy", fn(i8p, i8p) -> void); - - if self.sess().instrument_coverage() { - ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void); - } - - ifn!("llvm.type.test", fn(i8p, t_metadata) -> i1); - ifn!("llvm.type.checked.load", fn(i8p, t_i32, t_metadata) -> mk_struct! {i8p, i1}); - - if self.sess().opts.debuginfo != DebugInfo::None { - ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); - ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); - } - None - } - - pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value { - if let Some(eh_catch_typeinfo) = self.eh_catch_typeinfo.get() { - return eh_catch_typeinfo; - } - let tcx = self.tcx; - assert!(self.sess().target.os == "emscripten"); - let eh_catch_typeinfo = match tcx.lang_items().eh_catch_typeinfo() { - Some(def_id) => self.get_static(def_id), - _ => { - let ty = self - .type_struct(&[self.type_ptr_to(self.type_isize()), self.type_i8p()], false); - self.declare_global("rust_eh_catch_typeinfo", ty) - } - }; - let eh_catch_typeinfo = self.const_bitcast(eh_catch_typeinfo, self.type_i8p()); - self.eh_catch_typeinfo.set(Some(eh_catch_typeinfo)); - eh_catch_typeinfo - } -} - -impl CodegenCx<'_, '_> { - /// Generates a new symbol name with the given prefix. This symbol name must - /// only be used for definitions with `internal` or `private` linkage. - pub fn generate_local_symbol_name(&self, prefix: &str) -> String { - let idx = self.local_gen_sym_counter.get(); - self.local_gen_sym_counter.set(idx + 1); - // Include a '.' character, so there can be no accidental conflicts with - // user defined names - let mut name = String::with_capacity(prefix.len() + 6); - name.push_str(prefix); - name.push('.'); - base_n::push_str(idx as u128, base_n::ALPHANUMERIC_ONLY, &mut name); - name - } -} - -impl HasDataLayout for CodegenCx<'_, '_> { - #[inline] - fn data_layout(&self) -> &TargetDataLayout { - &self.tcx.data_layout - } -} - -impl HasTargetSpec for CodegenCx<'_, '_> { - #[inline] - fn target_spec(&self) -> &Target { - &self.tcx.sess.target - } -} - -impl<'tcx> ty::layout::HasTyCtxt<'tcx> for CodegenCx<'_, 'tcx> { - #[inline] - fn tcx(&self) -> TyCtxt<'tcx> { - self.tcx - } -} - -impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> { - fn param_env(&self) -> ty::ParamEnv<'tcx> { - ty::ParamEnv::reveal_all() - } -} - -impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { - type LayoutOfResult = TyAndLayout<'tcx>; - - #[inline] - fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) = err { - self.sess().span_fatal(span, &err.to_string()) - } else { - span_bug!(span, "failed to get layout for `{}`: {}", ty, err) - } - } -} - -impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { - type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; - - #[inline] - fn handle_fn_abi_err( - &self, - err: FnAbiError<'tcx>, - span: Span, - fn_abi_request: FnAbiRequest<'tcx>, - ) -> ! { - if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { - self.sess().span_fatal(span, &err.to_string()) - } else { - match fn_abi_request { - FnAbiRequest::OfFnPtr { sig, extra_args } => { - span_bug!( - span, - "`fn_abi_of_fn_ptr({}, {:?})` failed: {}", - sig, - extra_args, - err - ); - } - FnAbiRequest::OfInstance { instance, extra_args } => { - span_bug!( - span, - "`fn_abi_of_instance({}, {:?})` failed: {}", - instance, - extra_args, - err - ); - } - } - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs deleted file mode 100644 index 58f391692c4..00000000000 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ /dev/null @@ -1,334 +0,0 @@ -use crate::common::CodegenCx; -use crate::coverageinfo; -use crate::llvm; - -use llvm::coverageinfo::CounterMappingRegion; -use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; -use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods}; -use rustc_data_structures::fx::FxIndexSet; -use rustc_hir::def::DefKind; -use rustc_hir::def_id::DefIdSet; -use rustc_llvm::RustString; -use rustc_middle::bug; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::coverage::CodeRegion; -use rustc_middle::ty::TyCtxt; - -use std::ffi::CString; - -use tracing::debug; - -/// Generates and exports the Coverage Map. -/// -/// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions -/// 5 (LLVM 12, only) and 6 (zero-based encoded as 4 and 5, respectively), as defined at -/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). -/// These versions are supported by the LLVM coverage tools (`llvm-profdata` and `llvm-cov`) -/// bundled with Rust's fork of LLVM. -/// -/// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with -/// the same version. Clang's implementation of Coverage Map generation was referenced when -/// implementing this Rust version, and though the format documentation is very explicit and -/// detailed, some undocumented details in Clang's implementation (that may or may not be important) -/// were also replicated for Rust's Coverage Map. -pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { - let tcx = cx.tcx; - - // Ensure the installed version of LLVM supports at least Coverage Map - // Version 5 (encoded as a zero-based value: 4), which was introduced with - // LLVM 12. - let version = coverageinfo::mapping_version(); - if version < 4 { - tcx.sess.fatal("rustc option `-C instrument-coverage` requires LLVM 12 or higher."); - } - - debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name()); - - // In order to show that unused functions have coverage counts of zero (0), LLVM requires the - // functions exist. Generate synthetic functions with a (required) single counter, and add the - // MIR `Coverage` code regions to the `function_coverage_map`, before calling - // `ctx.take_function_coverage_map()`. - if cx.codegen_unit.is_code_coverage_dead_code_cgu() { - add_unused_functions(cx); - } - - let function_coverage_map = match cx.coverage_context() { - Some(ctx) => ctx.take_function_coverage_map(), - None => return, - }; - - if function_coverage_map.is_empty() { - // This module has no functions with coverage instrumentation - return; - } - - let mut mapgen = CoverageMapGenerator::new(tcx, version); - - // Encode coverage mappings and generate function records - let mut function_data = Vec::new(); - for (instance, function_coverage) in function_coverage_map { - debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); - let mangled_function_name = tcx.symbol_name(instance).to_string(); - let source_hash = function_coverage.source_hash(); - let is_used = function_coverage.is_used(); - let (expressions, counter_regions) = - function_coverage.get_expressions_and_counter_regions(); - - let coverage_mapping_buffer = llvm::build_byte_buffer(|coverage_mapping_buffer| { - mapgen.write_coverage_mapping(expressions, counter_regions, coverage_mapping_buffer); - }); - - if coverage_mapping_buffer.is_empty() { - if function_coverage.is_used() { - bug!( - "A used function should have had coverage mapping data but did not: {}", - mangled_function_name - ); - } else { - debug!("unused function had no coverage mapping data: {}", mangled_function_name); - continue; - } - } - - function_data.push((mangled_function_name, source_hash, is_used, coverage_mapping_buffer)); - } - - // Encode all filenames referenced by counters/expressions in this module - let filenames_buffer = llvm::build_byte_buffer(|filenames_buffer| { - coverageinfo::write_filenames_section_to_buffer(&mapgen.filenames, filenames_buffer); - }); - - let filenames_size = filenames_buffer.len(); - let filenames_val = cx.const_bytes(&filenames_buffer); - let filenames_ref = coverageinfo::hash_bytes(filenames_buffer); - - // Generate the LLVM IR representation of the coverage map and store it in a well-known global - let cov_data_val = mapgen.generate_coverage_map(cx, version, filenames_size, filenames_val); - - for (mangled_function_name, source_hash, is_used, coverage_mapping_buffer) in function_data { - save_function_record( - cx, - mangled_function_name, - source_hash, - filenames_ref, - coverage_mapping_buffer, - is_used, - ); - } - - // Save the coverage data value to LLVM IR - coverageinfo::save_cov_data_to_mod(cx, cov_data_val); -} - -struct CoverageMapGenerator { - filenames: FxIndexSet, -} - -impl CoverageMapGenerator { - fn new(tcx: TyCtxt<'_>, version: u32) -> Self { - let mut filenames = FxIndexSet::default(); - if version >= 5 { - // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) - // requires setting the first filename to the compilation directory. - // Since rustc generates coverage maps with relative paths, the - // compilation directory can be combined with the the relative paths - // to get absolute paths, if needed. - let working_dir = tcx - .sess - .opts - .working_dir - .remapped_path_if_available() - .to_string_lossy() - .to_string(); - let c_filename = - CString::new(working_dir).expect("null error converting filename to C string"); - filenames.insert(c_filename); - } - Self { filenames } - } - - /// Using the `expressions` and `counter_regions` collected for the current function, generate - /// the `mapping_regions` and `virtual_file_mapping`, and capture any new filenames. Then use - /// LLVM APIs to encode the `virtual_file_mapping`, `expressions`, and `mapping_regions` into - /// the given `coverage_mapping` byte buffer, compliant with the LLVM Coverage Mapping format. - fn write_coverage_mapping<'a>( - &mut self, - expressions: Vec, - counter_regions: impl Iterator, - coverage_mapping_buffer: &RustString, - ) { - let mut counter_regions = counter_regions.collect::>(); - if counter_regions.is_empty() { - return; - } - - let mut virtual_file_mapping = Vec::new(); - let mut mapping_regions = Vec::new(); - let mut current_file_name = None; - let mut current_file_id = 0; - - // Convert the list of (Counter, CodeRegion) pairs to an array of `CounterMappingRegion`, sorted - // by filename and position. Capture any new files to compute the `CounterMappingRegion`s - // `file_id` (indexing files referenced by the current function), and construct the - // function-specific `virtual_file_mapping` from `file_id` to its index in the module's - // `filenames` array. - counter_regions.sort_unstable_by_key(|(_counter, region)| *region); - for (counter, region) in counter_regions { - let CodeRegion { file_name, start_line, start_col, end_line, end_col } = *region; - let same_file = current_file_name.as_ref().map_or(false, |p| *p == file_name); - if !same_file { - if current_file_name.is_some() { - current_file_id += 1; - } - current_file_name = Some(file_name); - let c_filename = CString::new(file_name.to_string()) - .expect("null error converting filename to C string"); - debug!(" file_id: {} = '{:?}'", current_file_id, c_filename); - let (filenames_index, _) = self.filenames.insert_full(c_filename); - virtual_file_mapping.push(filenames_index as u32); - } - debug!("Adding counter {:?} to map for {:?}", counter, region); - mapping_regions.push(CounterMappingRegion::code_region( - counter, - current_file_id, - start_line, - start_col, - end_line, - end_col, - )); - } - - // Encode and append the current function's coverage mapping data - coverageinfo::write_mapping_to_buffer( - virtual_file_mapping, - expressions, - mapping_regions, - coverage_mapping_buffer, - ); - } - - /// Construct coverage map header and the array of function records, and combine them into the - /// coverage map. Save the coverage map data into the LLVM IR as a static global using a - /// specific, well-known section and name. - fn generate_coverage_map<'ll>( - self, - cx: &CodegenCx<'ll, '_>, - version: u32, - filenames_size: usize, - filenames_val: &'ll llvm::Value, - ) -> &'ll llvm::Value { - debug!("cov map: filenames_size = {}, 0-based version = {}", filenames_size, version); - - // Create the coverage data header (Note, fields 0 and 2 are now always zero, - // as of `llvm::coverage::CovMapVersion::Version4`.) - let zero_was_n_records_val = cx.const_u32(0); - let filenames_size_val = cx.const_u32(filenames_size as u32); - let zero_was_coverage_size_val = cx.const_u32(0); - let version_val = cx.const_u32(version); - let cov_data_header_val = cx.const_struct( - &[zero_was_n_records_val, filenames_size_val, zero_was_coverage_size_val, version_val], - /*packed=*/ false, - ); - - // Create the complete LLVM coverage data value to add to the LLVM IR - cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false) - } -} - -/// Construct a function record and combine it with the function's coverage mapping data. -/// Save the function record into the LLVM IR as a static global using a -/// specific, well-known section and name. -fn save_function_record( - cx: &CodegenCx<'_, '_>, - mangled_function_name: String, - source_hash: u64, - filenames_ref: u64, - coverage_mapping_buffer: Vec, - is_used: bool, -) { - // Concatenate the encoded coverage mappings - let coverage_mapping_size = coverage_mapping_buffer.len(); - let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer); - - let func_name_hash = coverageinfo::hash_str(&mangled_function_name); - let func_name_hash_val = cx.const_u64(func_name_hash); - let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); - let source_hash_val = cx.const_u64(source_hash); - let filenames_ref_val = cx.const_u64(filenames_ref); - let func_record_val = cx.const_struct( - &[ - func_name_hash_val, - coverage_mapping_size_val, - source_hash_val, - filenames_ref_val, - coverage_mapping_val, - ], - /*packed=*/ true, - ); - - coverageinfo::save_func_record_to_mod(cx, func_name_hash, func_record_val, is_used); -} - -/// When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s and counters for -/// the functions that went through codegen; such as public functions and "used" functions -/// (functions referenced by other "used" or public items). Any other functions considered unused, -/// or "Unreachable", were still parsed and processed through the MIR stage, but were not -/// codegenned. (Note that `-Clink-dead-code` can force some unused code to be codegenned, but -/// that flag is known to cause other errors, when combined with `-C instrument-coverage`; and -/// `-Clink-dead-code` will not generate code for unused generic functions.) -/// -/// We can find the unused functions (including generic functions) by the set difference of all MIR -/// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query -/// `codegened_and_inlined_items`). -/// -/// These unused functions are then codegen'd in one of the CGUs which is marked as the -/// "code coverage dead code cgu" during the partitioning process. This prevents us from generating -/// code regions for the same function more than once which can lead to linker errors regarding -/// duplicate symbols. -fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { - assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); - - let tcx = cx.tcx; - - let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); - - let eligible_def_ids: DefIdSet = tcx - .mir_keys(()) - .iter() - .filter_map(|local_def_id| { - let def_id = local_def_id.to_def_id(); - let kind = tcx.def_kind(def_id); - // `mir_keys` will give us `DefId`s for all kinds of things, not - // just "functions", like consts, statics, etc. Filter those out. - // If `ignore_unused_generics` was specified, filter out any - // generic functions from consideration as well. - if !matches!( - kind, - DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Generator - ) { - return None; - } else if ignore_unused_generics - && tcx.generics_of(def_id).requires_monomorphization(tcx) - { - return None; - } - Some(local_def_id.to_def_id()) - }) - .collect(); - - let codegenned_def_ids = tcx.codegened_and_inlined_items(()); - - for &non_codegenned_def_id in eligible_def_ids.difference(codegenned_def_ids) { - let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id); - - // If a function is marked `#[no_coverage]`, then skip generating a - // dead code stub for it. - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) { - debug!("skipping unused fn marked #[no_coverage]: {:?}", non_codegenned_def_id); - continue; - } - - debug!("generating unused fn: {:?}", non_codegenned_def_id); - cx.define_unused_fn(non_codegenned_def_id); - } -} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs deleted file mode 100644 index 98ba38356a4..00000000000 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ /dev/null @@ -1,385 +0,0 @@ -use crate::llvm; - -use crate::abi::Abi; -use crate::builder::Builder; -use crate::common::CodegenCx; - -use libc::c_uint; -use llvm::coverageinfo::CounterMappingRegion; -use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage}; -use rustc_codegen_ssa::traits::{ - BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, - MiscMethods, StaticMethods, -}; -use rustc_data_structures::fx::FxHashMap; -use rustc_hir as hir; -use rustc_hir::def_id::DefId; -use rustc_llvm::RustString; -use rustc_middle::bug; -use rustc_middle::mir::coverage::{ - CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op, -}; -use rustc_middle::ty; -use rustc_middle::ty::layout::FnAbiOf; -use rustc_middle::ty::subst::InternalSubsts; -use rustc_middle::ty::Instance; - -use std::cell::RefCell; -use std::ffi::CString; - -use std::iter; -use tracing::debug; - -pub mod mapgen; - -const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START; - -const VAR_ALIGN_BYTES: usize = 8; - -/// A context object for maintaining all state needed by the coverageinfo module. -pub struct CrateCoverageContext<'ll, 'tcx> { - // Coverage data for each instrumented function identified by DefId. - pub(crate) function_coverage_map: RefCell, FunctionCoverage<'tcx>>>, - pub(crate) pgo_func_name_var_map: RefCell, &'ll llvm::Value>>, -} - -impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { - pub fn new() -> Self { - Self { - function_coverage_map: Default::default(), - pgo_func_name_var_map: Default::default(), - } - } - - pub fn take_function_coverage_map(&self) -> FxHashMap, FunctionCoverage<'tcx>> { - self.function_coverage_map.replace(FxHashMap::default()) - } -} - -impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn coverageinfo_finalize(&self) { - mapgen::finalize(self) - } - - fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { - if let Some(coverage_context) = self.coverage_context() { - debug!("getting pgo_func_name_var for instance={:?}", instance); - let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); - pgo_func_name_var_map - .entry(instance) - .or_insert_with(|| create_pgo_func_name_var(self, instance)) - } else { - bug!("Could not get the `coverage_context`"); - } - } - - /// Functions with MIR-based coverage are normally codegenned _only_ if - /// called. LLVM coverage tools typically expect every function to be - /// defined (even if unused), with at least one call to LLVM intrinsic - /// `instrprof.increment`. - /// - /// Codegen a small function that will never be called, with one counter - /// that will never be incremented. - /// - /// For used/called functions, the coverageinfo was already added to the - /// `function_coverage_map` (keyed by function `Instance`) during codegen. - /// But in this case, since the unused function was _not_ previously - /// codegenned, collect the coverage `CodeRegion`s from the MIR and add - /// them. The first `CodeRegion` is used to add a single counter, with the - /// same counter ID used in the injected `instrprof.increment` intrinsic - /// call. Since the function is never called, all other `CodeRegion`s can be - /// added as `unreachable_region`s. - fn define_unused_fn(&self, def_id: DefId) { - let instance = declare_unused_fn(self, def_id); - codegen_unused_fn_and_counter(self, instance); - add_unused_function_coverage(self, instance, def_id); - } -} - -impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { - fn set_function_source_hash( - &mut self, - instance: Instance<'tcx>, - function_source_hash: u64, - ) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "ensuring function source hash is set for instance={:?}; function_source_hash={}", - instance, function_source_hash, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .set_function_source_hash(function_source_hash); - true - } else { - false - } - } - - fn add_coverage_counter( - &mut self, - instance: Instance<'tcx>, - id: CounterValueReference, - region: CodeRegion, - ) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", - instance, id, region, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_counter(id, region); - true - } else { - false - } - } - - fn add_coverage_counter_expression( - &mut self, - instance: Instance<'tcx>, - id: InjectedExpressionId, - lhs: ExpressionOperandId, - op: Op, - rhs: ExpressionOperandId, - region: Option, - ) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; \ - region: {:?}", - instance, id, lhs, op, rhs, region, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_counter_expression(id, lhs, op, rhs, region); - true - } else { - false - } - } - - fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "adding unreachable code to coverage_map: instance={:?}, at {:?}", - instance, region, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_unreachable_region(region); - true - } else { - false - } - } -} - -fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<'tcx> { - let tcx = cx.tcx; - - let instance = Instance::new( - def_id, - InternalSubsts::for_item(tcx, def_id, |param, _| { - if let ty::GenericParamDefKind::Lifetime = param.kind { - tcx.lifetimes.re_erased.into() - } else { - tcx.mk_param_from_def(param) - } - }), - ); - - let llfn = cx.declare_fn( - tcx.symbol_name(instance).name, - cx.fn_abi_of_fn_ptr( - ty::Binder::dummy(tcx.mk_fn_sig( - iter::once(tcx.mk_unit()), - tcx.mk_unit(), - false, - hir::Unsafety::Unsafe, - Abi::Rust, - )), - ty::List::empty(), - ), - ); - - llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); - llvm::set_visibility(llfn, llvm::Visibility::Default); - - assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none()); - - instance -} - -fn codegen_unused_fn_and_counter<'tcx>(cx: &CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { - let llfn = cx.get_fn(instance); - let llbb = Builder::append_block(cx, llfn, "unused_function"); - let mut bx = Builder::build(cx, llbb); - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(0); - let num_counters = bx.const_u32(1); - let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID)); - debug!( - "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, - index={:?}) for unused function: {:?}", - fn_name, hash, num_counters, index, instance - ); - bx.instrprof_increment(fn_name, hash, num_counters, index); - bx.ret_void(); -} - -fn add_unused_function_coverage<'tcx>( - cx: &CodegenCx<'_, 'tcx>, - instance: Instance<'tcx>, - def_id: DefId, -) { - let tcx = cx.tcx; - - let mut function_coverage = FunctionCoverage::unused(tcx, instance); - for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { - if index == 0 { - // Insert at least one real counter so the LLVM CoverageMappingReader will find expected - // definitions. - function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); - } else { - function_coverage.add_unreachable_region(code_region.clone()); - } - } - - if let Some(coverage_context) = cx.coverage_context() { - coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); - } else { - bug!("Could not get the `coverage_context`"); - } -} - -/// Calls llvm::createPGOFuncNameVar() with the given function instance's -/// mangled function name. The LLVM API returns an llvm::GlobalVariable -/// containing the function name, with the specific variable name and linkage -/// required by LLVM InstrProf source-based coverage instrumentation. Use -/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per -/// `Instance`. -fn create_pgo_func_name_var<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - instance: Instance<'tcx>, -) -> &'ll llvm::Value { - let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name) - .expect("error converting function name to C string"); - let llfn = cx.get_fn(instance); - unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } -} - -pub(crate) fn write_filenames_section_to_buffer<'a>( - filenames: impl IntoIterator, - buffer: &RustString, -) { - let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::>(); - unsafe { - llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( - c_str_vec.as_ptr(), - c_str_vec.len(), - buffer, - ); - } -} - -pub(crate) fn write_mapping_to_buffer( - virtual_file_mapping: Vec, - expressions: Vec, - mapping_regions: Vec, - buffer: &RustString, -) { - unsafe { - llvm::LLVMRustCoverageWriteMappingToBuffer( - virtual_file_mapping.as_ptr(), - virtual_file_mapping.len() as c_uint, - expressions.as_ptr(), - expressions.len() as c_uint, - mapping_regions.as_ptr(), - mapping_regions.len() as c_uint, - buffer, - ); - } -} - -pub(crate) fn hash_str(strval: &str) -> u64 { - let strval = CString::new(strval).expect("null error converting hashable str to C string"); - unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } -} - -pub(crate) fn hash_bytes(bytes: Vec) -> u64 { - unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } -} - -pub(crate) fn mapping_version() -> u32 { - unsafe { llvm::LLVMRustCoverageMappingVersion() } -} - -pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - cov_data_val: &'ll llvm::Value, -) { - let covmap_var_name = llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteMappingVarNameToString(s); - }) - .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); - debug!("covmap var name: {:?}", covmap_var_name); - - let covmap_section_name = llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); - }) - .expect("Rust Coverage section name failed UTF-8 conversion"); - debug!("covmap section name: {:?}", covmap_section_name); - - let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); - llvm::set_initializer(llglobal, cov_data_val); - llvm::set_global_constant(llglobal, true); - llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); - llvm::set_section(llglobal, &covmap_section_name); - llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); - cx.add_used_global(llglobal); -} - -pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - func_name_hash: u64, - func_record_val: &'ll llvm::Value, - is_used: bool, -) { - // Assign a name to the function record. This is used to merge duplicates. - // - // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that - // are included-but-not-used. If (or when) Rust generates functions that are - // included-but-not-used, note that a dummy description for a function included-but-not-used - // in a Crate can be replaced by full description provided by a different Crate. The two kinds - // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by - // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. - let func_record_var_name = - format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); - debug!("function record var name: {:?}", func_record_var_name); - - let func_record_section_name = llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); - }) - .expect("Rust Coverage function record section name failed UTF-8 conversion"); - debug!("function record section name: {:?}", func_record_section_name); - - let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); - llvm::set_initializer(llglobal, func_record_val); - llvm::set_global_constant(llglobal, true); - llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); - llvm::set_visibility(llglobal, llvm::Visibility::Hidden); - llvm::set_section(llglobal, &func_record_section_name); - llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); - llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); - cx.add_used_global(llglobal); -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs deleted file mode 100644 index 99e4ded62f1..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ /dev/null @@ -1,126 +0,0 @@ -use super::metadata::file_metadata; -use super::utils::DIB; -use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext}; -use rustc_codegen_ssa::traits::*; - -use crate::common::CodegenCx; -use crate::llvm; -use crate::llvm::debuginfo::{DILocation, DIScope}; -use rustc_middle::mir::{Body, SourceScope}; -use rustc_middle::ty::layout::FnAbiOf; -use rustc_middle::ty::{self, Instance}; -use rustc_session::config::DebugInfo; - -use rustc_index::bit_set::BitSet; -use rustc_index::vec::Idx; - -/// Produces DIScope DIEs for each MIR Scope which has variables defined in it. -// FIXME(eddyb) almost all of this should be in `rustc_codegen_ssa::mir::debuginfo`. -pub fn compute_mir_scopes<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - instance: Instance<'tcx>, - mir: &Body<'tcx>, - debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, -) { - // Find all scopes with variables defined in them. - let variables = if cx.sess().opts.debuginfo == DebugInfo::Full { - let mut vars = BitSet::new_empty(mir.source_scopes.len()); - // FIXME(eddyb) take into account that arguments always have debuginfo, - // irrespective of their name (assuming full debuginfo is enabled). - // NOTE(eddyb) actually, on second thought, those are always in the - // function scope, which always exists. - for var_debug_info in &mir.var_debug_info { - vars.insert(var_debug_info.source_info.scope); - } - Some(vars) - } else { - // Nothing to emit, of course. - None - }; - let mut instantiated = BitSet::new_empty(mir.source_scopes.len()); - // Instantiate all scopes. - for idx in 0..mir.source_scopes.len() { - let scope = SourceScope::new(idx); - make_mir_scope(cx, instance, mir, &variables, debug_context, &mut instantiated, scope); - } - assert!(instantiated.count() == mir.source_scopes.len()); -} - -fn make_mir_scope<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - instance: Instance<'tcx>, - mir: &Body<'tcx>, - variables: &Option>, - debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, - instantiated: &mut BitSet, - scope: SourceScope, -) { - if instantiated.contains(scope) { - return; - } - - let scope_data = &mir.source_scopes[scope]; - let parent_scope = if let Some(parent) = scope_data.parent_scope { - make_mir_scope(cx, instance, mir, variables, debug_context, instantiated, parent); - debug_context.scopes[parent] - } else { - // The root is the function itself. - let loc = cx.lookup_debug_loc(mir.span.lo()); - debug_context.scopes[scope] = DebugScope { - file_start_pos: loc.file.start_pos, - file_end_pos: loc.file.end_pos, - ..debug_context.scopes[scope] - }; - instantiated.insert(scope); - return; - }; - - if let Some(vars) = variables && !vars.contains(scope) && scope_data.inlined.is_none() { - // Do not create a DIScope if there are no variables defined in this - // MIR `SourceScope`, and it's not `inlined`, to avoid debuginfo bloat. - debug_context.scopes[scope] = parent_scope; - instantiated.insert(scope); - return; - } - - let loc = cx.lookup_debug_loc(scope_data.span.lo()); - let file_metadata = file_metadata(cx, &loc.file); - - let dbg_scope = match scope_data.inlined { - Some((callee, _)) => { - // FIXME(eddyb) this would be `self.monomorphize(&callee)` - // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. - let callee = cx.tcx.subst_and_normalize_erasing_regions( - instance.substs, - ty::ParamEnv::reveal_all(), - callee, - ); - let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); - cx.dbg_scope_fn(callee, callee_fn_abi, None) - } - None => unsafe { - llvm::LLVMRustDIBuilderCreateLexicalBlock( - DIB(cx), - parent_scope.dbg_scope, - file_metadata, - loc.line, - loc.col, - ) - }, - }; - - let inlined_at = scope_data.inlined.map(|(_, callsite_span)| { - // FIXME(eddyb) this doesn't account for the macro-related - // `Span` fixups that `rustc_codegen_ssa::mir::debuginfo` does. - let callsite_scope = parent_scope.adjust_dbg_scope_for_span(cx, callsite_span); - cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span) - }); - - debug_context.scopes[scope] = DebugScope { - dbg_scope, - inlined_at: inlined_at.or(parent_scope.inlined_at), - file_start_pos: loc.file.start_pos, - file_end_pos: loc.file.end_pos, - }; - instantiated.insert(scope); -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md deleted file mode 100644 index aaec4e68c17..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md +++ /dev/null @@ -1,131 +0,0 @@ -# Debug Info Module - -This module serves the purpose of generating debug symbols. We use LLVM's -[source level debugging](https://llvm.org/docs/SourceLevelDebugging.html) -features for generating the debug information. The general principle is -this: - -Given the right metadata in the LLVM IR, the LLVM code generator is able to -create DWARF debug symbols for the given code. The -[metadata](https://llvm.org/docs/LangRef.html#metadata-type) is structured -much like DWARF *debugging information entries* (DIE), representing type -information such as datatype layout, function signatures, block layout, -variable location and scope information, etc. It is the purpose of this -module to generate correct metadata and insert it into the LLVM IR. - -As the exact format of metadata trees may change between different LLVM -versions, we now use LLVM -[DIBuilder](https://llvm.org/docs/doxygen/html/classllvm_1_1DIBuilder.html) -to create metadata where possible. This will hopefully ease the adaption of -this module to future LLVM versions. - -The public API of the module is a set of functions that will insert the -correct metadata into the LLVM IR when called with the right parameters. -The module is thus driven from an outside client with functions like -`debuginfo::create_local_var_metadata(bx: block, local: &ast::local)`. - -Internally the module will try to reuse already created metadata by -utilizing a cache. The way to get a shared metadata node when needed is -thus to just call the corresponding function in this module: -```ignore (illustrative) -let file_metadata = file_metadata(cx, file); -``` -The function will take care of probing the cache for an existing node for -that exact file path. - -All private state used by the module is stored within either the -CodegenUnitDebugContext struct (owned by the CodegenCx) or the -FunctionDebugContext (owned by the FunctionCx). - -This file consists of three conceptual sections: -1. The public interface of the module -2. Module-internal metadata creation functions -3. Minor utility functions - - -## Recursive Types - -Some kinds of types, such as structs and enums can be recursive. That means -that the type definition of some type X refers to some other type which in -turn (transitively) refers to X. This introduces cycles into the type -referral graph. A naive algorithm doing an on-demand, depth-first traversal -of this graph when describing types, can get trapped in an endless loop -when it reaches such a cycle. - -For example, the following simple type for a singly-linked list... - -``` -struct List { - value: i32, - tail: Option>, -} -``` - -will generate the following callstack with a naive DFS algorithm: - -```ignore (illustrative) -describe(t = List) - describe(t = i32) - describe(t = Option>) - describe(t = Box) - describe(t = List) // at the beginning again... - ... -``` - -To break cycles like these, we use "stubs". That is, when -the algorithm encounters a possibly recursive type (any struct or enum), it -immediately creates a type description node and inserts it into the cache -*before* describing the members of the type. This type description is just -a stub (as type members are not described and added to it yet) but it -allows the algorithm to already refer to the type. After the stub is -inserted into the cache, the algorithm continues as before. If it now -encounters a recursive reference, it will hit the cache and does not try to -describe the type anew. This behavior is encapsulated in the -`type_map::build_type_with_children()` function. - - -## Source Locations and Line Information - -In addition to data type descriptions the debugging information must also -allow to map machine code locations back to source code locations in order -to be useful. This functionality is also handled in this module. The -following functions allow to control source mappings: - -+ `set_source_location()` -+ `clear_source_location()` -+ `start_emitting_source_locations()` - -`set_source_location()` allows to set the current source location. All IR -instructions created after a call to this function will be linked to the -given source location, until another location is specified with -`set_source_location()` or the source location is cleared with -`clear_source_location()`. In the later case, subsequent IR instruction -will not be linked to any source location. As you can see, this is a -stateful API (mimicking the one in LLVM), so be careful with source -locations set by previous calls. It's probably best to not rely on any -specific state being present at a given point in code. - -One topic that deserves some extra attention is *function prologues*. At -the beginning of a function's machine code there are typically a few -instructions for loading argument values into allocas and checking if -there's enough stack space for the function to execute. This *prologue* is -not visible in the source code and LLVM puts a special PROLOGUE END marker -into the line table at the first non-prologue instruction of the function. -In order to find out where the prologue ends, LLVM looks for the first -instruction in the function body that is linked to a source location. So, -when generating prologue instructions we have to make sure that we don't -emit source location information until the 'real' function body begins. For -this reason, source location emission is disabled by default for any new -function being codegened and is only activated after a call to the third -function from the list above, `start_emitting_source_locations()`. This -function should be called right before regularly starting to codegen the -top-level block of the given function. - -There is one exception to the above rule: `llvm.dbg.declare` instruction -must be linked to the source location of the variable being declared. For -function parameters these `llvm.dbg.declare` instructions typically occur -in the middle of the prologue, however, they are ignored by LLVM's prologue -detection. The `create_argument_metadata()` and related functions take care -of linking the `llvm.dbg.declare` instructions to the correct source -locations even while source location emission is still disabled, so there -is no need to do anything special with source location handling here. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs deleted file mode 100644 index 80fd9726fc7..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ /dev/null @@ -1,120 +0,0 @@ -// .debug_gdb_scripts binary section. - -use crate::llvm; - -use crate::builder::Builder; -use crate::common::CodegenCx; -use crate::value::Value; -use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; -use rustc_codegen_ssa::traits::*; -use rustc_hir::def_id::LOCAL_CRATE; -use rustc_middle::bug; -use rustc_session::config::{CrateType, DebugInfo}; - -use rustc_span::symbol::sym; -use rustc_span::DebuggerVisualizerType; - -/// Inserts a side-effect free instruction sequence that makes sure that the -/// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. -pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, '_, '_>) { - if needs_gdb_debug_scripts_section(bx) { - let gdb_debug_scripts_section = - bx.const_bitcast(get_or_insert_gdb_debug_scripts_section_global(bx), bx.type_i8p()); - // Load just the first byte as that's all that's necessary to force - // LLVM to keep around the reference to the global. - let volative_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); - unsafe { - llvm::LLVMSetAlignment(volative_load_instruction, 1); - } - } -} - -/// Allocates the global variable responsible for the .debug_gdb_scripts binary -/// section. -pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Value { - let c_section_var_name = "__rustc_debug_gdb_scripts_section__\0"; - let section_var_name = &c_section_var_name[..c_section_var_name.len() - 1]; - - let section_var = - unsafe { llvm::LLVMGetNamedGlobal(cx.llmod, c_section_var_name.as_ptr().cast()) }; - - section_var.unwrap_or_else(|| { - let section_name = b".debug_gdb_scripts\0"; - let mut section_contents = Vec::new(); - - // Add the pretty printers for the standard library first. - section_contents.extend_from_slice(b"\x01gdb_load_rust_pretty_printers.py\0"); - - // Next, add the pretty printers that were specified via the `#[debugger_visualizer]` attribute. - let visualizers = collect_debugger_visualizers_transitive( - cx.tcx, - DebuggerVisualizerType::GdbPrettyPrinter, - ); - let crate_name = cx.tcx.crate_name(LOCAL_CRATE); - for (index, visualizer) in visualizers.iter().enumerate() { - // The initial byte `4` instructs GDB that the following pretty printer - // is defined inline as opposed to in a standalone file. - section_contents.extend_from_slice(b"\x04"); - let vis_name = format!("pretty-printer-{}-{}\n", crate_name, index); - section_contents.extend_from_slice(vis_name.as_bytes()); - section_contents.extend_from_slice(&visualizer.src); - - // The final byte `0` tells GDB that the pretty printer has been - // fully defined and can continue searching for additional - // pretty printers. - section_contents.extend_from_slice(b"\0"); - } - - unsafe { - let section_contents = section_contents.as_slice(); - let llvm_type = cx.type_array(cx.type_i8(), section_contents.len() as u64); - - let section_var = cx - .define_global(section_var_name, llvm_type) - .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); - llvm::LLVMSetSection(section_var, section_name.as_ptr().cast()); - llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); - llvm::LLVMSetGlobalConstant(section_var, llvm::True); - llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); - llvm::LLVMRustSetLinkage(section_var, llvm::Linkage::LinkOnceODRLinkage); - // This should make sure that the whole section is not larger than - // the string it contains. Otherwise we get a warning from GDB. - llvm::LLVMSetAlignment(section_var, 1); - section_var - } - }) -} - -pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { - let omit_gdb_pretty_printer_section = - cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); - - // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create - // ODR violations at link time, this section will not be emitted for rlibs since - // each rlib could produce a different set of visualizers that would be embedded - // in the `.debug_gdb_scripts` section. For that reason, we make sure that the - // section is only emitted for leaf crates. - let embed_visualizers = cx.sess().crate_types().iter().any(|&crate_type| match crate_type { - CrateType::Executable | CrateType::Dylib | CrateType::Cdylib | CrateType::Staticlib => { - // These are crate types for which we will embed pretty printers since they - // are treated as leaf crates. - true - } - CrateType::ProcMacro => { - // We could embed pretty printers for proc macro crates too but it does not - // seem like a good default, since this is a rare use case and we don't - // want to slow down the common case. - false - } - CrateType::Rlib => { - // As per the above description, embedding pretty printers for rlibs could - // lead to ODR violations so we skip this crate type as well. - false - } - }); - - !omit_gdb_pretty_printer_section - && cx.sess().opts.debuginfo != DebugInfo::None - && cx.sess().target.emit_debug_gdb_scripts - && embed_visualizers -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs deleted file mode 100644 index bd84100e0e8..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ /dev/null @@ -1,1618 +0,0 @@ -use self::type_map::DINodeCreationResult; -use self::type_map::Stub; -use self::type_map::UniqueTypeId; - -use super::namespace::mangled_name_of_instance; -use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name}; -use super::utils::{ - create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, DIB, -}; -use super::CodegenUnitDebugContext; - -use crate::abi; -use crate::common::CodegenCx; -use crate::debuginfo::metadata::type_map::build_type_with_children; -use crate::debuginfo::utils::fat_pointer_kind; -use crate::debuginfo::utils::FatPtrKind; -use crate::llvm; -use crate::llvm::debuginfo::{ - DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, -}; -use crate::value::Value; - -use cstr::cstr; -use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo; -use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; -use rustc_codegen_ssa::traits::*; -use rustc_fs_util::path_to_c_string; -use rustc_hir::def::CtorKind; -use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_index::vec::{Idx, IndexVec}; -use rustc_middle::bug; -use rustc_middle::mir::{self, GeneratorLayout}; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_middle::ty::subst::GenericArgKind; -use rustc_middle::ty::{ - self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, -}; -use rustc_session::config::{self, DebugInfo, Lto}; -use rustc_span::symbol::Symbol; -use rustc_span::FileName; -use rustc_span::{self, FileNameDisplayPreference, SourceFile}; -use rustc_symbol_mangling::typeid_for_trait_ref; -use rustc_target::abi::{Align, Size}; -use smallvec::smallvec; -use tracing::debug; - -use libc::{c_char, c_longlong, c_uint}; -use std::borrow::Cow; -use std::fmt::{self, Write}; -use std::hash::{Hash, Hasher}; -use std::iter; -use std::path::{Path, PathBuf}; -use std::ptr; -use tracing::instrument; - -impl PartialEq for llvm::Metadata { - fn eq(&self, other: &Self) -> bool { - ptr::eq(self, other) - } -} - -impl Eq for llvm::Metadata {} - -impl Hash for llvm::Metadata { - fn hash(&self, hasher: &mut H) { - (self as *const Self).hash(hasher); - } -} - -impl fmt::Debug for llvm::Metadata { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - (self as *const Self).fmt(f) - } -} - -// From DWARF 5. -// See http://www.dwarfstd.org/ShowIssue.php?issue=140129.1. -const DW_LANG_RUST: c_uint = 0x1c; -#[allow(non_upper_case_globals)] -const DW_ATE_boolean: c_uint = 0x02; -#[allow(non_upper_case_globals)] -const DW_ATE_float: c_uint = 0x04; -#[allow(non_upper_case_globals)] -const DW_ATE_signed: c_uint = 0x05; -#[allow(non_upper_case_globals)] -const DW_ATE_unsigned: c_uint = 0x07; -#[allow(non_upper_case_globals)] -const DW_ATE_UTF: c_uint = 0x10; - -pub(super) const UNKNOWN_LINE_NUMBER: c_uint = 0; -pub(super) const UNKNOWN_COLUMN_NUMBER: c_uint = 0; - -const NO_SCOPE_METADATA: Option<&DIScope> = None; -/// A function that returns an empty list of generic parameter debuginfo nodes. -const NO_GENERICS: for<'ll> fn(&CodegenCx<'ll, '_>) -> SmallVec<&'ll DIType> = |_| SmallVec::new(); - -// SmallVec is used quite a bit in this module, so create a shorthand. -// The actual number of elements is not so important. -pub type SmallVec = smallvec::SmallVec<[T; 16]>; - -mod enums; -mod type_map; - -pub(crate) use type_map::TypeMap; - -/// Returns from the enclosing function if the type debuginfo node with the given -/// unique ID can be found in the type map. -macro_rules! return_if_di_node_created_in_meantime { - ($cx: expr, $unique_type_id: expr) => { - if let Some(di_node) = debug_context($cx).type_map.di_node_for_unique_id($unique_type_id) { - return DINodeCreationResult::new(di_node, true); - } - }; -} - -/// Extract size and alignment from a TyAndLayout. -fn size_and_align_of<'tcx>(ty_and_layout: TyAndLayout<'tcx>) -> (Size, Align) { - (ty_and_layout.size, ty_and_layout.align.abi) -} - -/// Creates debuginfo for a fixed size array (e.g. `[u64; 123]`). -/// For slices (that is, "arrays" of unknown size) use [build_slice_type_di_node]. -fn build_fixed_size_array_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, - array_type: Ty<'tcx>, -) -> DINodeCreationResult<'ll> { - let ty::Array(element_type, len) = array_type.kind() else { - bug!("build_fixed_size_array_di_node() called with non-ty::Array type `{:?}`", array_type) - }; - - let element_type_di_node = type_di_node(cx, *element_type); - - return_if_di_node_created_in_meantime!(cx, unique_type_id); - - let (size, align) = cx.size_and_align_of(array_type); - - let upper_bound = len.eval_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong; - - let subrange = - unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) }; - - let subscripts = create_DIArray(DIB(cx), &[subrange]); - let di_node = unsafe { - llvm::LLVMRustDIBuilderCreateArrayType( - DIB(cx), - size.bits(), - align.bits() as u32, - element_type_di_node, - subscripts, - ) - }; - - DINodeCreationResult::new(di_node, false) -} - -/// Creates debuginfo for built-in pointer-like things: -/// -/// - ty::Ref -/// - ty::RawPtr -/// - ty::Adt in the case it's Box -/// -/// At some point we might want to remove the special handling of Box -/// and treat it the same as other smart pointers (like Rc, Arc, ...). -fn build_pointer_or_reference_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - ptr_type: Ty<'tcx>, - pointee_type: Ty<'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - // The debuginfo generated by this function is only valid if `ptr_type` is really just - // a (fat) pointer. Make sure it is not called for e.g. `Box`. - debug_assert_eq!( - cx.size_and_align_of(ptr_type), - cx.size_and_align_of(cx.tcx.mk_mut_ptr(pointee_type)) - ); - - let pointee_type_di_node = type_di_node(cx, pointee_type); - - return_if_di_node_created_in_meantime!(cx, unique_type_id); - - let (thin_pointer_size, thin_pointer_align) = - cx.size_and_align_of(cx.tcx.mk_imm_ptr(cx.tcx.types.unit)); - let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); - - match fat_pointer_kind(cx, pointee_type) { - None => { - // This is a thin pointer. Create a regular pointer type and give it the correct name. - debug_assert_eq!( - (thin_pointer_size, thin_pointer_align), - cx.size_and_align_of(ptr_type), - "ptr_type={}, pointee_type={}", - ptr_type, - pointee_type, - ); - - let di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - pointee_type_di_node, - thin_pointer_size.bits(), - thin_pointer_align.bits() as u32, - 0, // Ignore DWARF address space. - ptr_type_debuginfo_name.as_ptr().cast(), - ptr_type_debuginfo_name.len(), - ) - }; - - DINodeCreationResult { di_node, already_stored_in_typemap: false } - } - Some(fat_pointer_kind) => { - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Struct, - unique_type_id, - &ptr_type_debuginfo_name, - cx.size_and_align_of(ptr_type), - NO_SCOPE_METADATA, - DIFlags::FlagZero, - ), - |cx, owner| { - // FIXME: If this fat pointer is a `Box` then we don't want to use its - // type layout and instead use the layout of the raw pointer inside - // of it. - // The proper way to handle this is to not treat Box as a pointer - // at all and instead emit regular struct debuginfo for it. We just - // need to make sure that we don't break existing debuginfo consumers - // by doing that (at least not without a warning period). - let layout_type = - if ptr_type.is_box() { cx.tcx.mk_mut_ptr(pointee_type) } else { ptr_type }; - - let layout = cx.layout_of(layout_type); - let addr_field = layout.field(cx, abi::FAT_PTR_ADDR); - let extra_field = layout.field(cx, abi::FAT_PTR_EXTRA); - - let (addr_field_name, extra_field_name) = match fat_pointer_kind { - FatPtrKind::Dyn => ("pointer", "vtable"), - FatPtrKind::Slice => ("data_ptr", "length"), - }; - - debug_assert_eq!(abi::FAT_PTR_ADDR, 0); - debug_assert_eq!(abi::FAT_PTR_EXTRA, 1); - - // The data pointer type is a regular, thin pointer, regardless of whether this - // is a slice or a trait object. - let data_ptr_type_di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - pointee_type_di_node, - addr_field.size.bits(), - addr_field.align.abi.bits() as u32, - 0, // Ignore DWARF address space. - std::ptr::null(), - 0, - ) - }; - - smallvec![ - build_field_di_node( - cx, - owner, - addr_field_name, - (addr_field.size, addr_field.align.abi), - layout.fields.offset(abi::FAT_PTR_ADDR), - DIFlags::FlagZero, - data_ptr_type_di_node, - ), - build_field_di_node( - cx, - owner, - extra_field_name, - (extra_field.size, extra_field.align.abi), - layout.fields.offset(abi::FAT_PTR_EXTRA), - DIFlags::FlagZero, - type_di_node(cx, extra_field.ty), - ), - ] - }, - NO_GENERICS, - ) - } - } -} - -fn build_subroutine_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - // It's possible to create a self-referential - // type in Rust by using 'impl trait': - // - // fn foo() -> impl Copy { foo } - // - // Unfortunately LLVM's API does not allow us to create recursive subroutine types. - // In order to work around that restriction we place a marker type in the type map, - // before creating the actual type. If the actual type is recursive, it will hit the - // marker type. So we end up with a type that looks like - // - // fn foo() -> - // - // Once that is created, we replace the marker in the typemap with the actual type. - debug_context(cx) - .type_map - .unique_id_to_di_node - .borrow_mut() - .insert(unique_type_id, recursion_marker_type_di_node(cx)); - - let fn_ty = unique_type_id.expect_ty(); - let signature = cx - .tcx - .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), fn_ty.fn_sig(cx.tcx)); - - let signature_di_nodes: SmallVec<_> = iter::once( - // return type - match signature.output().kind() { - ty::Tuple(tys) if tys.is_empty() => { - // this is a "void" function - None - } - _ => Some(type_di_node(cx, signature.output())), - }, - ) - .chain( - // regular arguments - signature.inputs().iter().map(|&argument_type| Some(type_di_node(cx, argument_type))), - ) - .collect(); - - debug_context(cx).type_map.unique_id_to_di_node.borrow_mut().remove(&unique_type_id); - - let fn_di_node = unsafe { - llvm::LLVMRustDIBuilderCreateSubroutineType( - DIB(cx), - create_DIArray(DIB(cx), &signature_di_nodes[..]), - ) - }; - - // This is actually a function pointer, so wrap it in pointer DI. - let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); - let di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - fn_di_node, - cx.tcx.data_layout.pointer_size.bits(), - cx.tcx.data_layout.pointer_align.abi.bits() as u32, - 0, // Ignore DWARF address space. - name.as_ptr().cast(), - name.len(), - ) - }; - - DINodeCreationResult::new(di_node, false) -} - -/// Create debuginfo for `dyn SomeTrait` types. Currently these are empty structs -/// we with the correct type name (e.g. "dyn SomeTrait + Sync"). -fn build_dyn_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - dyn_type: Ty<'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - if let ty::Dynamic(..) = dyn_type.kind() { - let type_name = compute_debuginfo_type_name(cx.tcx, dyn_type, true); - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Struct, - unique_type_id, - &type_name, - cx.size_and_align_of(dyn_type), - NO_SCOPE_METADATA, - DIFlags::FlagZero, - ), - |_, _| smallvec![], - NO_GENERICS, - ) - } else { - bug!( - "Only ty::Dynamic is valid for build_dyn_type_di_node(). Found {:?} instead.", - dyn_type - ) - } -} - -/// Create debuginfo for `[T]` and `str`. These are unsized. -/// -/// NOTE: We currently emit just emit the debuginfo for the element type here -/// (i.e. `T` for slices and `u8` for `str`), so that we end up with -/// `*const T` for the `data_ptr` field of the corresponding fat-pointer -/// debuginfo of `&[T]`. -/// -/// It would be preferable and more accurate if we emitted a DIArray of T -/// without an upper bound instead. That is, LLVM already supports emitting -/// debuginfo of arrays of unknown size. But GDB currently seems to end up -/// in an infinite loop when confronted with such a type. -/// -/// As a side effect of the current encoding every instance of a type like -/// `struct Foo { unsized_field: [u8] }` will look like -/// `struct Foo { unsized_field: u8 }` in debuginfo. If the length of the -/// slice is zero, then accessing `unsized_field` in the debugger would -/// result in an out-of-bounds access. -fn build_slice_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - slice_type: Ty<'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - let element_type = match slice_type.kind() { - ty::Slice(element_type) => *element_type, - ty::Str => cx.tcx.types.u8, - _ => { - bug!( - "Only ty::Slice is valid for build_slice_type_di_node(). Found {:?} instead.", - slice_type - ) - } - }; - - let element_type_di_node = type_di_node(cx, element_type); - return_if_di_node_created_in_meantime!(cx, unique_type_id); - DINodeCreationResult { di_node: element_type_di_node, already_stored_in_typemap: false } -} - -/// Get the debuginfo node for the given type. -/// -/// This function will look up the debuginfo node in the TypeMap. If it can't find it, it -/// will create the node by dispatching to the corresponding `build_*_di_node()` function. -pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { - let unique_type_id = UniqueTypeId::for_ty(cx.tcx, t); - - if let Some(existing_di_node) = debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) - { - return existing_di_node; - } - - debug!("type_di_node: {:?}", t); - - let DINodeCreationResult { di_node, already_stored_in_typemap } = match *t.kind() { - ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { - build_basic_type_di_node(cx, t) - } - ty::Tuple(elements) if elements.is_empty() => build_basic_type_di_node(cx, t), - ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t), - ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id), - ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id), - ty::Foreign(..) => build_foreign_type_di_node(cx, t, unique_type_id), - ty::RawPtr(ty::TypeAndMut { ty: pointee_type, .. }) | ty::Ref(_, pointee_type, _) => { - build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id) - } - // Box may have a non-ZST allocator A. In that case, we - // cannot treat Box as just an owned alias of `*mut T`. - ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { - build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) - } - ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), - ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), - ty::Generator(..) => enums::build_generator_di_node(cx, unique_type_id), - ty::Adt(def, ..) => match def.adt_kind() { - AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), - AdtKind::Union => build_union_type_di_node(cx, unique_type_id), - AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id), - }, - ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), - // Type parameters from polymorphized functions. - ty::Param(_) => build_param_type_di_node(cx, t), - _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t), - }; - - { - if already_stored_in_typemap { - // Make sure that we really do have a `TypeMap` entry for the unique type ID. - let di_node_for_uid = - match debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) { - Some(di_node) => di_node, - None => { - bug!( - "expected type debuginfo node for unique \ - type ID '{:?}' to already be in \ - the `debuginfo::TypeMap` but it \ - was not.", - unique_type_id, - ); - } - }; - - debug_assert_eq!(di_node_for_uid as *const _, di_node as *const _); - } else { - debug_context(cx).type_map.insert(unique_type_id, di_node); - } - } - - di_node -} - -// FIXME(mw): Cache this via a regular UniqueTypeId instead of an extra field in the debug context. -fn recursion_marker_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll DIType { - *debug_context(cx).recursion_marker_type.get_or_init(move || { - unsafe { - // The choice of type here is pretty arbitrary - - // anything reading the debuginfo for a recursive - // type is going to see *something* weird - the only - // question is what exactly it will see. - // - // FIXME: the name `` does not fit the naming scheme - // of other types. - // - // FIXME: it might make sense to use an actual pointer type here - // so that debuggers can show the address. - let name = ""; - llvm::LLVMRustDIBuilderCreateBasicType( - DIB(cx), - name.as_ptr().cast(), - name.len(), - cx.tcx.data_layout.pointer_size.bits(), - DW_ATE_unsigned, - ) - } - }) -} - -fn hex_encode(data: &[u8]) -> String { - let mut hex_string = String::with_capacity(data.len() * 2); - for byte in data.iter() { - write!(&mut hex_string, "{:02x}", byte).unwrap(); - } - hex_string -} - -pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { - let cache_key = Some((source_file.name_hash, source_file.src_hash)); - return debug_context(cx) - .created_files - .borrow_mut() - .entry(cache_key) - .or_insert_with(|| alloc_new_file_metadata(cx, source_file)); - - #[instrument(skip(cx, source_file), level = "debug")] - fn alloc_new_file_metadata<'ll>( - cx: &CodegenCx<'ll, '_>, - source_file: &SourceFile, - ) -> &'ll DIFile { - debug!(?source_file.name); - - let (directory, file_name) = match &source_file.name { - FileName::Real(filename) => { - let working_directory = &cx.sess().opts.working_dir; - debug!(?working_directory); - - let filename = cx - .sess() - .source_map() - .path_mapping() - .to_embeddable_absolute_path(filename.clone(), working_directory); - - // Construct the absolute path of the file - let abs_path = filename.remapped_path_if_available(); - debug!(?abs_path); - - if let Ok(rel_path) = - abs_path.strip_prefix(working_directory.remapped_path_if_available()) - { - // If the compiler's working directory (which also is the DW_AT_comp_dir of - // the compilation unit) is a prefix of the path we are about to emit, then - // only emit the part relative to the working directory. - // Because of path remapping we sometimes see strange things here: `abs_path` - // might actually look like a relative path - // (e.g. `/src/lib.rs`), so if we emit it without - // taking the working directory into account, downstream tooling will - // interpret it as `//src/lib.rs`, - // which makes no sense. Usually in such cases the working directory will also - // be remapped to `` or some other prefix of the path - // we are remapping, so we end up with - // `//src/lib.rs`. - // By moving the working directory portion into the `directory` part of the - // DIFile, we allow LLVM to emit just the relative path for DWARF, while - // still emitting the correct absolute path for CodeView. - ( - working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), - rel_path.to_string_lossy().into_owned(), - ) - } else { - ("".into(), abs_path.to_string_lossy().into_owned()) - } - } - other => ("".into(), other.prefer_remapped().to_string_lossy().into_owned()), - }; - - let hash_kind = match source_file.src_hash.kind { - rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5, - rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1, - rustc_span::SourceFileHashAlgorithm::Sha256 => llvm::ChecksumKind::SHA256, - }; - let hash_value = hex_encode(source_file.src_hash.hash_bytes()); - - unsafe { - llvm::LLVMRustDIBuilderCreateFile( - DIB(cx), - file_name.as_ptr().cast(), - file_name.len(), - directory.as_ptr().cast(), - directory.len(), - hash_kind, - hash_value.as_ptr().cast(), - hash_value.len(), - ) - } - } -} - -pub fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { - debug_context(cx).created_files.borrow_mut().entry(None).or_insert_with(|| unsafe { - let file_name = ""; - let directory = ""; - let hash_value = ""; - - llvm::LLVMRustDIBuilderCreateFile( - DIB(cx), - file_name.as_ptr().cast(), - file_name.len(), - directory.as_ptr().cast(), - directory.len(), - llvm::ChecksumKind::None, - hash_value.as_ptr().cast(), - hash_value.len(), - ) - }) -} - -trait MsvcBasicName { - fn msvc_basic_name(self) -> &'static str; -} - -impl MsvcBasicName for ty::IntTy { - fn msvc_basic_name(self) -> &'static str { - match self { - ty::IntTy::Isize => "ptrdiff_t", - ty::IntTy::I8 => "__int8", - ty::IntTy::I16 => "__int16", - ty::IntTy::I32 => "__int32", - ty::IntTy::I64 => "__int64", - ty::IntTy::I128 => "__int128", - } - } -} - -impl MsvcBasicName for ty::UintTy { - fn msvc_basic_name(self) -> &'static str { - match self { - ty::UintTy::Usize => "size_t", - ty::UintTy::U8 => "unsigned __int8", - ty::UintTy::U16 => "unsigned __int16", - ty::UintTy::U32 => "unsigned __int32", - ty::UintTy::U64 => "unsigned __int64", - ty::UintTy::U128 => "unsigned __int128", - } - } -} - -impl MsvcBasicName for ty::FloatTy { - fn msvc_basic_name(self) -> &'static str { - match self { - ty::FloatTy::F32 => "float", - ty::FloatTy::F64 => "double", - } - } -} - -fn build_basic_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - t: Ty<'tcx>, -) -> DINodeCreationResult<'ll> { - debug!("build_basic_type_di_node: {:?}", t); - - // When targeting MSVC, emit MSVC style type names for compatibility with - // .natvis visualizers (and perhaps other existing native debuggers?) - let cpp_like_debuginfo = cpp_like_debuginfo(cx.tcx); - - let (name, encoding) = match t.kind() { - ty::Never => ("!", DW_ATE_unsigned), - ty::Tuple(elements) if elements.is_empty() => { - if cpp_like_debuginfo { - return build_tuple_type_di_node(cx, UniqueTypeId::for_ty(cx.tcx, t)); - } else { - ("()", DW_ATE_unsigned) - } - } - ty::Bool => ("bool", DW_ATE_boolean), - ty::Char => ("char", DW_ATE_UTF), - ty::Int(int_ty) if cpp_like_debuginfo => (int_ty.msvc_basic_name(), DW_ATE_signed), - ty::Uint(uint_ty) if cpp_like_debuginfo => (uint_ty.msvc_basic_name(), DW_ATE_unsigned), - ty::Float(float_ty) if cpp_like_debuginfo => (float_ty.msvc_basic_name(), DW_ATE_float), - ty::Int(int_ty) => (int_ty.name_str(), DW_ATE_signed), - ty::Uint(uint_ty) => (uint_ty.name_str(), DW_ATE_unsigned), - ty::Float(float_ty) => (float_ty.name_str(), DW_ATE_float), - _ => bug!("debuginfo::build_basic_type_di_node - `t` is invalid type"), - }; - - let ty_di_node = unsafe { - llvm::LLVMRustDIBuilderCreateBasicType( - DIB(cx), - name.as_ptr().cast(), - name.len(), - cx.size_of(t).bits(), - encoding, - ) - }; - - if !cpp_like_debuginfo { - return DINodeCreationResult::new(ty_di_node, false); - } - - let typedef_name = match t.kind() { - ty::Int(int_ty) => int_ty.name_str(), - ty::Uint(uint_ty) => uint_ty.name_str(), - ty::Float(float_ty) => float_ty.name_str(), - _ => return DINodeCreationResult::new(ty_di_node, false), - }; - - let typedef_di_node = unsafe { - llvm::LLVMRustDIBuilderCreateTypedef( - DIB(cx), - ty_di_node, - typedef_name.as_ptr().cast(), - typedef_name.len(), - unknown_file_metadata(cx), - 0, - None, - ) - }; - - DINodeCreationResult::new(typedef_di_node, false) -} - -fn build_foreign_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - t: Ty<'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - debug!("build_foreign_type_di_node: {:?}", t); - - let &ty::Foreign(def_id) = unique_type_id.expect_ty().kind() else { - bug!("build_foreign_type_di_node() called with unexpected type: {:?}", unique_type_id.expect_ty()); - }; - - build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Struct, - unique_type_id, - &compute_debuginfo_type_name(cx.tcx, t, false), - cx.size_and_align_of(t), - Some(get_namespace_for_item(cx, def_id)), - DIFlags::FlagZero, - ), - |_, _| smallvec![], - NO_GENERICS, - ) -} - -fn build_param_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - t: Ty<'tcx>, -) -> DINodeCreationResult<'ll> { - debug!("build_param_type_di_node: {:?}", t); - let name = format!("{:?}", t); - DINodeCreationResult { - di_node: unsafe { - llvm::LLVMRustDIBuilderCreateBasicType( - DIB(cx), - name.as_ptr().cast(), - name.len(), - Size::ZERO.bits(), - DW_ATE_unsigned, - ) - }, - already_stored_in_typemap: false, - } -} - -pub fn build_compile_unit_di_node<'ll, 'tcx>( - tcx: TyCtxt<'tcx>, - codegen_unit_name: &str, - debug_context: &CodegenUnitDebugContext<'ll, 'tcx>, -) -> &'ll DIDescriptor { - let mut name_in_debuginfo = match tcx.sess.local_crate_source_file { - Some(ref path) => path.clone(), - None => PathBuf::from(tcx.crate_name(LOCAL_CRATE).as_str()), - }; - - // To avoid breaking split DWARF, we need to ensure that each codegen unit - // has a unique `DW_AT_name`. This is because there's a remote chance that - // different codegen units for the same module will have entirely - // identical DWARF entries for the purpose of the DWO ID, which would - // violate Appendix F ("Split Dwarf Object Files") of the DWARF 5 - // specification. LLVM uses the algorithm specified in section 7.32 "Type - // Signature Computation" to compute the DWO ID, which does not include - // any fields that would distinguish compilation units. So we must embed - // the codegen unit name into the `DW_AT_name`. (Issue #88521.) - // - // Additionally, the OSX linker has an idiosyncrasy where it will ignore - // some debuginfo if multiple object files with the same `DW_AT_name` are - // linked together. - // - // As a workaround for these two issues, we generate unique names for each - // object file. Those do not correspond to an actual source file but that - // is harmless. - name_in_debuginfo.push("@"); - name_in_debuginfo.push(codegen_unit_name); - - debug!("build_compile_unit_di_node: {:?}", name_in_debuginfo); - let rustc_producer = - format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"),); - // FIXME(#41252) Remove "clang LLVM" if we can get GDB and LLVM to play nice. - let producer = format!("clang LLVM ({})", rustc_producer); - - let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); - let work_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped); - let flags = "\0"; - let output_filenames = tcx.output_filenames(()); - let split_name = if tcx.sess.target_can_use_split_dwarf() { - output_filenames - .split_dwarf_path( - tcx.sess.split_debuginfo(), - tcx.sess.opts.unstable_opts.split_dwarf_kind, - Some(codegen_unit_name), - ) - // We get a path relative to the working directory from split_dwarf_path - .map(|f| tcx.sess.source_map().path_mapping().map_prefix(f).0) - } else { - None - } - .unwrap_or_default(); - let split_name = split_name.to_str().unwrap(); - - // FIXME(#60020): - // - // This should actually be - // - // let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); - // - // That is, we should set LLVM's emission kind to `LineTablesOnly` if - // we are compiling with "limited" debuginfo. However, some of the - // existing tools relied on slightly more debuginfo being generated than - // would be the case with `LineTablesOnly`, and we did not want to break - // these tools in a "drive-by fix", without a good idea or plan about - // what limited debuginfo should exactly look like. So for now we keep - // the emission kind as `FullDebug`. - // - // See https://github.com/rust-lang/rust/issues/60020 for details. - let kind = DebugEmissionKind::FullDebug; - assert!(tcx.sess.opts.debuginfo != DebugInfo::None); - - unsafe { - let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( - debug_context.builder, - name_in_debuginfo.as_ptr().cast(), - name_in_debuginfo.len(), - work_dir.as_ptr().cast(), - work_dir.len(), - llvm::ChecksumKind::None, - ptr::null(), - 0, - ); - - let unit_metadata = llvm::LLVMRustDIBuilderCreateCompileUnit( - debug_context.builder, - DW_LANG_RUST, - compile_unit_file, - producer.as_ptr().cast(), - producer.len(), - tcx.sess.opts.optimize != config::OptLevel::No, - flags.as_ptr().cast(), - 0, - // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead - // put the path supplied to `MCSplitDwarfFile` into the debug info of the final - // output(s). - split_name.as_ptr().cast(), - split_name.len(), - kind, - 0, - tcx.sess.opts.unstable_opts.split_dwarf_inlining, - ); - - if tcx.sess.opts.unstable_opts.profile { - let cu_desc_metadata = - llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata); - let default_gcda_path = &output_filenames.with_extension("gcda"); - let gcda_path = - tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); - - let gcov_cu_info = [ - path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), - path_to_mdstring(debug_context.llcontext, gcda_path), - cu_desc_metadata, - ]; - let gcov_metadata = llvm::LLVMMDNodeInContext( - debug_context.llcontext, - gcov_cu_info.as_ptr(), - gcov_cu_info.len() as c_uint, - ); - - let llvm_gcov_ident = cstr!("llvm.gcov"); - llvm::LLVMAddNamedMetadataOperand( - debug_context.llmod, - llvm_gcov_ident.as_ptr(), - gcov_metadata, - ); - } - - // Insert `llvm.ident` metadata on the wasm targets since that will - // get hooked up to the "producer" sections `processed-by` information. - if tcx.sess.target.is_like_wasm { - let name_metadata = llvm::LLVMMDStringInContext( - debug_context.llcontext, - rustc_producer.as_ptr().cast(), - rustc_producer.as_bytes().len() as c_uint, - ); - llvm::LLVMAddNamedMetadataOperand( - debug_context.llmod, - cstr!("llvm.ident").as_ptr(), - llvm::LLVMMDNodeInContext(debug_context.llcontext, &name_metadata, 1), - ); - } - - return unit_metadata; - }; - - fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll Value { - let path_str = path_to_c_string(path); - unsafe { - llvm::LLVMMDStringInContext( - llcx, - path_str.as_ptr(), - path_str.as_bytes().len() as c_uint, - ) - } - } -} - -/// Creates a `DW_TAG_member` entry inside the DIE represented by the given `type_di_node`. -fn build_field_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - owner: &'ll DIScope, - name: &str, - size_and_align: (Size, Align), - offset: Size, - flags: DIFlags, - type_di_node: &'ll DIType, -) -> &'ll DIType { - unsafe { - llvm::LLVMRustDIBuilderCreateMemberType( - DIB(cx), - owner, - name.as_ptr().cast(), - name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - size_and_align.0.bits(), - size_and_align.1.bits() as u32, - offset.bits(), - flags, - type_di_node, - ) - } -} - -/// Creates the debuginfo node for a Rust struct type. Maybe be a regular struct or a tuple-struct. -fn build_struct_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - let struct_type = unique_type_id.expect_ty(); - let ty::Adt(adt_def, _) = struct_type.kind() else { - bug!("build_struct_type_di_node() called with non-struct-type: {:?}", struct_type); - }; - debug_assert!(adt_def.is_struct()); - let containing_scope = get_namespace_for_item(cx, adt_def.did()); - let struct_type_and_layout = cx.layout_of(struct_type); - let variant_def = adt_def.non_enum_variant(); - - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Struct, - unique_type_id, - &compute_debuginfo_type_name(cx.tcx, struct_type, false), - size_and_align_of(struct_type_and_layout), - Some(containing_scope), - DIFlags::FlagZero, - ), - // Fields: - |cx, owner| { - variant_def - .fields - .iter() - .enumerate() - .map(|(i, f)| { - let field_name = if variant_def.ctor_kind == CtorKind::Fn { - // This is a tuple struct - tuple_field_name(i) - } else { - // This is struct with named fields - Cow::Borrowed(f.name.as_str()) - }; - let field_layout = struct_type_and_layout.field(cx, i); - build_field_di_node( - cx, - owner, - &field_name[..], - (field_layout.size, field_layout.align.abi), - struct_type_and_layout.fields.offset(i), - DIFlags::FlagZero, - type_di_node(cx, field_layout.ty), - ) - }) - .collect() - }, - |cx| build_generic_type_param_di_nodes(cx, struct_type), - ) -} - -//=----------------------------------------------------------------------------- -// Tuples -//=----------------------------------------------------------------------------- - -/// Returns names of captured upvars for closures and generators. -/// -/// Here are some examples: -/// - `name__field1__field2` when the upvar is captured by value. -/// - `_ref__name__field` when the upvar is captured by reference. -/// -/// For generators this only contains upvars that are shared by all states. -fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> SmallVec { - let body = tcx.optimized_mir(def_id); - - body.var_debug_info - .iter() - .filter_map(|var| { - let is_ref = match var.value { - mir::VarDebugInfoContents::Place(place) if place.local == mir::Local::new(1) => { - // The projection is either `[.., Field, Deref]` or `[.., Field]`. It - // implies whether the variable is captured by value or by reference. - matches!(place.projection.last().unwrap(), mir::ProjectionElem::Deref) - } - _ => return None, - }; - let prefix = if is_ref { "_ref__" } else { "" }; - Some(prefix.to_owned() + var.name.as_str()) - }) - .collect() -} - -/// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or generator. -/// For a generator, this will handle upvars shared by all states. -fn build_upvar_field_di_nodes<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - closure_or_generator_ty: Ty<'tcx>, - closure_or_generator_di_node: &'ll DIType, -) -> SmallVec<&'ll DIType> { - let (&def_id, up_var_tys) = match closure_or_generator_ty.kind() { - ty::Generator(def_id, substs, _) => { - let upvar_tys: SmallVec<_> = substs.as_generator().prefix_tys().collect(); - (def_id, upvar_tys) - } - ty::Closure(def_id, substs) => { - let upvar_tys: SmallVec<_> = substs.as_closure().upvar_tys().collect(); - (def_id, upvar_tys) - } - _ => { - bug!( - "build_upvar_field_di_nodes() called with non-closure-or-generator-type: {:?}", - closure_or_generator_ty - ) - } - }; - - debug_assert!( - up_var_tys - .iter() - .all(|&t| t == cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) - ); - - let capture_names = closure_saved_names_of_captured_variables(cx.tcx, def_id); - let layout = cx.layout_of(closure_or_generator_ty); - - up_var_tys - .into_iter() - .zip(capture_names.iter()) - .enumerate() - .map(|(index, (up_var_ty, capture_name))| { - build_field_di_node( - cx, - closure_or_generator_di_node, - capture_name, - cx.size_and_align_of(up_var_ty), - layout.fields.offset(index), - DIFlags::FlagZero, - type_di_node(cx, up_var_ty), - ) - }) - .collect() -} - -/// Builds the DW_TAG_structure_type debuginfo node for a Rust tuple type. -fn build_tuple_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - let tuple_type = unique_type_id.expect_ty(); - let &ty::Tuple(component_types) = tuple_type.kind() else { - bug!("build_tuple_type_di_node() called with non-tuple-type: {:?}", tuple_type) - }; - - let tuple_type_and_layout = cx.layout_of(tuple_type); - let type_name = compute_debuginfo_type_name(cx.tcx, tuple_type, false); - - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Struct, - unique_type_id, - &type_name, - size_and_align_of(tuple_type_and_layout), - NO_SCOPE_METADATA, - DIFlags::FlagZero, - ), - // Fields: - |cx, tuple_di_node| { - component_types - .into_iter() - .enumerate() - .map(|(index, component_type)| { - build_field_di_node( - cx, - tuple_di_node, - &tuple_field_name(index), - cx.size_and_align_of(component_type), - tuple_type_and_layout.fields.offset(index), - DIFlags::FlagZero, - type_di_node(cx, component_type), - ) - }) - .collect() - }, - NO_GENERICS, - ) -} - -/// Builds the debuginfo node for a closure environment. -fn build_closure_env_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - let closure_env_type = unique_type_id.expect_ty(); - let &ty::Closure(def_id, _substs) = closure_env_type.kind() else { - bug!("build_closure_env_di_node() called with non-closure-type: {:?}", closure_env_type) - }; - let containing_scope = get_namespace_for_item(cx, def_id); - let type_name = compute_debuginfo_type_name(cx.tcx, closure_env_type, false); - - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Struct, - unique_type_id, - &type_name, - cx.size_and_align_of(closure_env_type), - Some(containing_scope), - DIFlags::FlagZero, - ), - // Fields: - |cx, owner| build_upvar_field_di_nodes(cx, closure_env_type, owner), - NO_GENERICS, - ) -} - -/// Build the debuginfo node for a Rust `union` type. -fn build_union_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - let union_type = unique_type_id.expect_ty(); - let (union_def_id, variant_def) = match union_type.kind() { - ty::Adt(def, _) => (def.did(), def.non_enum_variant()), - _ => bug!("build_union_type_di_node on a non-ADT"), - }; - let containing_scope = get_namespace_for_item(cx, union_def_id); - let union_ty_and_layout = cx.layout_of(union_type); - let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); - - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Union, - unique_type_id, - &type_name, - size_and_align_of(union_ty_and_layout), - Some(containing_scope), - DIFlags::FlagZero, - ), - // Fields: - |cx, owner| { - variant_def - .fields - .iter() - .enumerate() - .map(|(i, f)| { - let field_layout = union_ty_and_layout.field(cx, i); - build_field_di_node( - cx, - owner, - f.name.as_str(), - size_and_align_of(field_layout), - Size::ZERO, - DIFlags::FlagZero, - type_di_node(cx, field_layout.ty), - ) - }) - .collect() - }, - // Generics: - |cx| build_generic_type_param_di_nodes(cx, union_type), - ) -} - -// FIXME(eddyb) maybe precompute this? Right now it's computed once -// per generator monomorphization, but it doesn't depend on substs. -fn generator_layout_and_saved_local_names<'tcx>( - tcx: TyCtxt<'tcx>, - def_id: DefId, -) -> (&'tcx GeneratorLayout<'tcx>, IndexVec>) { - let body = tcx.optimized_mir(def_id); - let generator_layout = body.generator_layout().unwrap(); - let mut generator_saved_local_names = IndexVec::from_elem(None, &generator_layout.field_tys); - - let state_arg = mir::Local::new(1); - for var in &body.var_debug_info { - let mir::VarDebugInfoContents::Place(place) = &var.value else { continue }; - if place.local != state_arg { - continue; - } - match place.projection[..] { - [ - // Deref of the `Pin<&mut Self>` state argument. - mir::ProjectionElem::Field(..), - mir::ProjectionElem::Deref, - // Field of a variant of the state. - mir::ProjectionElem::Downcast(_, variant), - mir::ProjectionElem::Field(field, _), - ] => { - let name = &mut generator_saved_local_names - [generator_layout.variant_fields[variant][field]]; - if name.is_none() { - name.replace(var.name); - } - } - _ => {} - } - } - (generator_layout, generator_saved_local_names) -} - -/// Computes the type parameters for a type, if any, for the given metadata. -fn build_generic_type_param_di_nodes<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - ty: Ty<'tcx>, -) -> SmallVec<&'ll DIType> { - if let ty::Adt(def, substs) = *ty.kind() { - if substs.types().next().is_some() { - let generics = cx.tcx.generics_of(def.did()); - let names = get_parameter_names(cx, generics); - let template_params: SmallVec<_> = iter::zip(substs, names) - .filter_map(|(kind, name)| { - if let GenericArgKind::Type(ty) = kind.unpack() { - let actual_type = - cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); - let actual_type_di_node = type_di_node(cx, actual_type); - let name = name.as_str(); - Some(unsafe { - llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( - DIB(cx), - None, - name.as_ptr().cast(), - name.len(), - actual_type_di_node, - ) - }) - } else { - None - } - }) - .collect(); - - return template_params; - } - } - - return smallvec![]; - - fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec { - let mut names = generics - .parent - .map_or_else(Vec::new, |def_id| get_parameter_names(cx, cx.tcx.generics_of(def_id))); - names.extend(generics.params.iter().map(|param| param.name)); - names - } -} - -/// Creates debug information for the given global variable. -/// -/// Adds the created debuginfo nodes directly to the crate's IR. -pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, global: &'ll Value) { - if cx.dbg_cx.is_none() { - return; - } - - // Only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo != DebugInfo::Full { - return; - } - - let tcx = cx.tcx; - - // We may want to remove the namespace scope if we're in an extern block (see - // https://github.com/rust-lang/rust/pull/46457#issuecomment-351750952). - let var_scope = get_namespace_for_item(cx, def_id); - let span = tcx.def_span(def_id); - - let (file_metadata, line_number) = if !span.is_dummy() { - let loc = cx.lookup_debug_loc(span.lo()); - (file_metadata(cx, &loc.file), loc.line) - } else { - (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) - }; - - let is_local_to_unit = is_node_local_to_unit(cx, def_id); - let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all()); - let type_di_node = type_di_node(cx, variable_type); - let var_name = tcx.item_name(def_id); - let var_name = var_name.as_str(); - let linkage_name = mangled_name_of_instance(cx, Instance::mono(tcx, def_id)).name; - // When empty, linkage_name field is omitted, - // which is what we want for no_mangle statics - let linkage_name = if var_name == linkage_name { "" } else { linkage_name }; - - let global_align = cx.align_of(variable_type); - - unsafe { - llvm::LLVMRustDIBuilderCreateStaticVariable( - DIB(cx), - Some(var_scope), - var_name.as_ptr().cast(), - var_name.len(), - linkage_name.as_ptr().cast(), - linkage_name.len(), - file_metadata, - line_number, - type_di_node, - is_local_to_unit, - global, - None, - global_align.bits() as u32, - ); - } -} - -/// Generates LLVM debuginfo for a vtable. -/// -/// The vtable type looks like a struct with a field for each function pointer and super-trait -/// pointer it contains (plus the `size` and `align` fields). -/// -/// Except for `size`, `align`, and `drop_in_place`, the field names don't try to mirror -/// the name of the method they implement. This can be implemented in the future once there -/// is a proper disambiguation scheme for dealing with methods from different traits that have -/// the same name. -fn build_vtable_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - ty: Ty<'tcx>, - poly_trait_ref: Option>, -) -> &'ll DIType { - let tcx = cx.tcx; - - let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref { - let trait_ref = poly_trait_ref.with_self_ty(tcx, ty); - let trait_ref = tcx.erase_regions(trait_ref); - - tcx.vtable_entries(trait_ref) - } else { - TyCtxt::COMMON_VTABLE_ENTRIES - }; - - // All function pointers are described as opaque pointers. This could be improved in the future - // by describing them as actual function pointers. - let void_pointer_ty = tcx.mk_imm_ptr(tcx.types.unit); - let void_pointer_type_di_node = type_di_node(cx, void_pointer_ty); - let usize_di_node = type_di_node(cx, tcx.types.usize); - let (pointer_size, pointer_align) = cx.size_and_align_of(void_pointer_ty); - // If `usize` is not pointer-sized and -aligned then the size and alignment computations - // for the vtable as a whole would be wrong. Let's make sure this holds even on weird - // platforms. - assert_eq!(cx.size_and_align_of(tcx.types.usize), (pointer_size, pointer_align)); - - let vtable_type_name = - compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::Type); - let unique_type_id = UniqueTypeId::for_vtable_ty(tcx, ty, poly_trait_ref); - let size = pointer_size * vtable_entries.len() as u64; - - // This gets mapped to a DW_AT_containing_type attribute which allows GDB to correlate - // the vtable to the type it is for. - let vtable_holder = type_di_node(cx, ty); - - build_type_with_children( - cx, - type_map::stub( - cx, - Stub::VTableTy { vtable_holder }, - unique_type_id, - &vtable_type_name, - (size, pointer_align), - NO_SCOPE_METADATA, - DIFlags::FlagArtificial, - ), - |cx, vtable_type_di_node| { - vtable_entries - .iter() - .enumerate() - .filter_map(|(index, vtable_entry)| { - let (field_name, field_type_di_node) = match vtable_entry { - ty::VtblEntry::MetadataDropInPlace => { - ("drop_in_place".to_string(), void_pointer_type_di_node) - } - ty::VtblEntry::Method(_) => { - // Note: This code does not try to give a proper name to each method - // because their might be multiple methods with the same name - // (coming from different traits). - (format!("__method{}", index), void_pointer_type_di_node) - } - ty::VtblEntry::TraitVPtr(_) => { - (format!("__super_trait_ptr{}", index), void_pointer_type_di_node) - } - ty::VtblEntry::MetadataAlign => ("align".to_string(), usize_di_node), - ty::VtblEntry::MetadataSize => ("size".to_string(), usize_di_node), - ty::VtblEntry::Vacant => return None, - }; - - let field_offset = pointer_size * index as u64; - - Some(build_field_di_node( - cx, - vtable_type_di_node, - &field_name, - (pointer_size, pointer_align), - field_offset, - DIFlags::FlagZero, - field_type_di_node, - )) - }) - .collect() - }, - NO_GENERICS, - ) - .di_node -} - -fn vcall_visibility_metadata<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - ty: Ty<'tcx>, - trait_ref: Option>, - vtable: &'ll Value, -) { - enum VCallVisibility { - Public = 0, - LinkageUnit = 1, - TranslationUnit = 2, - } - - let Some(trait_ref) = trait_ref else { return }; - - let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); - let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); - let trait_def_id = trait_ref_self.def_id(); - let trait_vis = cx.tcx.visibility(trait_def_id); - - let cgus = cx.sess().codegen_units(); - let single_cgu = cgus == 1; - - let lto = cx.sess().lto(); - - // Since LLVM requires full LTO for the virtual function elimination optimization to apply, - // only the `Lto::Fat` cases are relevant currently. - let vcall_visibility = match (lto, trait_vis, single_cgu) { - // If there is not LTO and the visibility in public, we have to assume that the vtable can - // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. - (Lto::No | Lto::ThinLocal, Visibility::Public, _) - | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => { - VCallVisibility::Public - } - // With LTO and a quasi-public visibility, the usages of the functions of the vtable are - // all known by the `LinkageUnit`. - // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also - // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. - (Lto::Fat | Lto::Thin, Visibility::Public, _) - | ( - Lto::ThinLocal | Lto::Thin | Lto::Fat, - Visibility::Restricted(_) | Visibility::Invisible, - false, - ) => VCallVisibility::LinkageUnit, - // If there is only one CGU, private vtables can only be seen by that CGU/translation unit - // and therefore we know of all usages of functions in the vtable. - (_, Visibility::Restricted(_) | Visibility::Invisible, true) => { - VCallVisibility::TranslationUnit - } - }; - - let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); - - unsafe { - let typeid = llvm::LLVMMDStringInContext( - cx.llcx, - trait_ref_typeid.as_ptr() as *const c_char, - trait_ref_typeid.as_bytes().len() as c_uint, - ); - let v = [cx.const_usize(0), typeid]; - llvm::LLVMRustGlobalAddMetadata( - vtable, - llvm::MD_type as c_uint, - llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( - cx.llcx, - v.as_ptr(), - v.len() as c_uint, - )), - ); - let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); - let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); - llvm::LLVMGlobalSetMetadata( - vtable, - llvm::MetadataType::MD_vcall_visibility as c_uint, - vcall_visibility_metadata, - ); - } -} - -/// Creates debug information for the given vtable, which is for the -/// given type. -/// -/// Adds the created metadata nodes directly to the crate's IR. -pub fn create_vtable_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - ty: Ty<'tcx>, - poly_trait_ref: Option>, - vtable: &'ll Value, -) { - // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in - // LLVM at the moment. - if cx.sess().opts.unstable_opts.virtual_function_elimination && cx.sess().lto() == Lto::Fat { - vcall_visibility_metadata(cx, ty, poly_trait_ref, vtable); - } - - if cx.dbg_cx.is_none() { - return; - } - - // Only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo != DebugInfo::Full { - return; - } - - let vtable_name = - compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable); - let vtable_type_di_node = build_vtable_type_di_node(cx, ty, poly_trait_ref); - let linkage_name = ""; - - unsafe { - llvm::LLVMRustDIBuilderCreateStaticVariable( - DIB(cx), - NO_SCOPE_METADATA, - vtable_name.as_ptr().cast(), - vtable_name.len(), - linkage_name.as_ptr().cast(), - linkage_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - vtable_type_di_node, - true, - vtable, - None, - 0, - ); - } -} - -/// Creates an "extension" of an existing `DIScope` into another file. -pub fn extend_scope_to_file<'ll>( - cx: &CodegenCx<'ll, '_>, - scope_metadata: &'ll DIScope, - file: &SourceFile, -) -> &'ll DILexicalBlock { - let file_metadata = file_metadata(cx, file); - unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlockFile(DIB(cx), scope_metadata, file_metadata) } -} - -pub fn tuple_field_name(field_index: usize) -> Cow<'static, str> { - const TUPLE_FIELD_NAMES: [&'static str; 16] = [ - "__0", "__1", "__2", "__3", "__4", "__5", "__6", "__7", "__8", "__9", "__10", "__11", - "__12", "__13", "__14", "__15", - ]; - TUPLE_FIELD_NAMES - .get(field_index) - .map(|s| Cow::from(*s)) - .unwrap_or_else(|| Cow::from(format!("__{}", field_index))) -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs deleted file mode 100644 index d6e2c8ccdf4..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ /dev/null @@ -1,514 +0,0 @@ -use std::borrow::Cow; - -use libc::c_uint; -use rustc_codegen_ssa::debuginfo::{ - type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo, -}; -use rustc_middle::{ - bug, - ty::{ - self, - layout::{LayoutOf, TyAndLayout}, - util::Discr, - AdtDef, GeneratorSubsts, - }, -}; -use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; -use smallvec::smallvec; - -use crate::{ - common::CodegenCx, - debuginfo::{ - metadata::{ - build_field_di_node, closure_saved_names_of_captured_variables, - enums::tag_base_type, - file_metadata, generator_layout_and_saved_local_names, size_and_align_of, - type_map::{self, UniqueTypeId}, - unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA, - UNKNOWN_LINE_NUMBER, - }, - utils::DIB, - }, - llvm::{ - self, - debuginfo::{DIFile, DIFlags, DIType}, - }, -}; - -/// In CPP-like mode, we generate a union of structs for each variant and an -/// explicit discriminant field roughly equivalent to the following C/C++ code: -/// -/// ```c -/// union enum$<{fully-qualified-name}> { -/// struct {variant 0 name} { -/// -/// } variant0; -/// -/// {name} discriminant; -/// } -/// ``` -/// -/// As you can see, the type name is wrapped `enum$`. This way we can have a -/// single NatVis rule for handling all enums. -/// -/// At the LLVM IR level this looks like -/// -/// ```txt -/// DW_TAG_union_type (top-level type for enum) -/// DW_TAG_member (member for variant 1) -/// DW_TAG_member (member for variant 2) -/// DW_TAG_member (member for variant 3) -/// DW_TAG_structure_type (type of variant 1) -/// DW_TAG_structure_type (type of variant 2) -/// DW_TAG_structure_type (type of variant 3) -/// DW_TAG_enumeration_type (type of tag) -/// ``` -/// -/// The above encoding applies for enums with a direct tag. For niche-tag we have to do things -/// differently in order to allow a NatVis visualizer to extract all the information needed: -/// We generate a union of two fields, one for the dataful variant -/// and one that just points to the discriminant (which is some field within the dataful variant). -/// We also create a DW_TAG_enumeration_type DIE that contains tag values for the non-dataful -/// variants and make the discriminant field that type. We then use NatVis to render the enum type -/// correctly in Windbg/VS. This will generate debuginfo roughly equivalent to the following C: -/// -/// ```c -/// union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> { -/// struct { -/// -/// } dataful_variant; -/// enum Discriminant$ { -/// -/// } discriminant; -/// } -/// ``` -/// -/// The NatVis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>` -/// and evaluates `this.discriminant`. If the value is between the min niche and max -/// niche, then the enum is in the dataful variant and `this.dataful_variant` is -/// rendered. Otherwise, the enum is in one of the non-dataful variants. In that -/// case, we just need to render the name of the `this.discriminant` enum. -pub(super) fn build_enum_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - let enum_type = unique_type_id.expect_ty(); - let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { - bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) - }; - - let enum_type_and_layout = cx.layout_of(enum_type); - let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); - - debug_assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); - - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - type_map::Stub::Union, - unique_type_id, - &enum_type_name, - cx.size_and_align_of(enum_type), - NO_SCOPE_METADATA, - DIFlags::FlagZero, - ), - |cx, enum_type_di_node| { - match enum_type_and_layout.variants { - Variants::Single { index: variant_index } => { - if enum_adt_def.variants().is_empty() { - // Uninhabited enums have Variants::Single. We don't generate - // any members for them. - return smallvec![]; - } - - build_single_variant_union_fields( - cx, - enum_adt_def, - enum_type_and_layout, - enum_type_di_node, - variant_index, - ) - } - Variants::Multiple { - tag_encoding: TagEncoding::Direct, - ref variants, - tag_field, - .. - } => build_union_fields_for_direct_tag_enum( - cx, - enum_adt_def, - enum_type_and_layout, - enum_type_di_node, - &mut variants.indices(), - tag_field, - ), - Variants::Multiple { - tag_encoding: TagEncoding::Niche { dataful_variant, .. }, - ref variants, - tag_field, - .. - } => build_union_fields_for_niche_tag_enum( - cx, - enum_adt_def, - enum_type_and_layout, - enum_type_di_node, - dataful_variant, - &mut variants.indices(), - tag_field, - ), - } - }, - NO_GENERICS, - ) -} - -/// A generator debuginfo node looks the same as a that of an enum type. -/// -/// See [build_enum_type_di_node] for more information. -pub(super) fn build_generator_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - let generator_type = unique_type_id.expect_ty(); - let generator_type_and_layout = cx.layout_of(generator_type); - let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); - - debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); - - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - type_map::Stub::Union, - unique_type_id, - &generator_type_name, - size_and_align_of(generator_type_and_layout), - NO_SCOPE_METADATA, - DIFlags::FlagZero, - ), - |cx, generator_type_di_node| match generator_type_and_layout.variants { - Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => { - build_union_fields_for_direct_tag_generator( - cx, - generator_type_and_layout, - generator_type_di_node, - ) - } - Variants::Single { .. } - | Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => { - bug!( - "Encountered generator with non-direct-tag layout: {:?}", - generator_type_and_layout - ) - } - }, - NO_GENERICS, - ) -} - -fn build_single_variant_union_fields<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_adt_def: AdtDef<'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, - enum_type_di_node: &'ll DIType, - variant_index: VariantIdx, -) -> SmallVec<&'ll DIType> { - let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); - let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( - cx, - enum_type_and_layout.ty, - enum_type_di_node, - variant_index, - enum_adt_def.variant(variant_index), - variant_layout, - ); - - // NOTE: The field name of the union is the same as the variant name, not "variant0". - let variant_name = enum_adt_def.variant(variant_index).name.as_str(); - - smallvec![build_field_di_node( - cx, - enum_type_di_node, - variant_name, - // NOTE: We use the size and align of the entire type, not from variant_layout - // since the later is sometimes smaller (if it has fewer fields). - size_and_align_of(enum_type_and_layout), - Size::ZERO, - DIFlags::FlagZero, - variant_struct_type_di_node, - )] -} - -fn build_union_fields_for_direct_tag_enum<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_adt_def: AdtDef<'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, - enum_type_di_node: &'ll DIType, - variant_indices: &mut dyn Iterator, - tag_field: usize, -) -> SmallVec<&'ll DIType> { - let variant_field_infos: SmallVec> = variant_indices - .map(|variant_index| { - let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); - - VariantFieldInfo { - variant_index, - variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( - cx, - enum_type_and_layout.ty, - enum_type_di_node, - variant_index, - enum_adt_def.variant(variant_index), - variant_layout, - ), - source_info: None, - } - }) - .collect(); - - let discr_type_name = cx.tcx.item_name(enum_adt_def.did()); - let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); - let discr_type_di_node = super::build_enumeration_type_di_node( - cx, - discr_type_name.as_str(), - tag_base_type, - &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { - (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) - }), - enum_type_di_node, - ); - - build_union_fields_for_direct_tag_enum_or_generator( - cx, - enum_type_and_layout, - enum_type_di_node, - &variant_field_infos, - discr_type_di_node, - tag_field, - ) -} - -fn build_union_fields_for_niche_tag_enum<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_adt_def: AdtDef<'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, - enum_type_di_node: &'ll DIType, - dataful_variant_index: VariantIdx, - variant_indices: &mut dyn Iterator, - tag_field: usize, -) -> SmallVec<&'ll DIType> { - let dataful_variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( - cx, - enum_type_and_layout.ty, - enum_type_di_node, - dataful_variant_index, - &enum_adt_def.variant(dataful_variant_index), - enum_type_and_layout.for_variant(cx, dataful_variant_index), - ); - - let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); - // Create an DW_TAG_enumerator for each variant except the dataful one. - let discr_type_di_node = super::build_enumeration_type_di_node( - cx, - "Discriminant$", - tag_base_type, - &mut variant_indices.filter_map(|variant_index| { - if let Some(discr_val) = - super::compute_discriminant_value(cx, enum_type_and_layout, variant_index) - { - let discr = Discr { val: discr_val as u128, ty: tag_base_type }; - let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); - Some((discr, variant_name)) - } else { - debug_assert_eq!(variant_index, dataful_variant_index); - None - } - }), - enum_type_di_node, - ); - - smallvec![ - build_field_di_node( - cx, - enum_type_di_node, - "dataful_variant", - size_and_align_of(enum_type_and_layout), - Size::ZERO, - DIFlags::FlagZero, - dataful_variant_struct_type_di_node, - ), - build_field_di_node( - cx, - enum_type_di_node, - "discriminant", - cx.size_and_align_of(tag_base_type), - enum_type_and_layout.fields.offset(tag_field), - DIFlags::FlagZero, - discr_type_di_node, - ), - ] -} - -fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - generator_type_and_layout: TyAndLayout<'tcx>, - generator_type_di_node: &'ll DIType, -) -> SmallVec<&'ll DIType> { - let Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } = generator_type_and_layout.variants else { - bug!("This function only supports layouts with directly encoded tags.") - }; - - let (generator_def_id, generator_substs) = match generator_type_and_layout.ty.kind() { - &ty::Generator(def_id, substs, _) => (def_id, substs.as_generator()), - _ => unreachable!(), - }; - - let (generator_layout, state_specific_upvar_names) = - generator_layout_and_saved_local_names(cx.tcx, generator_def_id); - - let common_upvar_names = closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); - let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx); - - // Build the type node for each field. - let variant_field_infos: SmallVec> = variant_range - .map(|variant_index| { - let variant_struct_type_di_node = super::build_generator_variant_struct_type_di_node( - cx, - variant_index, - generator_type_and_layout, - generator_type_di_node, - generator_layout, - &state_specific_upvar_names, - &common_upvar_names, - ); - - let span = generator_layout.variant_source_info[variant_index].span; - let source_info = if !span.is_dummy() { - let loc = cx.lookup_debug_loc(span.lo()); - Some((file_metadata(cx, &loc.file), loc.line as c_uint)) - } else { - None - }; - - VariantFieldInfo { variant_index, variant_struct_type_di_node, source_info } - }) - .collect(); - - let tag_base_type = tag_base_type(cx, generator_type_and_layout); - let discr_type_name = "Discriminant$"; - let discr_type_di_node = super::build_enumeration_type_di_node( - cx, - discr_type_name, - tag_base_type, - &mut generator_substs - .discriminants(generator_def_id, cx.tcx) - .map(|(variant_index, discr)| (discr, GeneratorSubsts::variant_name(variant_index))), - generator_type_di_node, - ); - - build_union_fields_for_direct_tag_enum_or_generator( - cx, - generator_type_and_layout, - generator_type_di_node, - &variant_field_infos[..], - discr_type_di_node, - tag_field, - ) -} - -/// This is a helper function shared between enums and generators that makes sure fields have the -/// expect names. -fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, - enum_type_di_node: &'ll DIType, - variant_field_infos: &[VariantFieldInfo<'ll>], - discr_type_di_node: &'ll DIType, - tag_field: usize, -) -> SmallVec<&'ll DIType> { - let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1); - - // We create a field in the union for each variant ... - unions_fields.extend(variant_field_infos.into_iter().map(|variant_member_info| { - let (file_di_node, line_number) = variant_member_info - .source_info - .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); - - let field_name = variant_union_field_name(variant_member_info.variant_index); - let (size, align) = size_and_align_of(enum_type_and_layout); - - // We use LLVMRustDIBuilderCreateMemberType() member type directly because - // the build_field_di_node() function does not support specifying a source location, - // which is something that we don't do anywhere else. - unsafe { - llvm::LLVMRustDIBuilderCreateMemberType( - DIB(cx), - enum_type_di_node, - field_name.as_ptr().cast(), - field_name.len(), - file_di_node, - line_number, - // NOTE: We use the size and align of the entire type, not from variant_layout - // since the later is sometimes smaller (if it has fewer fields). - size.bits(), - align.bits() as u32, - // Union fields are always at offset zero - Size::ZERO.bits(), - DIFlags::FlagZero, - variant_member_info.variant_struct_type_di_node, - ) - } - })); - - debug_assert_eq!( - cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), - cx.size_and_align_of(super::tag_base_type(cx, enum_type_and_layout)) - ); - - // ... and a field for the discriminant. - unions_fields.push(build_field_di_node( - cx, - enum_type_di_node, - "discriminant", - cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), - enum_type_and_layout.fields.offset(tag_field), - DIFlags::FlagZero, - discr_type_di_node, - )); - - unions_fields -} - -/// Information about a single field of the top-level DW_TAG_union_type. -struct VariantFieldInfo<'ll> { - variant_index: VariantIdx, - variant_struct_type_di_node: &'ll DIType, - source_info: Option<(&'ll DIFile, c_uint)>, -} - -fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> { - const PRE_ALLOCATED: [&str; 16] = [ - "variant0", - "variant1", - "variant2", - "variant3", - "variant4", - "variant5", - "variant6", - "variant7", - "variant8", - "variant9", - "variant10", - "variant11", - "variant12", - "variant13", - "variant14", - "variant15", - ]; - - PRE_ALLOCATED - .get(variant_index.as_usize()) - .map(|&s| Cow::from(s)) - .unwrap_or_else(|| format!("variant{}", variant_index.as_usize()).into()) -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs deleted file mode 100644 index 73e01d0453b..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ /dev/null @@ -1,437 +0,0 @@ -use rustc_codegen_ssa::debuginfo::{ - type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}, - wants_c_like_enum_debuginfo, -}; -use rustc_hir::def::CtorKind; -use rustc_index::vec::IndexVec; -use rustc_middle::{ - bug, - mir::{Field, GeneratorLayout, GeneratorSavedLocal}, - ty::{ - self, - layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, - util::Discr, - AdtDef, GeneratorSubsts, Ty, VariantDef, - }, -}; -use rustc_span::Symbol; -use rustc_target::abi::{HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants}; -use std::borrow::Cow; - -use crate::{ - common::CodegenCx, - debuginfo::{ - metadata::{ - build_field_di_node, build_generic_type_param_di_nodes, type_di_node, - type_map::{self, Stub}, - unknown_file_metadata, UNKNOWN_LINE_NUMBER, - }, - utils::{create_DIArray, get_namespace_for_item, DIB}, - }, - llvm::{ - self, - debuginfo::{DIFlags, DIType}, - }, -}; - -use super::{ - size_and_align_of, - type_map::{DINodeCreationResult, UniqueTypeId}, - SmallVec, -}; - -mod cpp_like; -mod native; - -pub(super) fn build_enum_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - let enum_type = unique_type_id.expect_ty(); - let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { - bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) - }; - - let enum_type_and_layout = cx.layout_of(enum_type); - - if wants_c_like_enum_debuginfo(enum_type_and_layout) { - return build_c_style_enum_di_node(cx, enum_adt_def, enum_type_and_layout); - } - - if cpp_like_debuginfo(cx.tcx) { - cpp_like::build_enum_type_di_node(cx, unique_type_id) - } else { - native::build_enum_type_di_node(cx, unique_type_id) - } -} - -pub(super) fn build_generator_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - if cpp_like_debuginfo(cx.tcx) { - cpp_like::build_generator_di_node(cx, unique_type_id) - } else { - native::build_generator_di_node(cx, unique_type_id) - } -} - -/// Build the debuginfo node for a C-style enum, i.e. an enum the variants of which have no fields. -/// -/// The resulting debuginfo will be a DW_TAG_enumeration_type. -fn build_c_style_enum_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_adt_def: AdtDef<'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, -) -> DINodeCreationResult<'ll> { - let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); - DINodeCreationResult { - di_node: build_enumeration_type_di_node( - cx, - &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false), - tag_base_type(cx, enum_type_and_layout), - &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { - (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) - }), - containing_scope, - ), - already_stored_in_typemap: false, - } -} - -/// Extract the type with which we want to describe the tag of the given enum or generator. -fn tag_base_type<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, -) -> Ty<'tcx> { - debug_assert!(match enum_type_and_layout.ty.kind() { - ty::Generator(..) => true, - ty::Adt(adt_def, _) => adt_def.is_enum(), - _ => false, - }); - - match enum_type_and_layout.layout.variants() { - // A single-variant enum has no discriminant. - Variants::Single { .. } => { - bug!("tag_base_type() called for enum without tag: {:?}", enum_type_and_layout) - } - - Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } => { - // Niche tags are always normalized to unsized integers of the correct size. - match tag.primitive() { - Primitive::Int(t, _) => t, - Primitive::F32 => Integer::I32, - Primitive::F64 => Integer::I64, - Primitive::Pointer => { - // If the niche is the NULL value of a reference, then `discr_enum_ty` will be - // a RawPtr. CodeView doesn't know what to do with enums whose base type is a - // pointer so we fix this up to just be `usize`. - // DWARF might be able to deal with this but with an integer type we are on - // the safe side there too. - cx.data_layout().ptr_sized_integer() - } - } - .to_ty(cx.tcx, false) - } - - Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => { - // Direct tags preserve the sign. - tag.primitive().to_ty(cx.tcx) - } - } -} - -/// Build a DW_TAG_enumeration_type debuginfo node, with the given base type and variants. -/// This is a helper function and does not register anything in the type map by itself. -/// -/// `variants` is an iterator of (discr-value, variant-name). -/// -// NOTE: Handling of discriminant values is somewhat inconsistent. They can appear as u128, -// u64, and i64. Here everything gets mapped to i64 because that's what LLVM's API expects. -fn build_enumeration_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - type_name: &str, - base_type: Ty<'tcx>, - variants: &mut dyn Iterator, Cow<'tcx, str>)>, - containing_scope: &'ll DIType, -) -> &'ll DIType { - let is_unsigned = match base_type.kind() { - ty::Int(_) => false, - ty::Uint(_) => true, - _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."), - }; - - let enumerator_di_nodes: SmallVec> = variants - .map(|(discr, variant_name)| { - unsafe { - Some(llvm::LLVMRustDIBuilderCreateEnumerator( - DIB(cx), - variant_name.as_ptr().cast(), - variant_name.len(), - // FIXME: what if enumeration has i128 discriminant? - discr.val as i64, - is_unsigned, - )) - } - }) - .collect(); - - let (size, align) = cx.size_and_align_of(base_type); - - unsafe { - llvm::LLVMRustDIBuilderCreateEnumerationType( - DIB(cx), - containing_scope, - type_name.as_ptr().cast(), - type_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - size.bits(), - align.bits() as u32, - create_DIArray(DIB(cx), &enumerator_di_nodes[..]), - type_di_node(cx, base_type), - true, - ) - } -} - -/// Build the debuginfo node for the struct type describing a single variant of an enum. -/// -/// ```txt -/// DW_TAG_structure_type (top-level type for enum) -/// DW_TAG_variant_part (variant part) -/// DW_AT_discr (reference to discriminant DW_TAG_member) -/// DW_TAG_member (discriminant member) -/// DW_TAG_variant (variant 1) -/// DW_TAG_variant (variant 2) -/// DW_TAG_variant (variant 3) -/// ---> DW_TAG_structure_type (type of variant 1) -/// ---> DW_TAG_structure_type (type of variant 2) -/// ---> DW_TAG_structure_type (type of variant 3) -/// ``` -/// -/// In CPP-like mode, we have the exact same descriptions for each variant too: -/// -/// ```txt -/// DW_TAG_union_type (top-level type for enum) -/// DW_TAG_member (member for variant 1) -/// DW_TAG_member (member for variant 2) -/// DW_TAG_member (member for variant 3) -/// ---> DW_TAG_structure_type (type of variant 1) -/// ---> DW_TAG_structure_type (type of variant 2) -/// ---> DW_TAG_structure_type (type of variant 3) -/// DW_TAG_enumeration_type (type of tag) -/// ``` -/// -/// The node looks like: -/// -/// ```txt -/// DW_TAG_structure_type -/// DW_AT_name -/// DW_AT_byte_size 0x00000010 -/// DW_AT_alignment 0x00000008 -/// DW_TAG_member -/// DW_AT_name -/// DW_AT_type <0x0000018e> -/// DW_AT_alignment 0x00000004 -/// DW_AT_data_member_location 4 -/// DW_TAG_member -/// DW_AT_name -/// DW_AT_type <0x00000195> -/// DW_AT_alignment 0x00000008 -/// DW_AT_data_member_location 8 -/// ... -/// ``` -/// -/// The type of a variant is always a struct type with the name of the variant -/// and a DW_TAG_member for each field (but not the discriminant). -fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_type: Ty<'tcx>, - enum_type_di_node: &'ll DIType, - variant_index: VariantIdx, - variant_def: &VariantDef, - variant_layout: TyAndLayout<'tcx>, -) -> &'ll DIType { - debug_assert_eq!(variant_layout.ty, enum_type); - - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Struct, - UniqueTypeId::for_enum_variant_struct_type(cx.tcx, enum_type, variant_index), - variant_def.name.as_str(), - // NOTE: We use size and align of enum_type, not from variant_layout: - cx.size_and_align_of(enum_type), - Some(enum_type_di_node), - DIFlags::FlagZero, - ), - |cx, struct_type_di_node| { - (0..variant_layout.fields.count()) - .map(|field_index| { - let field_name = if variant_def.ctor_kind != CtorKind::Fn { - // Fields have names - Cow::from(variant_def.fields[field_index].name.as_str()) - } else { - // Tuple-like - super::tuple_field_name(field_index) - }; - - let field_layout = variant_layout.field(cx, field_index); - - build_field_di_node( - cx, - struct_type_di_node, - &field_name, - (field_layout.size, field_layout.align.abi), - variant_layout.fields.offset(field_index), - DIFlags::FlagZero, - type_di_node(cx, field_layout.ty), - ) - }) - .collect() - }, - |cx| build_generic_type_param_di_nodes(cx, enum_type), - ) - .di_node -} - -/// Build the struct type for describing a single generator state. -/// See [build_generator_variant_struct_type_di_node]. -/// -/// ```txt -/// -/// DW_TAG_structure_type (top-level type for enum) -/// DW_TAG_variant_part (variant part) -/// DW_AT_discr (reference to discriminant DW_TAG_member) -/// DW_TAG_member (discriminant member) -/// DW_TAG_variant (variant 1) -/// DW_TAG_variant (variant 2) -/// DW_TAG_variant (variant 3) -/// ---> DW_TAG_structure_type (type of variant 1) -/// ---> DW_TAG_structure_type (type of variant 2) -/// ---> DW_TAG_structure_type (type of variant 3) -/// -/// ``` -pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - variant_index: VariantIdx, - generator_type_and_layout: TyAndLayout<'tcx>, - generator_type_di_node: &'ll DIType, - generator_layout: &GeneratorLayout<'tcx>, - state_specific_upvar_names: &IndexVec>, - common_upvar_names: &[String], -) -> &'ll DIType { - let variant_name = GeneratorSubsts::variant_name(variant_index); - let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( - cx.tcx, - generator_type_and_layout.ty, - variant_index, - ); - - let variant_layout = generator_type_and_layout.for_variant(cx, variant_index); - - let generator_substs = match generator_type_and_layout.ty.kind() { - ty::Generator(_, substs, _) => substs.as_generator(), - _ => unreachable!(), - }; - - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Struct, - unique_type_id, - &variant_name, - size_and_align_of(generator_type_and_layout), - Some(generator_type_di_node), - DIFlags::FlagZero, - ), - |cx, variant_struct_type_di_node| { - // Fields that just belong to this variant/state - let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) - .map(|field_index| { - let generator_saved_local = generator_layout.variant_fields[variant_index] - [Field::from_usize(field_index)]; - let field_name_maybe = state_specific_upvar_names[generator_saved_local]; - let field_name = field_name_maybe - .as_ref() - .map(|s| Cow::from(s.as_str())) - .unwrap_or_else(|| super::tuple_field_name(field_index)); - - let field_type = variant_layout.field(cx, field_index).ty; - - build_field_di_node( - cx, - variant_struct_type_di_node, - &field_name, - cx.size_and_align_of(field_type), - variant_layout.fields.offset(field_index), - DIFlags::FlagZero, - type_di_node(cx, field_type), - ) - }) - .collect(); - - // Fields that are common to all states - let common_fields: SmallVec<_> = generator_substs - .prefix_tys() - .enumerate() - .map(|(index, upvar_ty)| { - build_field_di_node( - cx, - variant_struct_type_di_node, - &common_upvar_names[index], - cx.size_and_align_of(upvar_ty), - generator_type_and_layout.fields.offset(index), - DIFlags::FlagZero, - type_di_node(cx, upvar_ty), - ) - }) - .collect(); - - state_specific_fields.into_iter().chain(common_fields.into_iter()).collect() - }, - |cx| build_generic_type_param_di_nodes(cx, generator_type_and_layout.ty), - ) - .di_node -} - -/// Returns the discriminant value corresponding to the variant index. -/// -/// Will return `None` if there is less than two variants (because then the enum won't have) -/// a tag, and if this is the dataful variant of a niche-layout enum (because then there is no -/// single discriminant value). -fn compute_discriminant_value<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, - variant_index: VariantIdx, -) -> Option { - match enum_type_and_layout.layout.variants() { - &Variants::Single { .. } => None, - &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => Some( - enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val - as u64, - ), - &Variants::Multiple { - tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, - tag, - .. - } => { - if variant_index == dataful_variant { - None - } else { - let value = (variant_index.as_u32() as u128) - .wrapping_sub(niche_variants.start().as_u32() as u128) - .wrapping_add(niche_start); - let value = tag.size(cx).truncate(value); - // NOTE(eddyb) do *NOT* remove this assert, until - // we pass the full 128-bit value to LLVM, otherwise - // truncation will be silent and remain undetected. - assert_eq!(value as u64 as u128, value); - Some(value as u64) - } - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs deleted file mode 100644 index f1935e0ec31..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ /dev/null @@ -1,441 +0,0 @@ -use std::borrow::Cow; - -use crate::{ - common::CodegenCx, - debuginfo::{ - metadata::{ - closure_saved_names_of_captured_variables, - enums::tag_base_type, - file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, - type_map::{self, Stub, StubInfo, UniqueTypeId}, - unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, - UNKNOWN_LINE_NUMBER, - }, - utils::{create_DIArray, get_namespace_for_item, DIB}, - }, - llvm::{ - self, - debuginfo::{DIFile, DIFlags, DIType}, - }, -}; -use libc::c_uint; -use rustc_codegen_ssa::{ - debuginfo::{type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo}, - traits::ConstMethods, -}; -use rustc_middle::{ - bug, - ty::{ - self, - layout::{LayoutOf, TyAndLayout}, - }, -}; -use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; -use smallvec::smallvec; - -/// Build the debuginfo node for an enum type. The listing below shows how such a -/// type looks like at the LLVM IR/DWARF level. It is a `DW_TAG_structure_type` -/// with a single `DW_TAG_variant_part` that in turn contains a `DW_TAG_variant` -/// for each variant of the enum. The variant-part also contains a single member -/// describing the discriminant, and a nested struct type for each of the variants. -/// -/// ```txt -/// ---> DW_TAG_structure_type (top-level type for enum) -/// DW_TAG_variant_part (variant part) -/// DW_AT_discr (reference to discriminant DW_TAG_member) -/// DW_TAG_member (discriminant member) -/// DW_TAG_variant (variant 1) -/// DW_TAG_variant (variant 2) -/// DW_TAG_variant (variant 3) -/// DW_TAG_structure_type (type of variant 1) -/// DW_TAG_structure_type (type of variant 2) -/// DW_TAG_structure_type (type of variant 3) -/// ``` -pub(super) fn build_enum_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - let enum_type = unique_type_id.expect_ty(); - let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { - bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) - }; - - let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); - let enum_type_and_layout = cx.layout_of(enum_type); - let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); - - debug_assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); - - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Struct, - unique_type_id, - &enum_type_name, - size_and_align_of(enum_type_and_layout), - Some(containing_scope), - DIFlags::FlagZero, - ), - |cx, enum_type_di_node| { - // Build the struct type for each variant. These will be referenced by the - // DW_TAG_variant DIEs inside of the DW_TAG_variant_part DIE. - // We also called the names for the corresponding DW_TAG_variant DIEs here. - let variant_member_infos: SmallVec<_> = enum_adt_def - .variant_range() - .map(|variant_index| VariantMemberInfo { - variant_index, - variant_name: Cow::from(enum_adt_def.variant(variant_index).name.as_str()), - variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( - cx, - enum_type, - enum_type_di_node, - variant_index, - enum_adt_def.variant(variant_index), - enum_type_and_layout.for_variant(cx, variant_index), - ), - source_info: None, - }) - .collect(); - - smallvec![build_enum_variant_part_di_node( - cx, - enum_type_and_layout, - enum_type_di_node, - &variant_member_infos[..], - )] - }, - // We don't seem to be emitting generic args on the enum type, it seems. Rather - // they get attached to the struct type of each variant. - NO_GENERICS, - ) -} - -/// Build the debuginfo node for a generator environment. It looks the same as the debuginfo for -/// an enum. See [build_enum_type_di_node] for more information. -/// -/// ```txt -/// -/// ---> DW_TAG_structure_type (top-level type for the generator) -/// DW_TAG_variant_part (variant part) -/// DW_AT_discr (reference to discriminant DW_TAG_member) -/// DW_TAG_member (discriminant member) -/// DW_TAG_variant (variant 1) -/// DW_TAG_variant (variant 2) -/// DW_TAG_variant (variant 3) -/// DW_TAG_structure_type (type of variant 1) -/// DW_TAG_structure_type (type of variant 2) -/// DW_TAG_structure_type (type of variant 3) -/// -/// ``` -pub(super) fn build_generator_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, -) -> DINodeCreationResult<'ll> { - let generator_type = unique_type_id.expect_ty(); - let &ty::Generator(generator_def_id, _, _ ) = generator_type.kind() else { - bug!("build_generator_di_node() called with non-generator type: `{:?}`", generator_type) - }; - - let containing_scope = get_namespace_for_item(cx, generator_def_id); - let generator_type_and_layout = cx.layout_of(generator_type); - - debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); - - let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); - - type_map::build_type_with_children( - cx, - type_map::stub( - cx, - Stub::Struct, - unique_type_id, - &generator_type_name, - size_and_align_of(generator_type_and_layout), - Some(containing_scope), - DIFlags::FlagZero, - ), - |cx, generator_type_di_node| { - let (generator_layout, state_specific_upvar_names) = - generator_layout_and_saved_local_names(cx.tcx, generator_def_id); - - let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = generator_type_and_layout.variants else { - bug!( - "Encountered generator with non-direct-tag layout: {:?}", - generator_type_and_layout - ) - }; - - let common_upvar_names = - closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); - - // Build variant struct types - let variant_struct_type_di_nodes: SmallVec<_> = variants - .indices() - .map(|variant_index| { - // FIXME: This is problematic because just a number is not a valid identifier. - // GeneratorSubsts::variant_name(variant_index), would be consistent - // with enums? - let variant_name = format!("{}", variant_index.as_usize()).into(); - - let span = generator_layout.variant_source_info[variant_index].span; - let source_info = if !span.is_dummy() { - let loc = cx.lookup_debug_loc(span.lo()); - Some((file_metadata(cx, &loc.file), loc.line)) - } else { - None - }; - - VariantMemberInfo { - variant_index, - variant_name, - variant_struct_type_di_node: - super::build_generator_variant_struct_type_di_node( - cx, - variant_index, - generator_type_and_layout, - generator_type_di_node, - generator_layout, - &state_specific_upvar_names, - &common_upvar_names, - ), - source_info, - } - }) - .collect(); - - smallvec![build_enum_variant_part_di_node( - cx, - generator_type_and_layout, - generator_type_di_node, - &variant_struct_type_di_nodes[..], - )] - }, - // We don't seem to be emitting generic args on the generator type, it seems. Rather - // they get attached to the struct type of each variant. - NO_GENERICS, - ) -} - -/// Builds the DW_TAG_variant_part of an enum or generator debuginfo node: -/// -/// ```txt -/// DW_TAG_structure_type (top-level type for enum) -/// ---> DW_TAG_variant_part (variant part) -/// DW_AT_discr (reference to discriminant DW_TAG_member) -/// DW_TAG_member (discriminant member) -/// DW_TAG_variant (variant 1) -/// DW_TAG_variant (variant 2) -/// DW_TAG_variant (variant 3) -/// DW_TAG_structure_type (type of variant 1) -/// DW_TAG_structure_type (type of variant 2) -/// DW_TAG_structure_type (type of variant 3) -/// ``` -fn build_enum_variant_part_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, - enum_type_di_node: &'ll DIType, - variant_member_infos: &[VariantMemberInfo<'_, 'll>], -) -> &'ll DIType { - let tag_member_di_node = - build_discr_member_di_node(cx, enum_type_and_layout, enum_type_di_node); - - let variant_part_unique_type_id = - UniqueTypeId::for_enum_variant_part(cx.tcx, enum_type_and_layout.ty); - - let stub = StubInfo::new( - cx, - variant_part_unique_type_id, - |cx, variant_part_unique_type_id_str| unsafe { - let variant_part_name = ""; - llvm::LLVMRustDIBuilderCreateVariantPart( - DIB(cx), - enum_type_di_node, - variant_part_name.as_ptr().cast(), - variant_part_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - enum_type_and_layout.size.bits(), - enum_type_and_layout.align.abi.bits() as u32, - DIFlags::FlagZero, - tag_member_di_node, - create_DIArray(DIB(cx), &[]), - variant_part_unique_type_id_str.as_ptr().cast(), - variant_part_unique_type_id_str.len(), - ) - }, - ); - - type_map::build_type_with_children( - cx, - stub, - |cx, variant_part_di_node| { - variant_member_infos - .iter() - .map(|variant_member_info| { - build_enum_variant_member_di_node( - cx, - enum_type_and_layout, - variant_part_di_node, - variant_member_info, - ) - }) - .collect() - }, - NO_GENERICS, - ) - .di_node -} - -/// Builds the DW_TAG_member describing where we can find the tag of an enum. -/// Returns `None` if the enum does not have a tag. -/// -/// ```txt -/// -/// DW_TAG_structure_type (top-level type for enum) -/// DW_TAG_variant_part (variant part) -/// DW_AT_discr (reference to discriminant DW_TAG_member) -/// ---> DW_TAG_member (discriminant member) -/// DW_TAG_variant (variant 1) -/// DW_TAG_variant (variant 2) -/// DW_TAG_variant (variant 3) -/// DW_TAG_structure_type (type of variant 1) -/// DW_TAG_structure_type (type of variant 2) -/// DW_TAG_structure_type (type of variant 3) -/// -/// ``` -fn build_discr_member_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_or_generator_type_and_layout: TyAndLayout<'tcx>, - enum_or_generator_type_di_node: &'ll DIType, -) -> Option<&'ll DIType> { - let tag_name = match enum_or_generator_type_and_layout.ty.kind() { - ty::Generator(..) => "__state", - _ => "", - }; - - // NOTE: This is actually wrong. This will become a member of - // of the DW_TAG_variant_part. But, due to LLVM's API, that - // can only be constructed with this DW_TAG_member already in created. - // In LLVM IR the wrong scope will be listed but when DWARF is - // generated from it, the DW_TAG_member will be a child the - // DW_TAG_variant_part. - let containing_scope = enum_or_generator_type_di_node; - - match enum_or_generator_type_and_layout.layout.variants() { - // A single-variant enum has no discriminant. - &Variants::Single { .. } => None, - - &Variants::Multiple { tag_field, .. } => { - let tag_base_type = tag_base_type(cx, enum_or_generator_type_and_layout); - let (size, align) = cx.size_and_align_of(tag_base_type); - - unsafe { - Some(llvm::LLVMRustDIBuilderCreateMemberType( - DIB(cx), - containing_scope, - tag_name.as_ptr().cast(), - tag_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - size.bits(), - align.bits() as u32, - enum_or_generator_type_and_layout.fields.offset(tag_field).bits(), - DIFlags::FlagArtificial, - type_di_node(cx, tag_base_type), - )) - } - } - } -} - -/// Build the debuginfo node for `DW_TAG_variant`: -/// -/// ```txt -/// DW_TAG_structure_type (top-level type for enum) -/// DW_TAG_variant_part (variant part) -/// DW_AT_discr (reference to discriminant DW_TAG_member) -/// DW_TAG_member (discriminant member) -/// ---> DW_TAG_variant (variant 1) -/// ---> DW_TAG_variant (variant 2) -/// ---> DW_TAG_variant (variant 3) -/// DW_TAG_structure_type (type of variant 1) -/// DW_TAG_structure_type (type of variant 2) -/// DW_TAG_structure_type (type of variant 3) -/// ``` -/// -/// This node looks like: -/// -/// ```txt -/// DW_TAG_variant -/// DW_AT_discr_value 0 -/// DW_TAG_member -/// DW_AT_name None -/// DW_AT_type <0x000002a1> -/// DW_AT_alignment 0x00000002 -/// DW_AT_data_member_location 0 -/// ``` -/// -/// The DW_AT_discr_value is optional, and is omitted if -/// - This is the only variant of a univariant enum (i.e. their is no discriminant) -/// - This is the "dataful" variant of a niche-layout enum -/// (where only the other variants are identified by a single value) -/// -/// There is only ever a single member, the type of which is a struct that describes the -/// fields of the variant (excluding the discriminant). The name of the member is the name -/// of the variant as given in the source code. The DW_AT_data_member_location is always -/// zero. -/// -/// Note that the LLVM DIBuilder API is a bit unintuitive here. The DW_TAG_variant subtree -/// (including the DW_TAG_member) is built by a single call to -/// `LLVMRustDIBuilderCreateVariantMemberType()`. -fn build_enum_variant_member_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, - variant_part_di_node: &'ll DIType, - variant_member_info: &VariantMemberInfo<'_, 'll>, -) -> &'ll DIType { - let variant_index = variant_member_info.variant_index; - let discr_value = super::compute_discriminant_value(cx, enum_type_and_layout, variant_index); - - let (file_di_node, line_number) = variant_member_info - .source_info - .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); - - unsafe { - llvm::LLVMRustDIBuilderCreateVariantMemberType( - DIB(cx), - variant_part_di_node, - variant_member_info.variant_name.as_ptr().cast(), - variant_member_info.variant_name.len(), - file_di_node, - line_number, - enum_type_and_layout.size.bits(), - enum_type_and_layout.align.abi.bits() as u32, - Size::ZERO.bits(), - discr_value.map(|v| cx.const_u64(v)), - DIFlags::FlagZero, - variant_member_info.variant_struct_type_di_node, - ) - } -} - -/// Information needed for building a `DW_TAG_variant`: -/// -/// ```txt -/// DW_TAG_structure_type (top-level type for enum) -/// DW_TAG_variant_part (variant part) -/// DW_AT_discr (reference to discriminant DW_TAG_member) -/// DW_TAG_member (discriminant member) -/// ---> DW_TAG_variant (variant 1) -/// ---> DW_TAG_variant (variant 2) -/// ---> DW_TAG_variant (variant 3) -/// DW_TAG_structure_type (type of variant 1) -/// DW_TAG_structure_type (type of variant 2) -/// DW_TAG_structure_type (type of variant 3) -struct VariantMemberInfo<'a, 'll> { - variant_index: VariantIdx, - variant_name: Cow<'a, str>, - variant_struct_type_di_node: &'ll DIType, - source_info: Option<(&'ll DIFile, c_uint)>, -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs deleted file mode 100644 index ce2f419c4ac..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ /dev/null @@ -1,267 +0,0 @@ -use std::cell::RefCell; - -use rustc_data_structures::{ - fingerprint::Fingerprint, - fx::FxHashMap, - stable_hasher::{HashStable, StableHasher}, -}; -use rustc_middle::{ - bug, - ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}, -}; -use rustc_target::abi::{Align, Size, VariantIdx}; - -use crate::{ - common::CodegenCx, - debuginfo::utils::{create_DIArray, debug_context, DIB}, - llvm::{ - self, - debuginfo::{DIFlags, DIScope, DIType}, - }, -}; - -use super::{unknown_file_metadata, SmallVec, UNKNOWN_LINE_NUMBER}; - -mod private { - // This type cannot be constructed outside of this module because - // it has a private field. We make use of this in order to prevent - // `UniqueTypeId` from being constructed directly, without asserting - // the preconditions. - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] - pub struct HiddenZst; -} - -/// A unique identifier for anything that we create a debuginfo node for. -/// The types it contains are expected to already be normalized (which -/// is debug_asserted in the constructors). -/// -/// Note that there are some things that only show up in debuginfo, like -/// the separate type descriptions for each enum variant. These get an ID -/// too because they have their own debuginfo node in LLVM IR. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] -pub(super) enum UniqueTypeId<'tcx> { - /// The ID of a regular type as it shows up at the language level. - Ty(Ty<'tcx>, private::HiddenZst), - /// The ID for the single DW_TAG_variant_part nested inside the top-level - /// DW_TAG_structure_type that describes enums and generators. - VariantPart(Ty<'tcx>, private::HiddenZst), - /// The ID for the artificial struct type describing a single enum variant. - VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst), - /// The ID of the artificial type we create for VTables. - VTableTy(Ty<'tcx>, Option>, private::HiddenZst), -} - -impl<'tcx> UniqueTypeId<'tcx> { - pub fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self { - debug_assert_eq!(t, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)); - UniqueTypeId::Ty(t, private::HiddenZst) - } - - pub fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self { - debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); - UniqueTypeId::VariantPart(enum_ty, private::HiddenZst) - } - - pub fn for_enum_variant_struct_type( - tcx: TyCtxt<'tcx>, - enum_ty: Ty<'tcx>, - variant_idx: VariantIdx, - ) -> Self { - debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); - UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst) - } - - pub fn for_vtable_ty( - tcx: TyCtxt<'tcx>, - self_type: Ty<'tcx>, - implemented_trait: Option>, - ) -> Self { - debug_assert_eq!( - self_type, - tcx.normalize_erasing_regions(ParamEnv::reveal_all(), self_type) - ); - debug_assert_eq!( - implemented_trait, - tcx.normalize_erasing_regions(ParamEnv::reveal_all(), implemented_trait) - ); - UniqueTypeId::VTableTy(self_type, implemented_trait, private::HiddenZst) - } - - /// Generates a string version of this [UniqueTypeId], which can be used as the `UniqueId` - /// argument of the various `LLVMRustDIBuilderCreate*Type()` methods. - /// - /// Right now this takes the form of a hex-encoded opaque hash value. - pub fn generate_unique_id_string(self, tcx: TyCtxt<'tcx>) -> String { - let mut hasher = StableHasher::new(); - tcx.with_stable_hashing_context(|mut hcx| { - hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher)) - }); - hasher.finish::().to_hex() - } - - pub fn expect_ty(self) -> Ty<'tcx> { - match self { - UniqueTypeId::Ty(ty, _) => ty, - _ => bug!("Expected `UniqueTypeId::Ty` but found `{:?}`", self), - } - } -} - -/// The `TypeMap` is where the debug context holds the type metadata nodes -/// created so far. The debuginfo nodes are identified by `UniqueTypeId`. -#[derive(Default)] -pub(crate) struct TypeMap<'ll, 'tcx> { - pub(super) unique_id_to_di_node: RefCell, &'ll DIType>>, -} - -impl<'ll, 'tcx> TypeMap<'ll, 'tcx> { - /// Adds a `UniqueTypeId` to metadata mapping to the `TypeMap`. The method will - /// fail if the mapping already exists. - pub(super) fn insert(&self, unique_type_id: UniqueTypeId<'tcx>, metadata: &'ll DIType) { - if self.unique_id_to_di_node.borrow_mut().insert(unique_type_id, metadata).is_some() { - bug!("type metadata for unique ID '{:?}' is already in the `TypeMap`!", unique_type_id); - } - } - - pub(super) fn di_node_for_unique_id( - &self, - unique_type_id: UniqueTypeId<'tcx>, - ) -> Option<&'ll DIType> { - self.unique_id_to_di_node.borrow().get(&unique_type_id).cloned() - } -} - -pub struct DINodeCreationResult<'ll> { - pub di_node: &'ll DIType, - pub already_stored_in_typemap: bool, -} - -impl<'ll> DINodeCreationResult<'ll> { - pub fn new(di_node: &'ll DIType, already_stored_in_typemap: bool) -> Self { - DINodeCreationResult { di_node, already_stored_in_typemap } - } -} - -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub enum Stub<'ll> { - Struct, - Union, - VTableTy { vtable_holder: &'ll DIType }, -} - -pub struct StubInfo<'ll, 'tcx> { - metadata: &'ll DIType, - unique_type_id: UniqueTypeId<'tcx>, -} - -impl<'ll, 'tcx> StubInfo<'ll, 'tcx> { - pub(super) fn new( - cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId<'tcx>, - build: impl FnOnce(&CodegenCx<'ll, 'tcx>, /* unique_type_id_str: */ &str) -> &'ll DIType, - ) -> StubInfo<'ll, 'tcx> { - let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); - let di_node = build(cx, &unique_type_id_str); - StubInfo { metadata: di_node, unique_type_id } - } -} - -/// Create a stub debuginfo node onto which fields and nested types can be attached. -pub(super) fn stub<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - kind: Stub<'ll>, - unique_type_id: UniqueTypeId<'tcx>, - name: &str, - (size, align): (Size, Align), - containing_scope: Option<&'ll DIScope>, - flags: DIFlags, -) -> StubInfo<'ll, 'tcx> { - let empty_array = create_DIArray(DIB(cx), &[]); - let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); - - let metadata = match kind { - Stub::Struct | Stub::VTableTy { .. } => { - let vtable_holder = match kind { - Stub::VTableTy { vtable_holder } => Some(vtable_holder), - _ => None, - }; - unsafe { - llvm::LLVMRustDIBuilderCreateStructType( - DIB(cx), - containing_scope, - name.as_ptr().cast(), - name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - size.bits(), - align.bits() as u32, - flags, - None, - empty_array, - 0, - vtable_holder, - unique_type_id_str.as_ptr().cast(), - unique_type_id_str.len(), - ) - } - } - Stub::Union => unsafe { - llvm::LLVMRustDIBuilderCreateUnionType( - DIB(cx), - containing_scope, - name.as_ptr().cast(), - name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - size.bits(), - align.bits() as u32, - flags, - Some(empty_array), - 0, - unique_type_id_str.as_ptr().cast(), - unique_type_id_str.len(), - ) - }, - }; - StubInfo { metadata, unique_type_id } -} - -/// This function enables creating debuginfo nodes that can recursively refer to themselves. -/// It will first insert the given stub into the type map and only then execute the `members` -/// and `generics` closures passed in. These closures have access to the stub so they can -/// directly attach fields to them. If the type of a field transitively refers back -/// to the type currently being built, the stub will already be found in the type map, -/// which effectively breaks the recursion cycle. -pub(super) fn build_type_with_children<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - stub_info: StubInfo<'ll, 'tcx>, - members: impl FnOnce(&CodegenCx<'ll, 'tcx>, &'ll DIType) -> SmallVec<&'ll DIType>, - generics: impl FnOnce(&CodegenCx<'ll, 'tcx>) -> SmallVec<&'ll DIType>, -) -> DINodeCreationResult<'ll> { - debug_assert_eq!( - debug_context(cx).type_map.di_node_for_unique_id(stub_info.unique_type_id), - None - ); - - debug_context(cx).type_map.insert(stub_info.unique_type_id, stub_info.metadata); - - let members: SmallVec<_> = - members(cx, stub_info.metadata).into_iter().map(|node| Some(node)).collect(); - let generics: SmallVec> = - generics(cx).into_iter().map(|node| Some(node)).collect(); - - if !(members.is_empty() && generics.is_empty()) { - unsafe { - let members_array = create_DIArray(DIB(cx), &members[..]); - let generics_array = create_DIArray(DIB(cx), &generics[..]); - llvm::LLVMRustDICompositeTypeReplaceArrays( - DIB(cx), - stub_info.metadata, - Some(members_array), - Some(generics_array), - ); - } - } - - DINodeCreationResult { di_node: stub_info.metadata, already_stored_in_typemap: true } -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs deleted file mode 100644 index cf591295b84..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ /dev/null @@ -1,614 +0,0 @@ -#![doc = include_str!("doc.md")] - -use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; - -use self::metadata::{file_metadata, type_di_node}; -use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER}; -use self::namespace::mangled_name_of_instance; -use self::utils::{create_DIArray, is_node_local_to_unit, DIB}; - -use crate::abi::FnAbi; -use crate::builder::Builder; -use crate::common::CodegenCx; -use crate::llvm; -use crate::llvm::debuginfo::{ - DIArray, DIBuilder, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DIType, - DIVariable, -}; -use crate::value::Value; - -use rustc_codegen_ssa::debuginfo::type_names; -use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; -use rustc_codegen_ssa::traits::*; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::Lrc; -use rustc_hir::def_id::{DefId, DefIdMap}; -use rustc_index::vec::IndexVec; -use rustc_middle::mir; -use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; -use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitable}; -use rustc_session::config::{self, DebugInfo}; -use rustc_session::Session; -use rustc_span::symbol::Symbol; -use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span}; -use rustc_target::abi::Size; - -use libc::c_uint; -use smallvec::SmallVec; -use std::cell::OnceCell; -use std::cell::RefCell; -use std::iter; -use tracing::debug; - -mod create_scope_map; -pub mod gdb; -pub mod metadata; -mod namespace; -mod utils; - -pub use self::create_scope_map::compute_mir_scopes; -pub use self::metadata::build_global_var_di_node; -pub use self::metadata::extend_scope_to_file; - -#[allow(non_upper_case_globals)] -const DW_TAG_auto_variable: c_uint = 0x100; -#[allow(non_upper_case_globals)] -const DW_TAG_arg_variable: c_uint = 0x101; - -/// A context object for maintaining all state needed by the debuginfo module. -pub struct CodegenUnitDebugContext<'ll, 'tcx> { - llcontext: &'ll llvm::Context, - llmod: &'ll llvm::Module, - builder: &'ll mut DIBuilder<'ll>, - created_files: RefCell, &'ll DIFile>>, - - type_map: metadata::TypeMap<'ll, 'tcx>, - namespace_map: RefCell>, - recursion_marker_type: OnceCell<&'ll DIType>, -} - -impl Drop for CodegenUnitDebugContext<'_, '_> { - fn drop(&mut self) { - unsafe { - llvm::LLVMRustDIBuilderDispose(&mut *(self.builder as *mut _)); - } - } -} - -impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { - pub fn new(llmod: &'ll llvm::Module) -> Self { - debug!("CodegenUnitDebugContext::new"); - let builder = unsafe { llvm::LLVMRustDIBuilderCreate(llmod) }; - // DIBuilder inherits context from the module, so we'd better use the same one - let llcontext = unsafe { llvm::LLVMGetModuleContext(llmod) }; - CodegenUnitDebugContext { - llcontext, - llmod, - builder, - created_files: Default::default(), - type_map: Default::default(), - namespace_map: RefCell::new(Default::default()), - recursion_marker_type: OnceCell::new(), - } - } - - pub fn finalize(&self, sess: &Session) { - unsafe { - llvm::LLVMRustDIBuilderFinalize(self.builder); - - if !sess.target.is_like_msvc { - // Debuginfo generation in LLVM by default uses a higher - // version of dwarf than macOS currently understands. We can - // instruct LLVM to emit an older version of dwarf, however, - // for macOS to understand. For more info see #11352 - // This can be overridden using --llvm-opts -dwarf-version,N. - // Android has the same issue (#22398) - let dwarf_version = sess - .opts - .unstable_opts - .dwarf_version - .unwrap_or(sess.target.default_dwarf_version); - llvm::LLVMRustAddModuleFlag( - self.llmod, - llvm::LLVMModFlagBehavior::Warning, - "Dwarf Version\0".as_ptr().cast(), - dwarf_version, - ); - } else { - // Indicate that we want CodeView debug information on MSVC - llvm::LLVMRustAddModuleFlag( - self.llmod, - llvm::LLVMModFlagBehavior::Warning, - "CodeView\0".as_ptr().cast(), - 1, - ) - } - - // Prevent bitcode readers from deleting the debug info. - let ptr = "Debug Info Version\0".as_ptr(); - llvm::LLVMRustAddModuleFlag( - self.llmod, - llvm::LLVMModFlagBehavior::Warning, - ptr.cast(), - llvm::LLVMRustDebugMetadataVersion(), - ); - } - } -} - -/// Creates any deferred debug metadata nodes -pub fn finalize(cx: &CodegenCx<'_, '_>) { - if let Some(dbg_cx) = &cx.dbg_cx { - debug!("finalize"); - - if gdb::needs_gdb_debug_scripts_section(cx) { - // Add a .debug_gdb_scripts section to this compile-unit. This will - // cause GDB to try and load the gdb_load_rust_pretty_printers.py file, - // which activates the Rust pretty printers for binary this section is - // contained in. - gdb::get_or_insert_gdb_debug_scripts_section_global(cx); - } - - dbg_cx.finalize(cx.sess()); - } -} - -impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { - // FIXME(eddyb) find a common convention for all of the debuginfo-related - // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). - fn dbg_var_addr( - &mut self, - dbg_var: &'ll DIVariable, - dbg_loc: &'ll DILocation, - variable_alloca: Self::Value, - direct_offset: Size, - indirect_offsets: &[Size], - ) { - // Convert the direct and indirect offsets to address ops. - // FIXME(eddyb) use `const`s instead of getting the values via FFI, - // the values should match the ones in the DWARF standard anyway. - let op_deref = || unsafe { llvm::LLVMRustDIBuilderCreateOpDeref() }; - let op_plus_uconst = || unsafe { llvm::LLVMRustDIBuilderCreateOpPlusUconst() }; - let mut addr_ops = SmallVec::<[u64; 8]>::new(); - - if direct_offset.bytes() > 0 { - addr_ops.push(op_plus_uconst()); - addr_ops.push(direct_offset.bytes() as u64); - } - for &offset in indirect_offsets { - addr_ops.push(op_deref()); - if offset.bytes() > 0 { - addr_ops.push(op_plus_uconst()); - addr_ops.push(offset.bytes() as u64); - } - } - - unsafe { - // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. - llvm::LLVMRustDIBuilderInsertDeclareAtEnd( - DIB(self.cx()), - variable_alloca, - dbg_var, - addr_ops.as_ptr(), - addr_ops.len() as c_uint, - dbg_loc, - self.llbb(), - ); - } - } - - fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { - unsafe { - let dbg_loc_as_llval = llvm::LLVMRustMetadataAsValue(self.cx().llcx, dbg_loc); - llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc_as_llval); - } - } - - fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { - gdb::insert_reference_to_gdb_debug_scripts_section_global(self) - } - - fn set_var_name(&mut self, value: &'ll Value, name: &str) { - // Avoid wasting time if LLVM value names aren't even enabled. - if self.sess().fewer_names() { - return; - } - - // Only function parameters and instructions are local to a function, - // don't change the name of anything else (e.g. globals). - let param_or_inst = unsafe { - llvm::LLVMIsAArgument(value).is_some() || llvm::LLVMIsAInstruction(value).is_some() - }; - if !param_or_inst { - return; - } - - // Avoid replacing the name if it already exists. - // While we could combine the names somehow, it'd - // get noisy quick, and the usefulness is dubious. - if llvm::get_value_name(value).is_empty() { - llvm::set_value_name(value, name.as_bytes()); - } - } -} - -/// A source code location used to generate debug information. -// FIXME(eddyb) rename this to better indicate it's a duplicate of -// `rustc_span::Loc` rather than `DILocation`, perhaps by making -// `lookup_char_pos` return the right information instead. -pub struct DebugLoc { - /// Information about the original source file. - pub file: Lrc, - /// The (1-based) line number. - pub line: u32, - /// The (1-based) column number. - pub col: u32, -} - -impl CodegenCx<'_, '_> { - /// Looks up debug source information about a `BytePos`. - // FIXME(eddyb) rename this to better indicate it's a duplicate of - // `lookup_char_pos` rather than `dbg_loc`, perhaps by making - // `lookup_char_pos` return the right information instead. - pub fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { - let (file, line, col) = match self.sess().source_map().lookup_line(pos) { - Ok(SourceFileAndLine { sf: file, line }) => { - let line_pos = file.line_begin_pos(pos); - - // Use 1-based indexing. - let line = (line + 1) as u32; - let col = (pos - line_pos).to_u32() + 1; - - (file, line, col) - } - Err(file) => (file, UNKNOWN_LINE_NUMBER, UNKNOWN_COLUMN_NUMBER), - }; - - // For MSVC, omit the column number. - // Otherwise, emit it. This mimics clang behaviour. - // See discussion in https://github.com/rust-lang/rust/issues/42921 - if self.sess().target.is_like_msvc { - DebugLoc { file, line, col: UNKNOWN_COLUMN_NUMBER } - } else { - DebugLoc { file, line, col } - } - } -} - -impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn create_function_debug_context( - &self, - instance: Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - llfn: &'ll Value, - mir: &mir::Body<'tcx>, - ) -> Option> { - if self.sess().opts.debuginfo == DebugInfo::None { - return None; - } - - // Initialize fn debug context (including scopes). - let empty_scope = DebugScope { - dbg_scope: self.dbg_scope_fn(instance, fn_abi, Some(llfn)), - inlined_at: None, - file_start_pos: BytePos(0), - file_end_pos: BytePos(0), - }; - let mut fn_debug_context = - FunctionDebugContext { scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes) }; - - // Fill in all the scopes, with the information from the MIR body. - compute_mir_scopes(self, instance, mir, &mut fn_debug_context); - - Some(fn_debug_context) - } - - fn dbg_scope_fn( - &self, - instance: Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - maybe_definition_llfn: Option<&'ll Value>, - ) -> &'ll DIScope { - let tcx = self.tcx; - - let def_id = instance.def_id(); - let containing_scope = get_containing_scope(self, instance); - let span = tcx.def_span(def_id); - let loc = self.lookup_debug_loc(span.lo()); - let file_metadata = file_metadata(self, &loc.file); - - let function_type_metadata = unsafe { - let fn_signature = get_function_signature(self, fn_abi); - llvm::LLVMRustDIBuilderCreateSubroutineType(DIB(self), fn_signature) - }; - - let mut name = String::new(); - type_names::push_item_name(tcx, def_id, false, &mut name); - - // Find the enclosing function, in case this is a closure. - let enclosing_fn_def_id = tcx.typeck_root_def_id(def_id); - - // We look up the generics of the enclosing function and truncate the substs - // to their length in order to cut off extra stuff that might be in there for - // closures or generators. - let generics = tcx.generics_of(enclosing_fn_def_id); - let substs = instance.substs.truncate_to(tcx, generics); - - type_names::push_generic_params( - tcx, - tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs), - &mut name, - ); - - let template_parameters = get_template_parameters(self, generics, substs); - - let linkage_name = &mangled_name_of_instance(self, instance).name; - // Omit the linkage_name if it is the same as subprogram name. - let linkage_name = if &name == linkage_name { "" } else { linkage_name }; - - // FIXME(eddyb) does this need to be separate from `loc.line` for some reason? - let scope_line = loc.line; - - let mut flags = DIFlags::FlagPrototyped; - - if fn_abi.ret.layout.abi.is_uninhabited() { - flags |= DIFlags::FlagNoReturn; - } - - let mut spflags = DISPFlags::SPFlagDefinition; - if is_node_local_to_unit(self, def_id) { - spflags |= DISPFlags::SPFlagLocalToUnit; - } - if self.sess().opts.optimize != config::OptLevel::No { - spflags |= DISPFlags::SPFlagOptimized; - } - if let Some((id, _)) = tcx.entry_fn(()) { - if id == def_id { - spflags |= DISPFlags::SPFlagMainSubprogram; - } - } - - unsafe { - return llvm::LLVMRustDIBuilderCreateFunction( - DIB(self), - containing_scope, - name.as_ptr().cast(), - name.len(), - linkage_name.as_ptr().cast(), - linkage_name.len(), - file_metadata, - loc.line, - function_type_metadata, - scope_line, - flags, - spflags, - maybe_definition_llfn, - template_parameters, - None, - ); - } - - fn get_function_signature<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - ) -> &'ll DIArray { - if cx.sess().opts.debuginfo == DebugInfo::Limited { - return create_DIArray(DIB(cx), &[]); - } - - let mut signature = Vec::with_capacity(fn_abi.args.len() + 1); - - // Return type -- llvm::DIBuilder wants this at index 0 - signature.push(if fn_abi.ret.is_ignore() { - None - } else { - Some(type_di_node(cx, fn_abi.ret.layout.ty)) - }); - - // Arguments types - if cx.sess().target.is_like_msvc { - // FIXME(#42800): - // There is a bug in MSDIA that leads to a crash when it encounters - // a fixed-size array of `u8` or something zero-sized in a - // function-type (see #40477). - // As a workaround, we replace those fixed-size arrays with a - // pointer-type. So a function `fn foo(a: u8, b: [u8; 4])` would - // appear as `fn foo(a: u8, b: *const u8)` in debuginfo, - // and a function `fn bar(x: [(); 7])` as `fn bar(x: *const ())`. - // This transformed type is wrong, but these function types are - // already inaccurate due to ABI adjustments (see #42800). - signature.extend(fn_abi.args.iter().map(|arg| { - let t = arg.layout.ty; - let t = match t.kind() { - ty::Array(ct, _) - if (*ct == cx.tcx.types.u8) || cx.layout_of(*ct).is_zst() => - { - cx.tcx.mk_imm_ptr(*ct) - } - _ => t, - }; - Some(type_di_node(cx, t)) - })); - } else { - signature - .extend(fn_abi.args.iter().map(|arg| Some(type_di_node(cx, arg.layout.ty)))); - } - - create_DIArray(DIB(cx), &signature[..]) - } - - fn get_template_parameters<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - generics: &ty::Generics, - substs: SubstsRef<'tcx>, - ) -> &'ll DIArray { - if substs.types().next().is_none() { - return create_DIArray(DIB(cx), &[]); - } - - // Again, only create type information if full debuginfo is enabled - let template_params: Vec<_> = if cx.sess().opts.debuginfo == DebugInfo::Full { - let names = get_parameter_names(cx, generics); - iter::zip(substs, names) - .filter_map(|(kind, name)| { - if let GenericArgKind::Type(ty) = kind.unpack() { - let actual_type = - cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); - let actual_type_metadata = type_di_node(cx, actual_type); - let name = name.as_str(); - Some(unsafe { - Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( - DIB(cx), - None, - name.as_ptr().cast(), - name.len(), - actual_type_metadata, - )) - }) - } else { - None - } - }) - .collect() - } else { - vec![] - }; - - create_DIArray(DIB(cx), &template_params) - } - - fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec { - let mut names = generics.parent.map_or_else(Vec::new, |def_id| { - get_parameter_names(cx, cx.tcx.generics_of(def_id)) - }); - names.extend(generics.params.iter().map(|param| param.name)); - names - } - - fn get_containing_scope<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - instance: Instance<'tcx>, - ) -> &'ll DIScope { - // First, let's see if this is a method within an inherent impl. Because - // if yes, we want to make the result subroutine DIE a child of the - // subroutine's self-type. - let self_type = cx.tcx.impl_of_method(instance.def_id()).and_then(|impl_def_id| { - // If the method does *not* belong to a trait, proceed - if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { - let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions( - instance.substs, - ty::ParamEnv::reveal_all(), - cx.tcx.type_of(impl_def_id), - ); - - // Only "class" methods are generally understood by LLVM, - // so avoid methods on other types (e.g., `<*mut T>::null`). - match impl_self_ty.kind() { - ty::Adt(def, ..) if !def.is_box() => { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full - && !impl_self_ty.needs_subst() - { - Some(type_di_node(cx, impl_self_ty)) - } else { - Some(namespace::item_namespace(cx, def.did())) - } - } - _ => None, - } - } else { - // For trait method impls we still use the "parallel namespace" - // strategy - None - } - }); - - self_type.unwrap_or_else(|| { - namespace::item_namespace( - cx, - DefId { - krate: instance.def_id().krate, - index: cx - .tcx - .def_key(instance.def_id()) - .parent - .expect("get_containing_scope: missing parent?"), - }, - ) - }) - } - } - - fn dbg_loc( - &self, - scope: &'ll DIScope, - inlined_at: Option<&'ll DILocation>, - span: Span, - ) -> &'ll DILocation { - let DebugLoc { line, col, .. } = self.lookup_debug_loc(span.lo()); - - unsafe { llvm::LLVMRustDIBuilderCreateDebugLocation(line, col, scope, inlined_at) } - } - - fn create_vtable_debuginfo( - &self, - ty: Ty<'tcx>, - trait_ref: Option>, - vtable: Self::Value, - ) { - metadata::create_vtable_di_node(self, ty, trait_ref, vtable) - } - - fn extend_scope_to_file( - &self, - scope_metadata: &'ll DIScope, - file: &rustc_span::SourceFile, - ) -> &'ll DILexicalBlock { - metadata::extend_scope_to_file(self, scope_metadata, file) - } - - fn debuginfo_finalize(&self) { - finalize(self) - } - - // FIXME(eddyb) find a common convention for all of the debuginfo-related - // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). - fn create_dbg_var( - &self, - variable_name: Symbol, - variable_type: Ty<'tcx>, - scope_metadata: &'ll DIScope, - variable_kind: VariableKind, - span: Span, - ) -> &'ll DIVariable { - let loc = self.lookup_debug_loc(span.lo()); - let file_metadata = file_metadata(self, &loc.file); - - let type_metadata = type_di_node(self, variable_type); - - let (argument_index, dwarf_tag) = match variable_kind { - ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), - LocalVariable => (0, DW_TAG_auto_variable), - }; - let align = self.align_of(variable_type); - - let name = variable_name.as_str(); - unsafe { - llvm::LLVMRustDIBuilderCreateVariable( - DIB(self), - dwarf_tag, - scope_metadata, - name.as_ptr().cast(), - name.len(), - file_metadata, - loc.line, - type_metadata, - true, - DIFlags::FlagZero, - argument_index, - align.bytes() as u32, - ) - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs deleted file mode 100644 index d5ea48c311b..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ /dev/null @@ -1,48 +0,0 @@ -// Namespace Handling. - -use super::utils::{debug_context, DIB}; -use rustc_codegen_ssa::debuginfo::type_names; -use rustc_middle::ty::{self, Instance}; - -use crate::common::CodegenCx; -use crate::llvm; -use crate::llvm::debuginfo::DIScope; -use rustc_hir::def_id::DefId; - -pub fn mangled_name_of_instance<'a, 'tcx>( - cx: &CodegenCx<'a, 'tcx>, - instance: Instance<'tcx>, -) -> ty::SymbolName<'tcx> { - let tcx = cx.tcx; - tcx.symbol_name(instance) -} - -pub fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { - if let Some(&scope) = debug_context(cx).namespace_map.borrow().get(&def_id) { - return scope; - } - - let def_key = cx.tcx.def_key(def_id); - let parent_scope = def_key - .parent - .map(|parent| item_namespace(cx, DefId { krate: def_id.krate, index: parent })); - - let namespace_name_string = { - let mut output = String::new(); - type_names::push_item_name(cx.tcx, def_id, false, &mut output); - output - }; - - let scope = unsafe { - llvm::LLVMRustDIBuilderCreateNameSpace( - DIB(cx), - parent_scope, - namespace_name_string.as_ptr().cast(), - namespace_name_string.len(), - false, // ExportSymbols (only relevant for C++ anonymous namespaces) - ) - }; - - debug_context(cx).namespace_map.borrow_mut().insert(def_id, scope); - scope -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs deleted file mode 100644 index 8f243673907..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ /dev/null @@ -1,99 +0,0 @@ -// Utility Functions. - -use super::namespace::item_namespace; -use super::CodegenUnitDebugContext; - -use rustc_hir::def_id::DefId; -use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; -use rustc_middle::ty::{self, DefIdTree, Ty}; -use tracing::trace; - -use crate::common::CodegenCx; -use crate::llvm; -use crate::llvm::debuginfo::{DIArray, DIBuilder, DIDescriptor, DIScope}; - -pub fn is_node_local_to_unit(cx: &CodegenCx<'_, '_>, def_id: DefId) -> bool { - // The is_local_to_unit flag indicates whether a function is local to the - // current compilation unit (i.e., if it is *static* in the C-sense). The - // *reachable* set should provide a good approximation of this, as it - // contains everything that might leak out of the current crate (by being - // externally visible or by being inlined into something externally - // visible). It might better to use the `exported_items` set from - // `driver::CrateAnalysis` in the future, but (atm) this set is not - // available in the codegen pass. - !cx.tcx.is_reachable_non_generic(def_id) -} - -#[allow(non_snake_case)] -pub fn create_DIArray<'ll>( - builder: &DIBuilder<'ll>, - arr: &[Option<&'ll DIDescriptor>], -) -> &'ll DIArray { - unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) } -} - -#[inline] -pub fn debug_context<'a, 'll, 'tcx>( - cx: &'a CodegenCx<'ll, 'tcx>, -) -> &'a CodegenUnitDebugContext<'ll, 'tcx> { - cx.dbg_cx.as_ref().unwrap() -} - -#[inline] -#[allow(non_snake_case)] -pub fn DIB<'a, 'll>(cx: &'a CodegenCx<'ll, '_>) -> &'a DIBuilder<'ll> { - cx.dbg_cx.as_ref().unwrap().builder -} - -pub fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { - item_namespace(cx, cx.tcx.parent(def_id)) -} - -#[derive(Debug, PartialEq, Eq)] -pub(crate) enum FatPtrKind { - Slice, - Dyn, -} - -/// Determines if `pointee_ty` is slice-like or trait-object-like, i.e. -/// if the second field of the fat pointer is a length or a vtable-pointer. -/// If `pointee_ty` does not require a fat pointer (because it is Sized) then -/// the function returns `None`. -pub(crate) fn fat_pointer_kind<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - pointee_ty: Ty<'tcx>, -) -> Option { - let pointee_tail_ty = cx.tcx.struct_tail_erasing_lifetimes(pointee_ty, cx.param_env()); - let layout = cx.layout_of(pointee_tail_ty); - trace!( - "fat_pointer_kind: {:?} has layout {:?} (is_unsized? {})", - pointee_tail_ty, - layout, - layout.is_unsized() - ); - - if !layout.is_unsized() { - return None; - } - - match *pointee_tail_ty.kind() { - ty::Str | ty::Slice(_) => Some(FatPtrKind::Slice), - ty::Dynamic(..) => Some(FatPtrKind::Dyn), - ty::Foreign(_) => { - // Assert that pointers to foreign types really are thin: - debug_assert_eq!( - cx.size_of(cx.tcx.mk_imm_ptr(pointee_tail_ty)), - cx.size_of(cx.tcx.mk_imm_ptr(cx.tcx.types.u8)) - ); - None - } - _ => { - // For all other pointee types we should already have returned None - // at the beginning of the function. - panic!( - "fat_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {:?}", - pointee_tail_ty - ) - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs deleted file mode 100644 index fa0ecd18fc8..00000000000 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! Declare various LLVM values. -//! -//! Prefer using functions and methods from this module rather than calling LLVM -//! functions directly. These functions do some additional work to ensure we do -//! the right thing given the preconceptions of codegen. -//! -//! Some useful guidelines: -//! -//! * Use declare_* family of methods if you are declaring, but are not -//! interested in defining the Value they return. -//! * Use define_* family of methods when you might be defining the Value. -//! * When in doubt, define. - -use crate::abi::{FnAbi, FnAbiLlvmExt}; -use crate::attributes; -use crate::context::CodegenCx; -use crate::llvm; -use crate::llvm::AttributePlace::Function; -use crate::type_::Type; -use crate::value::Value; -use rustc_codegen_ssa::traits::TypeMembershipMethods; -use rustc_middle::ty::Ty; -use rustc_symbol_mangling::typeid::typeid_for_fnabi; -use smallvec::SmallVec; -use tracing::debug; - -/// Declare a function. -/// -/// If there’s a value with the same name already declared, the function will -/// update the declaration and return existing Value instead. -fn declare_raw_fn<'ll>( - cx: &CodegenCx<'ll, '_>, - name: &str, - callconv: llvm::CallConv, - unnamed: llvm::UnnamedAddr, - ty: &'ll Type, -) -> &'ll Value { - debug!("declare_raw_fn(name={:?}, ty={:?})", name, ty); - let llfn = unsafe { - llvm::LLVMRustGetOrInsertFunction(cx.llmod, name.as_ptr().cast(), name.len(), ty) - }; - - llvm::SetFunctionCallConv(llfn, callconv); - llvm::SetUnnamedAddress(llfn, unnamed); - - let mut attrs = SmallVec::<[_; 4]>::new(); - - if cx.tcx.sess.opts.cg.no_redzone.unwrap_or(cx.tcx.sess.target.disable_redzone) { - attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx)); - } - - attrs.extend(attributes::non_lazy_bind_attr(cx)); - - attributes::apply_to_llfn(llfn, Function, &attrs); - - llfn -} - -impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { - /// Declare a global value. - /// - /// If there’s a value with the same name already declared, the function will - /// return its Value instead. - pub fn declare_global(&self, name: &str, ty: &'ll Type) -> &'ll Value { - debug!("declare_global(name={:?})", name); - unsafe { llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_ptr().cast(), name.len(), ty) } - } - - /// Declare a C ABI function. - /// - /// Only use this for foreign function ABIs and glue. For Rust functions use - /// `declare_fn` instead. - /// - /// If there’s a value with the same name already declared, the function will - /// update the declaration and return existing Value instead. - pub fn declare_cfn( - &self, - name: &str, - unnamed: llvm::UnnamedAddr, - fn_type: &'ll Type, - ) -> &'ll Value { - declare_raw_fn(self, name, llvm::CCallConv, unnamed, fn_type) - } - - /// Declare a Rust function. - /// - /// If there’s a value with the same name already declared, the function will - /// update the declaration and return existing Value instead. - pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Value { - debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi); - - // Function addresses in Rust are never significant, allowing functions to - // be merged. - let llfn = declare_raw_fn( - self, - name, - fn_abi.llvm_cconv(), - llvm::UnnamedAddr::Global, - fn_abi.llvm_type(self), - ); - fn_abi.apply_attrs_llfn(self, llfn); - - if self.tcx.sess.is_sanitizer_cfi_enabled() { - let typeid = typeid_for_fnabi(self.tcx, fn_abi); - self.set_type_metadata(llfn, typeid); - } - - llfn - } - - /// Declare a global with an intention to define it. - /// - /// Use this function when you intend to define a global. This function will - /// return `None` if the name already has a definition associated with it. In that - /// case an error should be reported to the user, because it usually happens due - /// to user’s fault (e.g., misuse of `#[no_mangle]` or `#[export_name]` attributes). - pub fn define_global(&self, name: &str, ty: &'ll Type) -> Option<&'ll Value> { - if self.get_defined_value(name).is_some() { - None - } else { - Some(self.declare_global(name, ty)) - } - } - - /// Declare a private global - /// - /// Use this function when you intend to define a global without a name. - pub fn define_private_global(&self, ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMRustInsertPrivateGlobal(self.llmod, ty) } - } - - /// Gets declared value by name. - pub fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { - debug!("get_declared_value(name={:?})", name); - unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_ptr().cast(), name.len()) } - } - - /// Gets defined or externally defined (AvailableExternally linkage) value by - /// name. - pub fn get_defined_value(&self, name: &str) -> Option<&'ll Value> { - self.get_declared_value(name).and_then(|val| { - let declaration = unsafe { llvm::LLVMIsDeclaration(val) != 0 }; - if !declaration { Some(val) } else { None } - }) - } -} diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs deleted file mode 100644 index 9f364749287..00000000000 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ /dev/null @@ -1,1924 +0,0 @@ -use crate::abi::{Abi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode}; -use crate::builder::Builder; -use crate::context::CodegenCx; -use crate::llvm; -use crate::type_::Type; -use crate::type_of::LayoutLlvmExt; -use crate::va_arg::emit_va_arg; -use crate::value::Value; - -use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh}; -use rustc_codegen_ssa::common::span_invalid_monomorphization_error; -use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; -use rustc_codegen_ssa::mir::operand::OperandRef; -use rustc_codegen_ssa::mir::place::PlaceRef; -use rustc_codegen_ssa::traits::*; -use rustc_hir as hir; -use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; -use rustc_middle::ty::{self, Ty}; -use rustc_middle::{bug, span_bug}; -use rustc_span::{sym, symbol::kw, Span, Symbol}; -use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; -use rustc_target::spec::{HasTargetSpec, PanicStrategy}; - -use std::cmp::Ordering; -use std::iter; - -fn get_simple_intrinsic<'ll>( - cx: &CodegenCx<'ll, '_>, - name: Symbol, -) -> Option<(&'ll Type, &'ll Value)> { - let llvm_name = match name { - sym::sqrtf32 => "llvm.sqrt.f32", - sym::sqrtf64 => "llvm.sqrt.f64", - sym::powif32 => "llvm.powi.f32", - sym::powif64 => "llvm.powi.f64", - sym::sinf32 => "llvm.sin.f32", - sym::sinf64 => "llvm.sin.f64", - sym::cosf32 => "llvm.cos.f32", - sym::cosf64 => "llvm.cos.f64", - sym::powf32 => "llvm.pow.f32", - sym::powf64 => "llvm.pow.f64", - sym::expf32 => "llvm.exp.f32", - sym::expf64 => "llvm.exp.f64", - sym::exp2f32 => "llvm.exp2.f32", - sym::exp2f64 => "llvm.exp2.f64", - sym::logf32 => "llvm.log.f32", - sym::logf64 => "llvm.log.f64", - sym::log10f32 => "llvm.log10.f32", - sym::log10f64 => "llvm.log10.f64", - sym::log2f32 => "llvm.log2.f32", - sym::log2f64 => "llvm.log2.f64", - sym::fmaf32 => "llvm.fma.f32", - sym::fmaf64 => "llvm.fma.f64", - sym::fabsf32 => "llvm.fabs.f32", - sym::fabsf64 => "llvm.fabs.f64", - sym::minnumf32 => "llvm.minnum.f32", - sym::minnumf64 => "llvm.minnum.f64", - sym::maxnumf32 => "llvm.maxnum.f32", - sym::maxnumf64 => "llvm.maxnum.f64", - sym::copysignf32 => "llvm.copysign.f32", - sym::copysignf64 => "llvm.copysign.f64", - sym::floorf32 => "llvm.floor.f32", - sym::floorf64 => "llvm.floor.f64", - sym::ceilf32 => "llvm.ceil.f32", - sym::ceilf64 => "llvm.ceil.f64", - sym::truncf32 => "llvm.trunc.f32", - sym::truncf64 => "llvm.trunc.f64", - sym::rintf32 => "llvm.rint.f32", - sym::rintf64 => "llvm.rint.f64", - sym::nearbyintf32 => "llvm.nearbyint.f32", - sym::nearbyintf64 => "llvm.nearbyint.f64", - sym::roundf32 => "llvm.round.f32", - sym::roundf64 => "llvm.round.f64", - _ => return None, - }; - Some(cx.get_intrinsic(llvm_name)) -} - -impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { - fn codegen_intrinsic_call( - &mut self, - instance: ty::Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - args: &[OperandRef<'tcx, &'ll Value>], - llresult: &'ll Value, - span: Span, - ) { - let tcx = self.tcx; - let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); - - let ty::FnDef(def_id, substs) = *callee_ty.kind() else { - bug!("expected fn item type, found {}", callee_ty); - }; - - let sig = callee_ty.fn_sig(tcx); - let sig = tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig); - let arg_tys = sig.inputs(); - let ret_ty = sig.output(); - let name = tcx.item_name(def_id); - - let llret_ty = self.layout_of(ret_ty).llvm_type(self); - let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); - - let simple = get_simple_intrinsic(self, name); - let llval = match name { - _ if simple.is_some() => { - let (simple_ty, simple_fn) = simple.unwrap(); - self.call( - simple_ty, - simple_fn, - &args.iter().map(|arg| arg.immediate()).collect::>(), - None, - ) - } - sym::likely => { - self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)]) - } - sym::unlikely => self - .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]), - kw::Try => { - try_intrinsic( - self, - args[0].immediate(), - args[1].immediate(), - args[2].immediate(), - llresult, - ); - return; - } - sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]), - sym::va_copy => { - self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()]) - } - sym::va_arg => { - match fn_abi.ret.layout.abi { - abi::Abi::Scalar(scalar) => { - match scalar.primitive() { - Primitive::Int(..) => { - if self.cx().size_of(ret_ty).bytes() < 4 { - // `va_arg` should not be called on an integer type - // less than 4 bytes in length. If it is, promote - // the integer to an `i32` and truncate the result - // back to the smaller type. - let promoted_result = emit_va_arg(self, args[0], tcx.types.i32); - self.trunc(promoted_result, llret_ty) - } else { - emit_va_arg(self, args[0], ret_ty) - } - } - Primitive::F64 | Primitive::Pointer => { - emit_va_arg(self, args[0], ret_ty) - } - // `va_arg` should never be used with the return type f32. - Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"), - } - } - _ => bug!("the va_arg intrinsic does not work with non-scalar types"), - } - } - - sym::volatile_load | sym::unaligned_volatile_load => { - let tp_ty = substs.type_at(0); - let ptr = args[0].immediate(); - let load = if let PassMode::Cast(ty) = fn_abi.ret.mode { - let llty = ty.llvm_type(self); - let ptr = self.pointercast(ptr, self.type_ptr_to(llty)); - self.volatile_load(llty, ptr) - } else { - self.volatile_load(self.layout_of(tp_ty).llvm_type(self), ptr) - }; - let align = if name == sym::unaligned_volatile_load { - 1 - } else { - self.align_of(tp_ty).bytes() as u32 - }; - unsafe { - llvm::LLVMSetAlignment(load, align); - } - self.to_immediate(load, self.layout_of(tp_ty)) - } - sym::volatile_store => { - let dst = args[0].deref(self.cx()); - args[1].val.volatile_store(self, dst); - return; - } - sym::unaligned_volatile_store => { - let dst = args[0].deref(self.cx()); - args[1].val.unaligned_volatile_store(self, dst); - return; - } - sym::prefetch_read_data - | sym::prefetch_write_data - | sym::prefetch_read_instruction - | sym::prefetch_write_instruction => { - let (rw, cache_type) = match name { - sym::prefetch_read_data => (0, 1), - sym::prefetch_write_data => (1, 1), - sym::prefetch_read_instruction => (0, 0), - sym::prefetch_write_instruction => (1, 0), - _ => bug!(), - }; - self.call_intrinsic( - "llvm.prefetch", - &[ - args[0].immediate(), - self.const_i32(rw), - args[1].immediate(), - self.const_i32(cache_type), - ], - ) - } - sym::ctlz - | sym::ctlz_nonzero - | sym::cttz - | sym::cttz_nonzero - | sym::ctpop - | sym::bswap - | sym::bitreverse - | sym::rotate_left - | sym::rotate_right - | sym::saturating_add - | sym::saturating_sub => { - let ty = arg_tys[0]; - match int_type_width_signed(ty, self) { - Some((width, signed)) => match name { - sym::ctlz | sym::cttz => { - let y = self.const_bool(false); - self.call_intrinsic( - &format!("llvm.{}.i{}", name, width), - &[args[0].immediate(), y], - ) - } - sym::ctlz_nonzero => { - let y = self.const_bool(true); - let llvm_name = &format!("llvm.ctlz.i{}", width); - self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) - } - sym::cttz_nonzero => { - let y = self.const_bool(true); - let llvm_name = &format!("llvm.cttz.i{}", width); - self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) - } - sym::ctpop => self.call_intrinsic( - &format!("llvm.ctpop.i{}", width), - &[args[0].immediate()], - ), - sym::bswap => { - if width == 8 { - args[0].immediate() // byte swap a u8/i8 is just a no-op - } else { - self.call_intrinsic( - &format!("llvm.bswap.i{}", width), - &[args[0].immediate()], - ) - } - } - sym::bitreverse => self.call_intrinsic( - &format!("llvm.bitreverse.i{}", width), - &[args[0].immediate()], - ), - sym::rotate_left | sym::rotate_right => { - let is_left = name == sym::rotate_left; - let val = args[0].immediate(); - let raw_shift = args[1].immediate(); - // rotate = funnel shift with first two args the same - let llvm_name = - &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); - self.call_intrinsic(llvm_name, &[val, val, raw_shift]) - } - sym::saturating_add | sym::saturating_sub => { - let is_add = name == sym::saturating_add; - let lhs = args[0].immediate(); - let rhs = args[1].immediate(); - let llvm_name = &format!( - "llvm.{}{}.sat.i{}", - if signed { 's' } else { 'u' }, - if is_add { "add" } else { "sub" }, - width - ); - self.call_intrinsic(llvm_name, &[lhs, rhs]) - } - _ => bug!(), - }, - None => { - span_invalid_monomorphization_error( - tcx.sess, - span, - &format!( - "invalid monomorphization of `{}` intrinsic: \ - expected basic integer type, found `{}`", - name, ty - ), - ); - return; - } - } - } - - sym::raw_eq => { - use abi::Abi::*; - let tp_ty = substs.type_at(0); - let layout = self.layout_of(tp_ty).layout; - let use_integer_compare = match layout.abi() { - Scalar(_) | ScalarPair(_, _) => true, - Uninhabited | Vector { .. } => false, - Aggregate { .. } => { - // For rusty ABIs, small aggregates are actually passed - // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), - // so we re-use that same threshold here. - layout.size() <= self.data_layout().pointer_size * 2 - } - }; - - let a = args[0].immediate(); - let b = args[1].immediate(); - if layout.size().bytes() == 0 { - self.const_bool(true) - } else if use_integer_compare { - let integer_ty = self.type_ix(layout.size().bits()); - let ptr_ty = self.type_ptr_to(integer_ty); - let a_ptr = self.bitcast(a, ptr_ty); - let a_val = self.load(integer_ty, a_ptr, layout.align().abi); - let b_ptr = self.bitcast(b, ptr_ty); - let b_val = self.load(integer_ty, b_ptr, layout.align().abi); - self.icmp(IntPredicate::IntEQ, a_val, b_val) - } else { - let i8p_ty = self.type_i8p(); - let a_ptr = self.bitcast(a, i8p_ty); - let b_ptr = self.bitcast(b, i8p_ty); - let n = self.const_usize(layout.size().bytes()); - let cmp = self.call_intrinsic("memcmp", &[a_ptr, b_ptr, n]); - match self.cx.sess().target.arch.as_ref() { - "avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)), - _ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)), - } - } - } - - sym::black_box => { - args[0].val.store(self, result); - - // We need to "use" the argument in some way LLVM can't introspect, and on - // targets that support it we can typically leverage inline assembly to do - // this. LLVM's interpretation of inline assembly is that it's, well, a black - // box. This isn't the greatest implementation since it probably deoptimizes - // more than we want, but it's so far good enough. - crate::asm::inline_asm_call( - self, - "", - "r,~{memory}", - &[result.llval], - self.type_void(), - true, - false, - llvm::AsmDialect::Att, - &[span], - false, - None, - ) - .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); - - // We have copied the value to `result` already. - return; - } - - _ if name.as_str().starts_with("simd_") => { - match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { - Ok(llval) => llval, - Err(()) => return, - } - } - - _ => bug!("unknown intrinsic '{}'", name), - }; - - if !fn_abi.ret.is_ignore() { - if let PassMode::Cast(ty) = fn_abi.ret.mode { - let ptr_llty = self.type_ptr_to(ty.llvm_type(self)); - let ptr = self.pointercast(result.llval, ptr_llty); - self.store(llval, ptr, result.align); - } else { - OperandRef::from_immediate_or_packed_pair(self, llval, result.layout) - .val - .store(self, result); - } - } - } - - fn abort(&mut self) { - self.call_intrinsic("llvm.trap", &[]); - } - - fn assume(&mut self, val: Self::Value) { - self.call_intrinsic("llvm.assume", &[val]); - } - - fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value { - self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) - } - - fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value { - // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time - // optimization pass replaces calls to this intrinsic with code to test type membership. - let i8p_ty = self.type_i8p(); - let bitcast = self.bitcast(pointer, i8p_ty); - self.call_intrinsic("llvm.type.test", &[bitcast, typeid]) - } - - fn type_checked_load( - &mut self, - llvtable: &'ll Value, - vtable_byte_offset: u64, - typeid: &'ll Value, - ) -> Self::Value { - let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); - self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]) - } - - fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { - self.call_intrinsic("llvm.va_start", &[va_list]) - } - - fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value { - self.call_intrinsic("llvm.va_end", &[va_list]) - } -} - -fn try_intrinsic<'ll>( - bx: &mut Builder<'_, 'll, '_>, - try_func: &'ll Value, - data: &'ll Value, - catch_func: &'ll Value, - dest: &'ll Value, -) { - if bx.sess().panic_strategy() == PanicStrategy::Abort { - let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.call(try_func_ty, try_func, &[data], None); - // Return 0 unconditionally from the intrinsic call; - // we can never unwind. - let ret_align = bx.tcx().data_layout.i32_align.abi; - bx.store(bx.const_i32(0), dest, ret_align); - } else if wants_msvc_seh(bx.sess()) { - codegen_msvc_try(bx, try_func, data, catch_func, dest); - } else if bx.sess().target.os == "emscripten" { - codegen_emcc_try(bx, try_func, data, catch_func, dest); - } else { - codegen_gnu_try(bx, try_func, data, catch_func, dest); - } -} - -// MSVC's definition of the `rust_try` function. -// -// This implementation uses the new exception handling instructions in LLVM -// which have support in LLVM for SEH on MSVC targets. Although these -// instructions are meant to work for all targets, as of the time of this -// writing, however, LLVM does not recommend the usage of these new instructions -// as the old ones are still more optimized. -fn codegen_msvc_try<'ll>( - bx: &mut Builder<'_, 'll, '_>, - try_func: &'ll Value, - data: &'ll Value, - catch_func: &'ll Value, - dest: &'ll Value, -) { - let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { - bx.set_personality_fn(bx.eh_personality()); - - let normal = bx.append_sibling_block("normal"); - let catchswitch = bx.append_sibling_block("catchswitch"); - let catchpad_rust = bx.append_sibling_block("catchpad_rust"); - let catchpad_foreign = bx.append_sibling_block("catchpad_foreign"); - let caught = bx.append_sibling_block("caught"); - - let try_func = llvm::get_param(bx.llfn(), 0); - let data = llvm::get_param(bx.llfn(), 1); - let catch_func = llvm::get_param(bx.llfn(), 2); - - // We're generating an IR snippet that looks like: - // - // declare i32 @rust_try(%try_func, %data, %catch_func) { - // %slot = alloca i8* - // invoke %try_func(%data) to label %normal unwind label %catchswitch - // - // normal: - // ret i32 0 - // - // catchswitch: - // %cs = catchswitch within none [%catchpad_rust, %catchpad_foreign] unwind to caller - // - // catchpad_rust: - // %tok = catchpad within %cs [%type_descriptor, 8, %slot] - // %ptr = load %slot - // call %catch_func(%data, %ptr) - // catchret from %tok to label %caught - // - // catchpad_foreign: - // %tok = catchpad within %cs [null, 64, null] - // call %catch_func(%data, null) - // catchret from %tok to label %caught - // - // caught: - // ret i32 1 - // } - // - // This structure follows the basic usage of throw/try/catch in LLVM. - // For example, compile this C++ snippet to see what LLVM generates: - // - // struct rust_panic { - // rust_panic(const rust_panic&); - // ~rust_panic(); - // - // void* x[2]; - // }; - // - // int __rust_try( - // void (*try_func)(void*), - // void *data, - // void (*catch_func)(void*, void*) noexcept - // ) { - // try { - // try_func(data); - // return 0; - // } catch(rust_panic& a) { - // catch_func(data, &a); - // return 1; - // } catch(...) { - // catch_func(data, NULL); - // return 1; - // } - // } - // - // More information can be found in libstd's seh.rs implementation. - let ptr_align = bx.tcx().data_layout.pointer_align.abi; - let slot = bx.alloca(bx.type_i8p(), ptr_align); - let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], normal, catchswitch, None); - - bx.switch_to_block(normal); - bx.ret(bx.const_i32(0)); - - bx.switch_to_block(catchswitch); - let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]); - - // We can't use the TypeDescriptor defined in libpanic_unwind because it - // might be in another DLL and the SEH encoding only supports specifying - // a TypeDescriptor from the current module. - // - // However this isn't an issue since the MSVC runtime uses string - // comparison on the type name to match TypeDescriptors rather than - // pointer equality. - // - // So instead we generate a new TypeDescriptor in each module that uses - // `try` and let the linker merge duplicate definitions in the same - // module. - // - // When modifying, make sure that the type_name string exactly matches - // the one used in src/libpanic_unwind/seh.rs. - let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_i8p()); - let type_name = bx.const_bytes(b"rust_panic\0"); - let type_info = - bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_i8p()), type_name], false); - let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info)); - unsafe { - llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); - llvm::SetUniqueComdat(bx.llmod, tydesc); - llvm::LLVMSetInitializer(tydesc, type_info); - } - - // The flag value of 8 indicates that we are catching the exception by - // reference instead of by value. We can't use catch by value because - // that requires copying the exception object, which we don't support - // since our exception object effectively contains a Box. - // - // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang - bx.switch_to_block(catchpad_rust); - let flags = bx.const_i32(8); - let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); - let ptr = bx.load(bx.type_i8p(), slot, ptr_align); - let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, catch_func, &[data, ptr], Some(&funclet)); - bx.catch_ret(&funclet, caught); - - // The flag value of 64 indicates a "catch-all". - bx.switch_to_block(catchpad_foreign); - let flags = bx.const_i32(64); - let null = bx.const_null(bx.type_i8p()); - let funclet = bx.catch_pad(cs, &[null, flags, null]); - bx.call(catch_ty, catch_func, &[data, null], Some(&funclet)); - bx.catch_ret(&funclet, caught); - - bx.switch_to_block(caught); - bx.ret(bx.const_i32(1)); - }); - - // Note that no invoke is used here because by definition this function - // can't panic (that's what it's catching). - let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); - let i32_align = bx.tcx().data_layout.i32_align.abi; - bx.store(ret, dest, i32_align); -} - -// Definition of the standard `try` function for Rust using the GNU-like model -// of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke` -// instructions). -// -// This codegen is a little surprising because we always call a shim -// function instead of inlining the call to `invoke` manually here. This is done -// because in LLVM we're only allowed to have one personality per function -// definition. The call to the `try` intrinsic is being inlined into the -// function calling it, and that function may already have other personality -// functions in play. By calling a shim we're guaranteed that our shim will have -// the right personality function. -fn codegen_gnu_try<'ll>( - bx: &mut Builder<'_, 'll, '_>, - try_func: &'ll Value, - data: &'ll Value, - catch_func: &'ll Value, - dest: &'ll Value, -) { - let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { - // Codegens the shims described above: - // - // bx: - // invoke %try_func(%data) normal %normal unwind %catch - // - // normal: - // ret 0 - // - // catch: - // (%ptr, _) = landingpad - // call %catch_func(%data, %ptr) - // ret 1 - let then = bx.append_sibling_block("then"); - let catch = bx.append_sibling_block("catch"); - - let try_func = llvm::get_param(bx.llfn(), 0); - let data = llvm::get_param(bx.llfn(), 1); - let catch_func = llvm::get_param(bx.llfn(), 2); - let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], then, catch, None); - - bx.switch_to_block(then); - bx.ret(bx.const_i32(0)); - - // Type indicator for the exception being thrown. - // - // The first value in this tuple is a pointer to the exception object - // being thrown. The second value is a "selector" indicating which of - // the landing pad clauses the exception's type had been matched to. - // rust_try ignores the selector. - bx.switch_to_block(catch); - let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); - let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1); - let tydesc = bx.const_null(bx.type_i8p()); - bx.add_clause(vals, tydesc); - let ptr = bx.extract_value(vals, 0); - let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, catch_func, &[data, ptr], None); - bx.ret(bx.const_i32(1)); - }); - - // Note that no invoke is used here because by definition this function - // can't panic (that's what it's catching). - let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); - let i32_align = bx.tcx().data_layout.i32_align.abi; - bx.store(ret, dest, i32_align); -} - -// Variant of codegen_gnu_try used for emscripten where Rust panics are -// implemented using C++ exceptions. Here we use exceptions of a specific type -// (`struct rust_panic`) to represent Rust panics. -fn codegen_emcc_try<'ll>( - bx: &mut Builder<'_, 'll, '_>, - try_func: &'ll Value, - data: &'ll Value, - catch_func: &'ll Value, - dest: &'ll Value, -) { - let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { - // Codegens the shims described above: - // - // bx: - // invoke %try_func(%data) normal %normal unwind %catch - // - // normal: - // ret 0 - // - // catch: - // (%ptr, %selector) = landingpad - // %rust_typeid = @llvm.eh.typeid.for(@_ZTI10rust_panic) - // %is_rust_panic = %selector == %rust_typeid - // %catch_data = alloca { i8*, i8 } - // %catch_data[0] = %ptr - // %catch_data[1] = %is_rust_panic - // call %catch_func(%data, %catch_data) - // ret 1 - let then = bx.append_sibling_block("then"); - let catch = bx.append_sibling_block("catch"); - - let try_func = llvm::get_param(bx.llfn(), 0); - let data = llvm::get_param(bx.llfn(), 1); - let catch_func = llvm::get_param(bx.llfn(), 2); - let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], then, catch, None); - - bx.switch_to_block(then); - bx.ret(bx.const_i32(0)); - - // Type indicator for the exception being thrown. - // - // The first value in this tuple is a pointer to the exception object - // being thrown. The second value is a "selector" indicating which of - // the landing pad clauses the exception's type had been matched to. - bx.switch_to_block(catch); - let tydesc = bx.eh_catch_typeinfo(); - let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); - let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2); - bx.add_clause(vals, tydesc); - bx.add_clause(vals, bx.const_null(bx.type_i8p())); - let ptr = bx.extract_value(vals, 0); - let selector = bx.extract_value(vals, 1); - - // Check if the typeid we got is the one for a Rust panic. - let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[tydesc]); - let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid); - let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool()); - - // We need to pass two values to catch_func (ptr and is_rust_panic), so - // create an alloca and pass a pointer to that. - let ptr_align = bx.tcx().data_layout.pointer_align.abi; - let i8_align = bx.tcx().data_layout.i8_align.abi; - let catch_data_type = bx.type_struct(&[bx.type_i8p(), bx.type_bool()], false); - let catch_data = bx.alloca(catch_data_type, ptr_align); - let catch_data_0 = - bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(0)]); - bx.store(ptr, catch_data_0, ptr_align); - let catch_data_1 = - bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(1)]); - bx.store(is_rust_panic, catch_data_1, i8_align); - let catch_data = bx.bitcast(catch_data, bx.type_i8p()); - - let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, catch_func, &[data, catch_data], None); - bx.ret(bx.const_i32(1)); - }); - - // Note that no invoke is used here because by definition this function - // can't panic (that's what it's catching). - let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); - let i32_align = bx.tcx().data_layout.i32_align.abi; - bx.store(ret, dest, i32_align); -} - -// Helper function to give a Block to a closure to codegen a shim function. -// This is currently primarily used for the `try` intrinsic functions above. -fn gen_fn<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - name: &str, - rust_fn_sig: ty::PolyFnSig<'tcx>, - codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), -) -> (&'ll Type, &'ll Value) { - let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); - let llty = fn_abi.llvm_type(cx); - let llfn = cx.declare_fn(name, fn_abi); - cx.set_frame_pointer_type(llfn); - cx.apply_target_cpu_attr(llfn); - // FIXME(eddyb) find a nicer way to do this. - unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) }; - let llbb = Builder::append_block(cx, llfn, "entry-block"); - let bx = Builder::build(cx, llbb); - codegen(bx); - (llty, llfn) -} - -// Helper function used to get a handle to the `__rust_try` function used to -// catch exceptions. -// -// This function is only generated once and is then cached. -fn get_rust_try_fn<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), -) -> (&'ll Type, &'ll Value) { - if let Some(llfn) = cx.rust_try_fn.get() { - return llfn; - } - - // Define the type up front for the signature of the rust_try function. - let tcx = cx.tcx; - let i8p = tcx.mk_mut_ptr(tcx.types.i8); - // `unsafe fn(*mut i8) -> ()` - let try_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( - iter::once(i8p), - tcx.mk_unit(), - false, - hir::Unsafety::Unsafe, - Abi::Rust, - ))); - // `unsafe fn(*mut i8, *mut i8) -> ()` - let catch_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( - [i8p, i8p].iter().cloned(), - tcx.mk_unit(), - false, - hir::Unsafety::Unsafe, - Abi::Rust, - ))); - // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` - let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig( - [try_fn_ty, i8p, catch_fn_ty].into_iter(), - tcx.types.i32, - false, - hir::Unsafety::Unsafe, - Abi::Rust, - )); - let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen); - cx.rust_try_fn.set(Some(rust_try)); - rust_try -} - -fn generic_simd_intrinsic<'ll, 'tcx>( - bx: &mut Builder<'_, 'll, 'tcx>, - name: Symbol, - callee_ty: Ty<'tcx>, - args: &[OperandRef<'tcx, &'ll Value>], - ret_ty: Ty<'tcx>, - llret_ty: &'ll Type, - span: Span, -) -> Result<&'ll Value, ()> { - // macros for error handling: - #[allow(unused_macro_rules)] - macro_rules! emit_error { - ($msg: tt) => { - emit_error!($msg, ) - }; - ($msg: tt, $($fmt: tt)*) => { - span_invalid_monomorphization_error( - bx.sess(), span, - &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), - name, $($fmt)*)); - } - } - - macro_rules! return_error { - ($($fmt: tt)*) => { - { - emit_error!($($fmt)*); - return Err(()); - } - } - } - - macro_rules! require { - ($cond: expr, $($fmt: tt)*) => { - if !$cond { - return_error!($($fmt)*); - } - }; - } - - macro_rules! require_simd { - ($ty: expr, $position: expr) => { - require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty) - }; - } - - let tcx = bx.tcx(); - let sig = - tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx)); - let arg_tys = sig.inputs(); - - if name == sym::simd_select_bitmask { - require_simd!(arg_tys[1], "argument"); - let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); - - let expected_int_bits = (len.max(8) - 1).next_power_of_two(); - let expected_bytes = len / 8 + ((len % 8 > 0) as u64); - - let mask_ty = arg_tys[0]; - let mask = match mask_ty.kind() { - ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), - ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), - ty::Array(elem, len) - if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) - == Some(expected_bytes) => - { - let place = PlaceRef::alloca(bx, args[0].layout); - args[0].val.store(bx, place); - let int_ty = bx.type_ix(expected_bytes * 8); - let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty)); - bx.load(int_ty, ptr, Align::ONE) - } - _ => return_error!( - "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`", - mask_ty, - expected_int_bits, - expected_bytes - ), - }; - - let i1 = bx.type_i1(); - let im = bx.type_ix(len); - let i1xn = bx.type_vector(i1, len); - let m_im = bx.trunc(mask, im); - let m_i1s = bx.bitcast(m_im, i1xn); - return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); - } - - // every intrinsic below takes a SIMD vector as its first argument - require_simd!(arg_tys[0], "input"); - let in_ty = arg_tys[0]; - - let comparison = match name { - sym::simd_eq => Some(hir::BinOpKind::Eq), - sym::simd_ne => Some(hir::BinOpKind::Ne), - sym::simd_lt => Some(hir::BinOpKind::Lt), - sym::simd_le => Some(hir::BinOpKind::Le), - sym::simd_gt => Some(hir::BinOpKind::Gt), - sym::simd_ge => Some(hir::BinOpKind::Ge), - _ => None, - }; - - let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx()); - if let Some(cmp_op) = comparison { - require_simd!(ret_ty, "return"); - - let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); - require!( - in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len - ); - require!( - bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, - "expected return type with integer elements, found `{}` with non-integer `{}`", - ret_ty, - out_ty - ); - - return Ok(compare_simd_types( - bx, - args[0].immediate(), - args[1].immediate(), - in_elem, - llret_ty, - cmp_op, - )); - } - - if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") { - // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer. - // If there is no suffix, use the index array length. - let n: u64 = if stripped.is_empty() { - // Make sure this is actually an array, since typeck only checks the length-suffixed - // version of this intrinsic. - match args[2].layout.ty.kind() { - ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { - len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| { - span_bug!(span, "could not evaluate shuffle index array length") - }) - } - _ => return_error!( - "simd_shuffle index must be an array of `u32`, got `{}`", - args[2].layout.ty - ), - } - } else { - stripped.parse().unwrap_or_else(|_| { - span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") - }) - }; - - require_simd!(ret_ty, "return"); - let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); - require!( - out_len == n, - "expected return type of length {}, found `{}` with length {}", - n, - ret_ty, - out_len - ); - require!( - in_elem == out_ty, - "expected return element type `{}` (element of input `{}`), \ - found `{}` with element type `{}`", - in_elem, - in_ty, - ret_ty, - out_ty - ); - - let total_len = u128::from(in_len) * 2; - - let vector = args[2].immediate(); - - let indices: Option> = (0..n) - .map(|i| { - let arg_idx = i; - let val = bx.const_get_elt(vector, i as u64); - match bx.const_to_opt_u128(val, true) { - None => { - emit_error!("shuffle index #{} is not a constant", arg_idx); - None - } - Some(idx) if idx >= total_len => { - emit_error!( - "shuffle index #{} is out of bounds (limit {})", - arg_idx, - total_len - ); - None - } - Some(idx) => Some(bx.const_i32(idx as i32)), - } - }) - .collect(); - let Some(indices) = indices else { - return Ok(bx.const_null(llret_ty)); - }; - - return Ok(bx.shuffle_vector( - args[0].immediate(), - args[1].immediate(), - bx.const_vector(&indices), - )); - } - - if name == sym::simd_insert { - require!( - in_elem == arg_tys[2], - "expected inserted type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - arg_tys[2] - ); - return Ok(bx.insert_element( - args[0].immediate(), - args[2].immediate(), - args[1].immediate(), - )); - } - if name == sym::simd_extract { - require!( - ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty - ); - return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); - } - - if name == sym::simd_select { - let m_elem_ty = in_elem; - let m_len = in_len; - require_simd!(arg_tys[1], "argument"); - let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); - require!( - m_len == v_len, - "mismatched lengths: mask length `{}` != other vector length `{}`", - m_len, - v_len - ); - match m_elem_ty.kind() { - ty::Int(_) => {} - _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty), - } - // truncate the mask to a vector of i1s - let i1 = bx.type_i1(); - let i1xn = bx.type_vector(i1, m_len as u64); - let m_i1s = bx.trunc(args[0].immediate(), i1xn); - return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); - } - - if name == sym::simd_bitmask { - // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a - // vector mask and returns the most significant bit (MSB) of each lane in the form - // of either: - // * an unsigned integer - // * an array of `u8` - // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits. - // - // The bit order of the result depends on the byte endianness, LSB-first for little - // endian and MSB-first for big endian. - let expected_int_bits = in_len.max(8); - let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64); - - // Integer vector : - let (i_xn, in_elem_bitwidth) = match in_elem.kind() { - ty::Int(i) => ( - args[0].immediate(), - i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), - ), - ty::Uint(i) => ( - args[0].immediate(), - i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), - ), - _ => return_error!( - "vector argument `{}`'s element type `{}`, expected integer element type", - in_ty, - in_elem - ), - }; - - // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position. - let shift_indices = - vec![ - bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _); - in_len as _ - ]; - let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice())); - // Truncate vector to an - let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len)); - // Bitcast to iN: - let i_ = bx.bitcast(i1xn, bx.type_ix(in_len)); - - match ret_ty.kind() { - ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => { - // Zero-extend iN to the bitmask type: - return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); - } - ty::Array(elem, len) - if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) - == Some(expected_bytes) => - { - // Zero-extend iN to the array length: - let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); - - // Convert the integer to a byte array - let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE); - bx.store(ze, ptr, Align::ONE); - let array_ty = bx.type_array(bx.type_i8(), expected_bytes); - let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty)); - return Ok(bx.load(array_ty, ptr, Align::ONE)); - } - _ => return_error!( - "cannot return `{}`, expected `u{}` or `[u8; {}]`", - ret_ty, - expected_int_bits, - expected_bytes - ), - } - } - - fn simd_simple_float_intrinsic<'ll, 'tcx>( - name: Symbol, - in_elem: Ty<'_>, - in_ty: Ty<'_>, - in_len: u64, - bx: &mut Builder<'_, 'll, 'tcx>, - span: Span, - args: &[OperandRef<'tcx, &'ll Value>], - ) -> Result<&'ll Value, ()> { - #[allow(unused_macro_rules)] - macro_rules! emit_error { - ($msg: tt) => { - emit_error!($msg, ) - }; - ($msg: tt, $($fmt: tt)*) => { - span_invalid_monomorphization_error( - bx.sess(), span, - &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), - name, $($fmt)*)); - } - } - macro_rules! return_error { - ($($fmt: tt)*) => { - { - emit_error!($($fmt)*); - return Err(()); - } - } - } - - let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() { - let elem_ty = bx.cx.type_float_from_ty(*f); - match f.bit_width() { - 32 => ("f32", elem_ty), - 64 => ("f64", elem_ty), - _ => { - return_error!( - "unsupported element type `{}` of floating-point vector `{}`", - f.name_str(), - in_ty - ); - } - } - } else { - return_error!("`{}` is not a floating-point type", in_ty); - }; - - let vec_ty = bx.type_vector(elem_ty, in_len); - - let (intr_name, fn_ty) = match name { - sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), - sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)), - sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)), - sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)), - sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)), - _ => return_error!("unrecognized intrinsic `{}`", name), - }; - let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); - let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty); - let c = - bx.call(fn_ty, f, &args.iter().map(|arg| arg.immediate()).collect::>(), None); - Ok(c) - } - - if std::matches!( - name, - sym::simd_ceil - | sym::simd_fabs - | sym::simd_fcos - | sym::simd_fexp2 - | sym::simd_fexp - | sym::simd_flog10 - | sym::simd_flog2 - | sym::simd_flog - | sym::simd_floor - | sym::simd_fma - | sym::simd_fpow - | sym::simd_fpowi - | sym::simd_fsin - | sym::simd_fsqrt - | sym::simd_round - | sym::simd_trunc - ) { - return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args); - } - - // FIXME: use: - // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182 - // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81 - fn llvm_vector_str( - elem_ty: Ty<'_>, - vec_len: u64, - no_pointers: usize, - bx: &Builder<'_, '_, '_>, - ) -> String { - let p0s: String = "p0".repeat(no_pointers); - match *elem_ty.kind() { - ty::Int(v) => format!( - "v{}{}i{}", - vec_len, - p0s, - // Normalize to prevent crash if v: IntTy::Isize - v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() - ), - ty::Uint(v) => format!( - "v{}{}i{}", - vec_len, - p0s, - // Normalize to prevent crash if v: UIntTy::Usize - v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() - ), - ty::Float(v) => format!("v{}{}f{}", vec_len, p0s, v.bit_width()), - _ => unreachable!(), - } - } - - fn llvm_vector_ty<'ll>( - cx: &CodegenCx<'ll, '_>, - elem_ty: Ty<'_>, - vec_len: u64, - mut no_pointers: usize, - ) -> &'ll Type { - // FIXME: use cx.layout_of(ty).llvm_type() ? - let mut elem_ty = match *elem_ty.kind() { - ty::Int(v) => cx.type_int_from_ty(v), - ty::Uint(v) => cx.type_uint_from_ty(v), - ty::Float(v) => cx.type_float_from_ty(v), - _ => unreachable!(), - }; - while no_pointers > 0 { - elem_ty = cx.type_ptr_to(elem_ty); - no_pointers -= 1; - } - cx.type_vector(elem_ty, vec_len) - } - - if name == sym::simd_gather { - // simd_gather(values: , pointers: , - // mask: ) -> - // * N: number of elements in the input vectors - // * T: type of the element to load - // * M: any integer width is supported, will be truncated to i1 - - // All types must be simd vector types - require_simd!(in_ty, "first"); - require_simd!(arg_tys[1], "second"); - require_simd!(arg_tys[2], "third"); - require_simd!(ret_ty, "return"); - - // Of the same length: - let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); - let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); - require!( - in_len == out_len, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "second", - in_len, - in_ty, - arg_tys[1], - out_len - ); - require!( - in_len == out_len2, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "third", - in_len, - in_ty, - arg_tys[2], - out_len2 - ); - - // The return type must match the first argument type - require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty); - - // This counts how many pointers - fn ptr_count(t: Ty<'_>) -> usize { - match t.kind() { - ty::RawPtr(p) => 1 + ptr_count(p.ty), - _ => 0, - } - } - - // Non-ptr type - fn non_ptr(t: Ty<'_>) -> Ty<'_> { - match t.kind() { - ty::RawPtr(p) => non_ptr(p.ty), - _ => t, - } - } - - // The second argument must be a simd vector with an element type that's a pointer - // to the element type of the first argument - let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); - let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); - let (pointer_count, underlying_ty) = match element_ty1.kind() { - ty::RawPtr(p) if p.ty == in_elem => (ptr_count(element_ty1), non_ptr(element_ty1)), - _ => { - require!( - false, - "expected element type `{}` of second argument `{}` \ - to be a pointer to the element type `{}` of the first \ - argument `{}`, found `{}` != `*_ {}`", - element_ty1, - arg_tys[1], - in_elem, - in_ty, - element_ty1, - in_elem - ); - unreachable!(); - } - }; - assert!(pointer_count > 0); - assert_eq!(pointer_count - 1, ptr_count(element_ty0)); - assert_eq!(underlying_ty, non_ptr(element_ty0)); - - // The element type of the third argument must be a signed integer type of any width: - let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); - match element_ty2.kind() { - ty::Int(_) => (), - _ => { - require!( - false, - "expected element type `{}` of third argument `{}` \ - to be a signed integer type", - element_ty2, - arg_tys[2] - ); - } - } - - // Alignment of T, must be a constant integer value: - let alignment_ty = bx.type_i32(); - let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); - - // Truncate the mask vector to a vector of i1s: - let (mask, mask_ty) = { - let i1 = bx.type_i1(); - let i1xn = bx.type_vector(i1, in_len); - (bx.trunc(args[2].immediate(), i1xn), i1xn) - }; - - // Type of the vector of pointers: - let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); - let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx); - - // Type of the vector of elements: - let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); - let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx); - - let llvm_intrinsic = - format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); - let fn_ty = bx.type_func( - &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], - llvm_elem_vec_ty, - ); - let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = - bx.call(fn_ty, f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None); - return Ok(v); - } - - if name == sym::simd_scatter { - // simd_scatter(values: , pointers: , - // mask: ) -> () - // * N: number of elements in the input vectors - // * T: type of the element to load - // * M: any integer width is supported, will be truncated to i1 - - // All types must be simd vector types - require_simd!(in_ty, "first"); - require_simd!(arg_tys[1], "second"); - require_simd!(arg_tys[2], "third"); - - // Of the same length: - let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx()); - let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); - require!( - in_len == element_len1, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "second", - in_len, - in_ty, - arg_tys[1], - element_len1 - ); - require!( - in_len == element_len2, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "third", - in_len, - in_ty, - arg_tys[2], - element_len2 - ); - - // This counts how many pointers - fn ptr_count(t: Ty<'_>) -> usize { - match t.kind() { - ty::RawPtr(p) => 1 + ptr_count(p.ty), - _ => 0, - } - } - - // Non-ptr type - fn non_ptr(t: Ty<'_>) -> Ty<'_> { - match t.kind() { - ty::RawPtr(p) => non_ptr(p.ty), - _ => t, - } - } - - // The second argument must be a simd vector with an element type that's a pointer - // to the element type of the first argument - let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); - let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); - let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); - let (pointer_count, underlying_ty) = match element_ty1.kind() { - ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => { - (ptr_count(element_ty1), non_ptr(element_ty1)) - } - _ => { - require!( - false, - "expected element type `{}` of second argument `{}` \ - to be a pointer to the element type `{}` of the first \ - argument `{}`, found `{}` != `*mut {}`", - element_ty1, - arg_tys[1], - in_elem, - in_ty, - element_ty1, - in_elem - ); - unreachable!(); - } - }; - assert!(pointer_count > 0); - assert_eq!(pointer_count - 1, ptr_count(element_ty0)); - assert_eq!(underlying_ty, non_ptr(element_ty0)); - - // The element type of the third argument must be a signed integer type of any width: - match element_ty2.kind() { - ty::Int(_) => (), - _ => { - require!( - false, - "expected element type `{}` of third argument `{}` \ - be a signed integer type", - element_ty2, - arg_tys[2] - ); - } - } - - // Alignment of T, must be a constant integer value: - let alignment_ty = bx.type_i32(); - let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); - - // Truncate the mask vector to a vector of i1s: - let (mask, mask_ty) = { - let i1 = bx.type_i1(); - let i1xn = bx.type_vector(i1, in_len); - (bx.trunc(args[2].immediate(), i1xn), i1xn) - }; - - let ret_t = bx.type_void(); - - // Type of the vector of pointers: - let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); - let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx); - - // Type of the vector of elements: - let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); - let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx); - - let llvm_intrinsic = - format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); - let fn_ty = - bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t); - let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = - bx.call(fn_ty, f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None); - return Ok(v); - } - - macro_rules! arith_red { - ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident, - $identity:expr) => { - if name == sym::$name { - require!( - ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty - ); - return match in_elem.kind() { - ty::Int(_) | ty::Uint(_) => { - let r = bx.$integer_reduce(args[0].immediate()); - if $ordered { - // if overflow occurs, the result is the - // mathematical result modulo 2^n: - Ok(bx.$op(args[1].immediate(), r)) - } else { - Ok(bx.$integer_reduce(args[0].immediate())) - } - } - ty::Float(f) => { - let acc = if $ordered { - // ordered arithmetic reductions take an accumulator - args[1].immediate() - } else { - // unordered arithmetic reductions use the identity accumulator - match f.bit_width() { - 32 => bx.const_real(bx.type_f32(), $identity), - 64 => bx.const_real(bx.type_f64(), $identity), - v => return_error!( - r#" -unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, - sym::$name, - in_ty, - in_elem, - v, - ret_ty - ), - } - }; - Ok(bx.$float_reduce(acc, args[0].immediate())) - } - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, - in_ty, - in_elem, - ret_ty - ), - }; - } - }; - } - - arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, 0.0); - arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); - arith_red!( - simd_reduce_add_unordered: vector_reduce_add, - vector_reduce_fadd_fast, - false, - add, - 0.0 - ); - arith_red!( - simd_reduce_mul_unordered: vector_reduce_mul, - vector_reduce_fmul_fast, - false, - mul, - 1.0 - ); - - macro_rules! minmax_red { - ($name:ident: $int_red:ident, $float_red:ident) => { - if name == sym::$name { - require!( - ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty - ); - return match in_elem.kind() { - ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)), - ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)), - ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())), - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, - in_ty, - in_elem, - ret_ty - ), - }; - } - }; - } - - minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin); - minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax); - - minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast); - minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast); - - macro_rules! bitwise_red { - ($name:ident : $red:ident, $boolean:expr) => { - if name == sym::$name { - let input = if !$boolean { - require!( - ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty - ); - args[0].immediate() - } else { - match in_elem.kind() { - ty::Int(_) | ty::Uint(_) => {} - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, - in_ty, - in_elem, - ret_ty - ), - } - - // boolean reductions operate on vectors of i1s: - let i1 = bx.type_i1(); - let i1xn = bx.type_vector(i1, in_len as u64); - bx.trunc(args[0].immediate(), i1xn) - }; - return match in_elem.kind() { - ty::Int(_) | ty::Uint(_) => { - let r = bx.$red(input); - Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) }) - } - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, - in_ty, - in_elem, - ret_ty - ), - }; - } - }; - } - - bitwise_red!(simd_reduce_and: vector_reduce_and, false); - bitwise_red!(simd_reduce_or: vector_reduce_or, false); - bitwise_red!(simd_reduce_xor: vector_reduce_xor, false); - bitwise_red!(simd_reduce_all: vector_reduce_and, true); - bitwise_red!(simd_reduce_any: vector_reduce_or, true); - - if name == sym::simd_cast || name == sym::simd_as { - require_simd!(ret_ty, "return"); - let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); - require!( - in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len - ); - // casting cares about nominal type, not just structural type - if in_elem == out_elem { - return Ok(args[0].immediate()); - } - - enum Style { - Float, - Int(/* is signed? */ bool), - Unsupported, - } - - let (in_style, in_width) = match in_elem.kind() { - // vectors of pointer-sized integers should've been - // disallowed before here, so this unwrap is safe. - ty::Int(i) => ( - Style::Int(true), - i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), - ), - ty::Uint(u) => ( - Style::Int(false), - u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), - ), - ty::Float(f) => (Style::Float, f.bit_width()), - _ => (Style::Unsupported, 0), - }; - let (out_style, out_width) = match out_elem.kind() { - ty::Int(i) => ( - Style::Int(true), - i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), - ), - ty::Uint(u) => ( - Style::Int(false), - u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), - ), - ty::Float(f) => (Style::Float, f.bit_width()), - _ => (Style::Unsupported, 0), - }; - - match (in_style, out_style) { - (Style::Int(in_is_signed), Style::Int(_)) => { - return Ok(match in_width.cmp(&out_width) { - Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty), - Ordering::Equal => args[0].immediate(), - Ordering::Less => { - if in_is_signed { - bx.sext(args[0].immediate(), llret_ty) - } else { - bx.zext(args[0].immediate(), llret_ty) - } - } - }); - } - (Style::Int(in_is_signed), Style::Float) => { - return Ok(if in_is_signed { - bx.sitofp(args[0].immediate(), llret_ty) - } else { - bx.uitofp(args[0].immediate(), llret_ty) - }); - } - (Style::Float, Style::Int(out_is_signed)) => { - return Ok(match (out_is_signed, name == sym::simd_as) { - (false, false) => bx.fptoui(args[0].immediate(), llret_ty), - (true, false) => bx.fptosi(args[0].immediate(), llret_ty), - (_, true) => bx.cast_float_to_int(out_is_signed, args[0].immediate(), llret_ty), - }); - } - (Style::Float, Style::Float) => { - return Ok(match in_width.cmp(&out_width) { - Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty), - Ordering::Equal => args[0].immediate(), - Ordering::Less => bx.fpext(args[0].immediate(), llret_ty), - }); - } - _ => { /* Unsupported. Fallthrough. */ } - } - require!( - false, - "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", - in_ty, - in_elem, - ret_ty, - out_elem - ); - } - macro_rules! arith_binary { - ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { - $(if name == sym::$name { - match in_elem.kind() { - $($(ty::$p(_))|* => { - return Ok(bx.$call(args[0].immediate(), args[1].immediate())) - })* - _ => {}, - } - require!(false, - "unsupported operation on `{}` with element `{}`", - in_ty, - in_elem) - })* - } - } - arith_binary! { - simd_add: Uint, Int => add, Float => fadd; - simd_sub: Uint, Int => sub, Float => fsub; - simd_mul: Uint, Int => mul, Float => fmul; - simd_div: Uint => udiv, Int => sdiv, Float => fdiv; - simd_rem: Uint => urem, Int => srem, Float => frem; - simd_shl: Uint, Int => shl; - simd_shr: Uint => lshr, Int => ashr; - simd_and: Uint, Int => and; - simd_or: Uint, Int => or; - simd_xor: Uint, Int => xor; - simd_fmax: Float => maxnum; - simd_fmin: Float => minnum; - - } - macro_rules! arith_unary { - ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { - $(if name == sym::$name { - match in_elem.kind() { - $($(ty::$p(_))|* => { - return Ok(bx.$call(args[0].immediate())) - })* - _ => {}, - } - require!(false, - "unsupported operation on `{}` with element `{}`", - in_ty, - in_elem) - })* - } - } - arith_unary! { - simd_neg: Int => neg, Float => fneg; - } - - if name == sym::simd_arith_offset { - // This also checks that the first operand is a ptr type. - let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| { - span_bug!(span, "must be called with a vector of pointer types as first argument") - }); - let layout = bx.layout_of(pointee.ty); - let ptrs = args[0].immediate(); - // The second argument must be a ptr-sized integer. - // (We don't care about the signedness, this is wrapping anyway.) - let (_offsets_len, offsets_elem) = arg_tys[1].simd_size_and_type(bx.tcx()); - if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) { - span_bug!( - span, - "must be called with a vector of pointer-sized integers as second argument" - ); - } - let offsets = args[1].immediate(); - - return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets])); - } - - if name == sym::simd_saturating_add || name == sym::simd_saturating_sub { - let lhs = args[0].immediate(); - let rhs = args[1].immediate(); - let is_add = name == sym::simd_saturating_add; - let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _; - let (signed, elem_width, elem_ty) = match *in_elem.kind() { - ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), - ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), - _ => { - return_error!( - "expected element type `{}` of vector type `{}` \ - to be a signed or unsigned integer type", - arg_tys[0].simd_size_and_type(bx.tcx()).1, - arg_tys[0] - ); - } - }; - let llvm_intrinsic = &format!( - "llvm.{}{}.sat.v{}i{}", - if signed { 's' } else { 'u' }, - if is_add { "add" } else { "sub" }, - in_len, - elem_width - ); - let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); - - let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); - let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call(fn_ty, f, &[lhs, rhs], None); - return Ok(v); - } - - span_bug!(span, "unknown SIMD intrinsic"); -} - -// Returns the width of an int Ty, and if it's signed or not -// Returns None if the type is not an integer -// FIXME: there’s multiple of this functions, investigate using some of the already existing -// stuffs. -fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> { - match ty.kind() { - ty::Int(t) => { - Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), true)) - } - ty::Uint(t) => { - Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), false)) - } - _ => None, - } -} diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs deleted file mode 100644 index 636d689a34b..00000000000 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ /dev/null @@ -1,442 +0,0 @@ -//! The Rust compiler. -//! -//! # Note -//! -//! This API is completely unstable and subject to change. - -#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![feature(hash_raw_entry)] -#![feature(let_chains)] -#![feature(let_else)] -#![feature(extern_types)] -#![feature(once_cell)] -#![feature(iter_intersperse)] -#![recursion_limit = "256"] -#![allow(rustc::potential_query_instability)] - -#[macro_use] -extern crate rustc_macros; - -use back::write::{create_informational_target_machine, create_target_machine}; - -pub use llvm_util::target_features; -use rustc_ast::expand::allocator::AllocatorKind; -use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; -use rustc_codegen_ssa::back::write::{ - CodegenContext, FatLTOInput, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, -}; -use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::ModuleCodegen; -use rustc_codegen_ssa::{CodegenResults, CompiledModule}; -use rustc_data_structures::fx::FxHashMap; -use rustc_errors::{ErrorGuaranteed, FatalError, Handler}; -use rustc_metadata::EncodedMetadata; -use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::ty::query::Providers; -use rustc_middle::ty::TyCtxt; -use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; -use rustc_session::Session; -use rustc_span::symbol::Symbol; - -use std::any::Any; -use std::ffi::CStr; - -mod back { - pub mod archive; - pub mod lto; - mod profiling; - pub mod write; -} - -mod abi; -mod allocator; -mod asm; -mod attributes; -mod base; -mod builder; -mod callee; -mod common; -mod consts; -mod context; -mod coverageinfo; -mod debuginfo; -mod declare; -mod intrinsic; - -// The following is a work around that replaces `pub mod llvm;` and that fixes issue 53912. -#[path = "llvm/mod.rs"] -mod llvm_; -pub mod llvm { - pub use super::llvm_::*; -} - -mod llvm_util; -mod mono_item; -mod type_; -mod type_of; -mod va_arg; -mod value; - -#[derive(Clone)] -pub struct LlvmCodegenBackend(()); - -struct TimeTraceProfiler { - enabled: bool, -} - -impl TimeTraceProfiler { - fn new(enabled: bool) -> Self { - if enabled { - unsafe { llvm::LLVMTimeTraceProfilerInitialize() } - } - TimeTraceProfiler { enabled } - } -} - -impl Drop for TimeTraceProfiler { - fn drop(&mut self) { - if self.enabled { - unsafe { llvm::LLVMTimeTraceProfilerFinishThread() } - } - } -} - -impl ExtraBackendMethods for LlvmCodegenBackend { - fn codegen_allocator<'tcx>( - &self, - tcx: TyCtxt<'tcx>, - module_name: &str, - kind: AllocatorKind, - has_alloc_error_handler: bool, - ) -> ModuleLlvm { - let mut module_llvm = ModuleLlvm::new_metadata(tcx, module_name); - unsafe { - allocator::codegen(tcx, &mut module_llvm, module_name, kind, has_alloc_error_handler); - } - module_llvm - } - fn compile_codegen_unit( - &self, - tcx: TyCtxt<'_>, - cgu_name: Symbol, - ) -> (ModuleCodegen, u64) { - base::compile_codegen_unit(tcx, cgu_name) - } - fn target_machine_factory( - &self, - sess: &Session, - optlvl: OptLevel, - target_features: &[String], - ) -> TargetMachineFactoryFn { - back::write::target_machine_factory(sess, optlvl, target_features) - } - fn target_cpu<'b>(&self, sess: &'b Session) -> &'b str { - llvm_util::target_cpu(sess) - } - fn tune_cpu<'b>(&self, sess: &'b Session) -> Option<&'b str> { - llvm_util::tune_cpu(sess) - } - - fn spawn_thread(time_trace: bool, f: F) -> std::thread::JoinHandle - where - F: FnOnce() -> T, - F: Send + 'static, - T: Send + 'static, - { - std::thread::spawn(move || { - let _profiler = TimeTraceProfiler::new(time_trace); - f() - }) - } - - fn spawn_named_thread( - time_trace: bool, - name: String, - f: F, - ) -> std::io::Result> - where - F: FnOnce() -> T, - F: Send + 'static, - T: Send + 'static, - { - std::thread::Builder::new().name(name).spawn(move || { - let _profiler = TimeTraceProfiler::new(time_trace); - f() - }) - } -} - -impl WriteBackendMethods for LlvmCodegenBackend { - type Module = ModuleLlvm; - type ModuleBuffer = back::lto::ModuleBuffer; - type Context = llvm::Context; - type TargetMachine = &'static mut llvm::TargetMachine; - type ThinData = back::lto::ThinData; - type ThinBuffer = back::lto::ThinBuffer; - fn print_pass_timings(&self) { - unsafe { - llvm::LLVMRustPrintPassTimings(); - } - } - fn run_link( - cgcx: &CodegenContext, - diag_handler: &Handler, - modules: Vec>, - ) -> Result, FatalError> { - back::write::link(cgcx, diag_handler, modules) - } - fn run_fat_lto( - cgcx: &CodegenContext, - modules: Vec>, - cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result, FatalError> { - back::lto::run_fat(cgcx, modules, cached_modules) - } - fn run_thin_lto( - cgcx: &CodegenContext, - modules: Vec<(String, Self::ThinBuffer)>, - cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result<(Vec>, Vec), FatalError> { - back::lto::run_thin(cgcx, modules, cached_modules) - } - unsafe fn optimize( - cgcx: &CodegenContext, - diag_handler: &Handler, - module: &ModuleCodegen, - config: &ModuleConfig, - ) -> Result<(), FatalError> { - back::write::optimize(cgcx, diag_handler, module, config) - } - fn optimize_fat( - cgcx: &CodegenContext, - module: &mut ModuleCodegen, - ) -> Result<(), FatalError> { - let diag_handler = cgcx.create_diag_handler(); - back::lto::run_pass_manager(cgcx, &diag_handler, module, false) - } - unsafe fn optimize_thin( - cgcx: &CodegenContext, - thin: ThinModule, - ) -> Result, FatalError> { - back::lto::optimize_thin_module(thin, cgcx) - } - unsafe fn codegen( - cgcx: &CodegenContext, - diag_handler: &Handler, - module: ModuleCodegen, - config: &ModuleConfig, - ) -> Result { - back::write::codegen(cgcx, diag_handler, module, config) - } - fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer) { - back::lto::prepare_thin(module) - } - fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer) { - (module.name, back::lto::ModuleBuffer::new(module.module_llvm.llmod())) - } -} - -unsafe impl Send for LlvmCodegenBackend {} // Llvm is on a per-thread basis -unsafe impl Sync for LlvmCodegenBackend {} - -impl LlvmCodegenBackend { - pub fn new() -> Box { - Box::new(LlvmCodegenBackend(())) - } -} - -impl CodegenBackend for LlvmCodegenBackend { - fn init(&self, sess: &Session) { - llvm_util::init(sess); // Make sure llvm is inited - } - - fn provide(&self, providers: &mut Providers) { - providers.global_backend_features = - |tcx, ()| llvm_util::global_llvm_features(tcx.sess, true) - } - - fn print(&self, req: PrintRequest, sess: &Session) { - match req { - PrintRequest::RelocationModels => { - println!("Available relocation models:"); - for name in &[ - "static", - "pic", - "pie", - "dynamic-no-pic", - "ropi", - "rwpi", - "ropi-rwpi", - "default", - ] { - println!(" {}", name); - } - println!(); - } - PrintRequest::CodeModels => { - println!("Available code models:"); - for name in &["tiny", "small", "kernel", "medium", "large"] { - println!(" {}", name); - } - println!(); - } - PrintRequest::TlsModels => { - println!("Available TLS models:"); - for name in &["global-dynamic", "local-dynamic", "initial-exec", "local-exec"] { - println!(" {}", name); - } - println!(); - } - PrintRequest::StackProtectorStrategies => { - println!( - r#"Available stack protector strategies: - all - Generate stack canaries in all functions. - - strong - Generate stack canaries in a function if it either: - - has a local variable of `[T; N]` type, regardless of `T` and `N` - - takes the address of a local variable. - - (Note that a local variable being borrowed is not equivalent to its - address being taken: e.g. some borrows may be removed by optimization, - while by-value argument passing may be implemented with reference to a - local stack variable in the ABI.) - - basic - Generate stack canaries in functions with local variables of `[T; N]` - type, where `T` is byte-sized and `N` >= 8. - - none - Do not generate stack canaries. -"# - ); - } - req => llvm_util::print(req, sess), - } - } - - fn print_passes(&self) { - llvm_util::print_passes(); - } - - fn print_version(&self) { - llvm_util::print_version(); - } - - fn target_features(&self, sess: &Session, allow_unstable: bool) -> Vec { - target_features(sess, allow_unstable) - } - - fn codegen_crate<'tcx>( - &self, - tcx: TyCtxt<'tcx>, - metadata: EncodedMetadata, - need_metadata_module: bool, - ) -> Box { - Box::new(rustc_codegen_ssa::base::codegen_crate( - LlvmCodegenBackend(()), - tcx, - crate::llvm_util::target_cpu(tcx.sess).to_string(), - metadata, - need_metadata_module, - )) - } - - fn join_codegen( - &self, - ongoing_codegen: Box, - sess: &Session, - outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed> { - let (codegen_results, work_products) = ongoing_codegen - .downcast::>() - .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box") - .join(sess); - - sess.time("llvm_dump_timing_file", || { - if sess.opts.unstable_opts.llvm_time_trace { - let file_name = outputs.with_extension("llvm_timings.json"); - llvm_util::time_trace_profiler_finish(&file_name); - } - }); - - Ok((codegen_results, work_products)) - } - - fn link( - &self, - sess: &Session, - codegen_results: CodegenResults, - outputs: &OutputFilenames, - ) -> Result<(), ErrorGuaranteed> { - use crate::back::archive::LlvmArchiveBuilderBuilder; - use rustc_codegen_ssa::back::link::link_binary; - - // Run the linker on any artifacts that resulted from the LLVM run. - // This should produce either a finished executable or library. - link_binary(sess, &LlvmArchiveBuilderBuilder, &codegen_results, outputs) - } -} - -pub struct ModuleLlvm { - llcx: &'static mut llvm::Context, - llmod_raw: *const llvm::Module, - tm: &'static mut llvm::TargetMachine, -} - -unsafe impl Send for ModuleLlvm {} -unsafe impl Sync for ModuleLlvm {} - -impl ModuleLlvm { - fn new(tcx: TyCtxt<'_>, mod_name: &str) -> Self { - unsafe { - let llcx = llvm::LLVMRustContextCreate(tcx.sess.fewer_names()); - let llmod_raw = context::create_module(tcx, llcx, mod_name) as *const _; - ModuleLlvm { llmod_raw, llcx, tm: create_target_machine(tcx, mod_name) } - } - } - - fn new_metadata(tcx: TyCtxt<'_>, mod_name: &str) -> Self { - unsafe { - let llcx = llvm::LLVMRustContextCreate(tcx.sess.fewer_names()); - let llmod_raw = context::create_module(tcx, llcx, mod_name) as *const _; - ModuleLlvm { llmod_raw, llcx, tm: create_informational_target_machine(tcx.sess) } - } - } - - fn parse( - cgcx: &CodegenContext, - name: &CStr, - buffer: &[u8], - handler: &Handler, - ) -> Result { - unsafe { - let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let llmod_raw = back::lto::parse_module(llcx, name, buffer, handler)?; - let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, name.to_str().unwrap()); - let tm = match (cgcx.tm_factory)(tm_factory_config) { - Ok(m) => m, - Err(e) => { - handler.struct_err(&e).emit(); - return Err(FatalError); - } - }; - - Ok(ModuleLlvm { llmod_raw, llcx, tm }) - } - } - - fn llmod(&self) -> &llvm::Module { - unsafe { &*self.llmod_raw } - } -} - -impl Drop for ModuleLlvm { - fn drop(&mut self) { - unsafe { - llvm::LLVMRustDisposeTargetMachine(&mut *(self.tm as *mut _)); - llvm::LLVMContextDispose(&mut *(self.llcx as *mut _)); - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs deleted file mode 100644 index 64db4f7462d..00000000000 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ /dev/null @@ -1,105 +0,0 @@ -//! A wrapper around LLVM's archive (.a) code - -use rustc_fs_util::path_to_c_string; -use std::path::Path; -use std::slice; -use std::str; - -pub struct ArchiveRO { - pub raw: &'static mut super::Archive, -} - -unsafe impl Send for ArchiveRO {} - -pub struct Iter<'a> { - raw: &'a mut super::ArchiveIterator<'a>, -} - -pub struct Child<'a> { - pub raw: &'a mut super::ArchiveChild<'a>, -} - -impl ArchiveRO { - /// Opens a static archive for read-only purposes. This is more optimized - /// than the `open` method because it uses LLVM's internal `Archive` class - /// rather than shelling out to `ar` for everything. - /// - /// If this archive is used with a mutable method, then an error will be - /// raised. - pub fn open(dst: &Path) -> Result { - unsafe { - let s = path_to_c_string(dst); - let ar = super::LLVMRustOpenArchive(s.as_ptr()).ok_or_else(|| { - super::last_error().unwrap_or_else(|| "failed to open archive".to_owned()) - })?; - Ok(ArchiveRO { raw: ar }) - } - } - - pub fn iter(&self) -> Iter<'_> { - unsafe { Iter { raw: super::LLVMRustArchiveIteratorNew(self.raw) } } - } -} - -impl Drop for ArchiveRO { - fn drop(&mut self) { - unsafe { - super::LLVMRustDestroyArchive(&mut *(self.raw as *mut _)); - } - } -} - -impl<'a> Iterator for Iter<'a> { - type Item = Result, String>; - - fn next(&mut self) -> Option, String>> { - unsafe { - match super::LLVMRustArchiveIteratorNext(self.raw) { - Some(raw) => Some(Ok(Child { raw })), - None => super::last_error().map(Err), - } - } - } -} - -impl<'a> Drop for Iter<'a> { - fn drop(&mut self) { - unsafe { - super::LLVMRustArchiveIteratorFree(&mut *(self.raw as *mut _)); - } - } -} - -impl<'a> Child<'a> { - pub fn name(&self) -> Option<&'a str> { - unsafe { - let mut name_len = 0; - let name_ptr = super::LLVMRustArchiveChildName(self.raw, &mut name_len); - if name_ptr.is_null() { - None - } else { - let name = slice::from_raw_parts(name_ptr as *const u8, name_len as usize); - str::from_utf8(name).ok().map(|s| s.trim()) - } - } - } - - pub fn data(&self) -> &'a [u8] { - unsafe { - let mut data_len = 0; - let data_ptr = super::LLVMRustArchiveChildData(self.raw, &mut data_len); - if data_ptr.is_null() { - panic!("failed to read data from archive child"); - } - slice::from_raw_parts(data_ptr as *const u8, data_len as usize) - } - } -} - -impl<'a> Drop for Child<'a> { - fn drop(&mut self) { - unsafe { - super::LLVMRustArchiveChildFree(&mut *(self.raw as *mut _)); - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs deleted file mode 100644 index 45de284d22a..00000000000 --- a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs +++ /dev/null @@ -1,213 +0,0 @@ -//! LLVM diagnostic reports. - -pub use self::Diagnostic::*; -pub use self::OptimizationDiagnosticKind::*; - -use crate::value::Value; -use libc::c_uint; - -use super::{DiagnosticInfo, SMDiagnostic}; -use rustc_span::InnerSpan; - -#[derive(Copy, Clone)] -pub enum OptimizationDiagnosticKind { - OptimizationRemark, - OptimizationMissed, - OptimizationAnalysis, - OptimizationAnalysisFPCommute, - OptimizationAnalysisAliasing, - OptimizationFailure, - OptimizationRemarkOther, -} - -pub struct OptimizationDiagnostic<'ll> { - pub kind: OptimizationDiagnosticKind, - pub pass_name: String, - pub function: &'ll Value, - pub line: c_uint, - pub column: c_uint, - pub filename: String, - pub message: String, -} - -impl<'ll> OptimizationDiagnostic<'ll> { - unsafe fn unpack(kind: OptimizationDiagnosticKind, di: &'ll DiagnosticInfo) -> Self { - let mut function = None; - let mut line = 0; - let mut column = 0; - - let mut message = None; - let mut filename = None; - let pass_name = super::build_string(|pass_name| { - message = super::build_string(|message| { - filename = super::build_string(|filename| { - super::LLVMRustUnpackOptimizationDiagnostic( - di, - pass_name, - &mut function, - &mut line, - &mut column, - filename, - message, - ) - }) - .ok() - }) - .ok() - }) - .ok(); - - let mut filename = filename.unwrap_or_default(); - if filename.is_empty() { - filename.push_str(""); - } - - OptimizationDiagnostic { - kind, - pass_name: pass_name.expect("got a non-UTF8 pass name from LLVM"), - function: function.unwrap(), - line, - column, - filename, - message: message.expect("got a non-UTF8 OptimizationDiagnostic message from LLVM"), - } - } -} - -pub struct SrcMgrDiagnostic { - pub level: super::DiagnosticLevel, - pub message: String, - pub source: Option<(String, Vec)>, -} - -impl SrcMgrDiagnostic { - pub unsafe fn unpack(diag: &SMDiagnostic) -> SrcMgrDiagnostic { - // Recover the post-substitution assembly code from LLVM for better - // diagnostics. - let mut have_source = false; - let mut buffer = String::new(); - let mut level = super::DiagnosticLevel::Error; - let mut loc = 0; - let mut ranges = [0; 8]; - let mut num_ranges = ranges.len() / 2; - let message = super::build_string(|message| { - buffer = super::build_string(|buffer| { - have_source = super::LLVMRustUnpackSMDiagnostic( - diag, - message, - buffer, - &mut level, - &mut loc, - ranges.as_mut_ptr(), - &mut num_ranges, - ); - }) - .expect("non-UTF8 inline asm"); - }) - .expect("non-UTF8 SMDiagnostic"); - - SrcMgrDiagnostic { - message, - level, - source: have_source.then(|| { - let mut spans = vec![InnerSpan::new(loc as usize, loc as usize)]; - for i in 0..num_ranges { - spans.push(InnerSpan::new(ranges[i * 2] as usize, ranges[i * 2 + 1] as usize)); - } - (buffer, spans) - }), - } - } -} - -#[derive(Clone)] -pub struct InlineAsmDiagnostic { - pub level: super::DiagnosticLevel, - pub cookie: c_uint, - pub message: String, - pub source: Option<(String, Vec)>, -} - -impl InlineAsmDiagnostic { - unsafe fn unpackInlineAsm(di: &DiagnosticInfo) -> Self { - let mut cookie = 0; - let mut message = None; - let mut level = super::DiagnosticLevel::Error; - - super::LLVMRustUnpackInlineAsmDiagnostic(di, &mut level, &mut cookie, &mut message); - - InlineAsmDiagnostic { - level, - cookie, - message: super::twine_to_string(message.unwrap()), - source: None, - } - } - - unsafe fn unpackSrcMgr(di: &DiagnosticInfo) -> Self { - let mut cookie = 0; - let smdiag = SrcMgrDiagnostic::unpack(super::LLVMRustGetSMDiagnostic(di, &mut cookie)); - InlineAsmDiagnostic { - level: smdiag.level, - cookie, - message: smdiag.message, - source: smdiag.source, - } - } -} - -pub enum Diagnostic<'ll> { - Optimization(OptimizationDiagnostic<'ll>), - InlineAsm(InlineAsmDiagnostic), - PGO(&'ll DiagnosticInfo), - Linker(&'ll DiagnosticInfo), - Unsupported(&'ll DiagnosticInfo), - - /// LLVM has other types that we do not wrap here. - UnknownDiagnostic(&'ll DiagnosticInfo), -} - -impl<'ll> Diagnostic<'ll> { - pub unsafe fn unpack(di: &'ll DiagnosticInfo) -> Self { - use super::DiagnosticKind as Dk; - let kind = super::LLVMRustGetDiagInfoKind(di); - - match kind { - Dk::InlineAsm => InlineAsm(InlineAsmDiagnostic::unpackInlineAsm(di)), - - Dk::OptimizationRemark => { - Optimization(OptimizationDiagnostic::unpack(OptimizationRemark, di)) - } - Dk::OptimizationRemarkOther => { - Optimization(OptimizationDiagnostic::unpack(OptimizationRemarkOther, di)) - } - Dk::OptimizationRemarkMissed => { - Optimization(OptimizationDiagnostic::unpack(OptimizationMissed, di)) - } - - Dk::OptimizationRemarkAnalysis => { - Optimization(OptimizationDiagnostic::unpack(OptimizationAnalysis, di)) - } - - Dk::OptimizationRemarkAnalysisFPCommute => { - Optimization(OptimizationDiagnostic::unpack(OptimizationAnalysisFPCommute, di)) - } - - Dk::OptimizationRemarkAnalysisAliasing => { - Optimization(OptimizationDiagnostic::unpack(OptimizationAnalysisAliasing, di)) - } - - Dk::OptimizationFailure => { - Optimization(OptimizationDiagnostic::unpack(OptimizationFailure, di)) - } - - Dk::PGOProfile => PGO(di), - Dk::Linker => Linker(di), - Dk::Unsupported => Unsupported(di), - - Dk::SrcMgr => InlineAsm(InlineAsmDiagnostic::unpackSrcMgr(di)), - - _ => UnknownDiagnostic(di), - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs deleted file mode 100644 index 3139f93bfef..00000000000 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ /dev/null @@ -1,2547 +0,0 @@ -#![allow(non_camel_case_types)] -#![allow(non_upper_case_globals)] - -use rustc_codegen_ssa::coverageinfo::map as coverage_map; - -use super::debuginfo::{ - DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, - DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DILocation, DINameSpace, - DISPFlags, DIScope, DISubprogram, DISubrange, DITemplateTypeParameter, DIType, DIVariable, - DebugEmissionKind, -}; - -use libc::{c_char, c_int, c_uint, size_t}; -use libc::{c_ulonglong, c_void}; - -use std::marker::PhantomData; - -use super::RustString; - -pub type Bool = c_uint; - -pub const True: Bool = 1 as Bool; -pub const False: Bool = 0 as Bool; - -#[derive(Copy, Clone, PartialEq)] -#[repr(C)] -#[allow(dead_code)] // Variants constructed by C++. -pub enum LLVMRustResult { - Success, - Failure, -} - -// Rust version of the C struct with the same name in rustc_llvm/llvm-wrapper/RustWrapper.cpp. -#[repr(C)] -pub struct LLVMRustCOFFShortExport { - pub name: *const c_char, - pub ordinal_present: bool, - // value of `ordinal` only important when `ordinal_present` is true - pub ordinal: u16, -} - -impl LLVMRustCOFFShortExport { - pub fn new(name: *const c_char, ordinal: Option) -> LLVMRustCOFFShortExport { - LLVMRustCOFFShortExport { - name, - ordinal_present: ordinal.is_some(), - ordinal: ordinal.unwrap_or(0), - } - } -} - -/// Translation of LLVM's MachineTypes enum, defined in llvm\include\llvm\BinaryFormat\COFF.h. -/// -/// We include only architectures supported on Windows. -#[derive(Copy, Clone, PartialEq)] -#[repr(C)] -pub enum LLVMMachineType { - AMD64 = 0x8664, - I386 = 0x14c, - ARM64 = 0xaa64, - ARM = 0x01c0, -} - -/// LLVM's Module::ModFlagBehavior, defined in llvm/include/llvm/IR/Module.h. -/// -/// When merging modules (e.g. during LTO), their metadata flags are combined. Conflicts are -/// resolved according to the merge behaviors specified here. Flags differing only in merge -/// behavior are still considered to be in conflict. -/// -/// In order for Rust-C LTO to work, we must specify behaviors compatible with Clang. Notably, -/// 'Error' and 'Warning' cannot be mixed for a given flag. -#[derive(Copy, Clone, PartialEq)] -#[repr(C)] -pub enum LLVMModFlagBehavior { - Error = 1, - Warning = 2, - Require = 3, - Override = 4, - Append = 5, - AppendUnique = 6, - Max = 7, -} - -// Consts for the LLVM CallConv type, pre-cast to usize. - -/// LLVM CallingConv::ID. Should we wrap this? -#[derive(Copy, Clone, PartialEq, Debug)] -#[repr(C)] -pub enum CallConv { - CCallConv = 0, - FastCallConv = 8, - ColdCallConv = 9, - X86StdcallCallConv = 64, - X86FastcallCallConv = 65, - ArmAapcsCallConv = 67, - Msp430Intr = 69, - X86_ThisCall = 70, - PtxKernel = 71, - X86_64_SysV = 78, - X86_64_Win64 = 79, - X86_VectorCall = 80, - X86_Intr = 83, - AvrNonBlockingInterrupt = 84, - AvrInterrupt = 85, - AmdGpuKernel = 91, -} - -/// LLVMRustLinkage -#[derive(Copy, Clone, PartialEq)] -#[repr(C)] -pub enum Linkage { - ExternalLinkage = 0, - AvailableExternallyLinkage = 1, - LinkOnceAnyLinkage = 2, - LinkOnceODRLinkage = 3, - WeakAnyLinkage = 4, - WeakODRLinkage = 5, - AppendingLinkage = 6, - InternalLinkage = 7, - PrivateLinkage = 8, - ExternalWeakLinkage = 9, - CommonLinkage = 10, -} - -// LLVMRustVisibility -#[repr(C)] -#[derive(Copy, Clone, PartialEq)] -pub enum Visibility { - Default = 0, - Hidden = 1, - Protected = 2, -} - -/// LLVMUnnamedAddr -#[repr(C)] -pub enum UnnamedAddr { - No, - Local, - Global, -} - -/// LLVMDLLStorageClass -#[derive(Copy, Clone)] -#[repr(C)] -pub enum DLLStorageClass { - #[allow(dead_code)] - Default = 0, - DllImport = 1, // Function to be imported from DLL. - #[allow(dead_code)] - DllExport = 2, // Function to be accessible from DLL. -} - -/// Matches LLVMRustAttribute in LLVMWrapper.h -/// Semantically a subset of the C++ enum llvm::Attribute::AttrKind, -/// though it is not ABI compatible (since it's a C++ enum) -#[repr(C)] -#[derive(Copy, Clone, Debug)] -pub enum AttributeKind { - AlwaysInline = 0, - ByVal = 1, - Cold = 2, - InlineHint = 3, - MinSize = 4, - Naked = 5, - NoAlias = 6, - NoCapture = 7, - NoInline = 8, - NonNull = 9, - NoRedZone = 10, - NoReturn = 11, - NoUnwind = 12, - OptimizeForSize = 13, - ReadOnly = 14, - SExt = 15, - StructRet = 16, - UWTable = 17, - ZExt = 18, - InReg = 19, - SanitizeThread = 20, - SanitizeAddress = 21, - SanitizeMemory = 22, - NonLazyBind = 23, - OptimizeNone = 24, - ReturnsTwice = 25, - ReadNone = 26, - InaccessibleMemOnly = 27, - SanitizeHWAddress = 28, - WillReturn = 29, - StackProtectReq = 30, - StackProtectStrong = 31, - StackProtect = 32, - NoUndef = 33, - SanitizeMemTag = 34, - NoCfCheck = 35, - ShadowCallStack = 36, - AllocSize = 37, - AllocatedPointer = 38, - AllocAlign = 39, -} - -/// LLVMIntPredicate -#[derive(Copy, Clone)] -#[repr(C)] -pub enum IntPredicate { - IntEQ = 32, - IntNE = 33, - IntUGT = 34, - IntUGE = 35, - IntULT = 36, - IntULE = 37, - IntSGT = 38, - IntSGE = 39, - IntSLT = 40, - IntSLE = 41, -} - -impl IntPredicate { - pub fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self { - match intpre { - rustc_codegen_ssa::common::IntPredicate::IntEQ => IntPredicate::IntEQ, - rustc_codegen_ssa::common::IntPredicate::IntNE => IntPredicate::IntNE, - rustc_codegen_ssa::common::IntPredicate::IntUGT => IntPredicate::IntUGT, - rustc_codegen_ssa::common::IntPredicate::IntUGE => IntPredicate::IntUGE, - rustc_codegen_ssa::common::IntPredicate::IntULT => IntPredicate::IntULT, - rustc_codegen_ssa::common::IntPredicate::IntULE => IntPredicate::IntULE, - rustc_codegen_ssa::common::IntPredicate::IntSGT => IntPredicate::IntSGT, - rustc_codegen_ssa::common::IntPredicate::IntSGE => IntPredicate::IntSGE, - rustc_codegen_ssa::common::IntPredicate::IntSLT => IntPredicate::IntSLT, - rustc_codegen_ssa::common::IntPredicate::IntSLE => IntPredicate::IntSLE, - } - } -} - -/// LLVMRealPredicate -#[derive(Copy, Clone)] -#[repr(C)] -pub enum RealPredicate { - RealPredicateFalse = 0, - RealOEQ = 1, - RealOGT = 2, - RealOGE = 3, - RealOLT = 4, - RealOLE = 5, - RealONE = 6, - RealORD = 7, - RealUNO = 8, - RealUEQ = 9, - RealUGT = 10, - RealUGE = 11, - RealULT = 12, - RealULE = 13, - RealUNE = 14, - RealPredicateTrue = 15, -} - -impl RealPredicate { - pub fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self { - match realp { - rustc_codegen_ssa::common::RealPredicate::RealPredicateFalse => { - RealPredicate::RealPredicateFalse - } - rustc_codegen_ssa::common::RealPredicate::RealOEQ => RealPredicate::RealOEQ, - rustc_codegen_ssa::common::RealPredicate::RealOGT => RealPredicate::RealOGT, - rustc_codegen_ssa::common::RealPredicate::RealOGE => RealPredicate::RealOGE, - rustc_codegen_ssa::common::RealPredicate::RealOLT => RealPredicate::RealOLT, - rustc_codegen_ssa::common::RealPredicate::RealOLE => RealPredicate::RealOLE, - rustc_codegen_ssa::common::RealPredicate::RealONE => RealPredicate::RealONE, - rustc_codegen_ssa::common::RealPredicate::RealORD => RealPredicate::RealORD, - rustc_codegen_ssa::common::RealPredicate::RealUNO => RealPredicate::RealUNO, - rustc_codegen_ssa::common::RealPredicate::RealUEQ => RealPredicate::RealUEQ, - rustc_codegen_ssa::common::RealPredicate::RealUGT => RealPredicate::RealUGT, - rustc_codegen_ssa::common::RealPredicate::RealUGE => RealPredicate::RealUGE, - rustc_codegen_ssa::common::RealPredicate::RealULT => RealPredicate::RealULT, - rustc_codegen_ssa::common::RealPredicate::RealULE => RealPredicate::RealULE, - rustc_codegen_ssa::common::RealPredicate::RealUNE => RealPredicate::RealUNE, - rustc_codegen_ssa::common::RealPredicate::RealPredicateTrue => { - RealPredicate::RealPredicateTrue - } - } - } -} - -/// LLVMTypeKind -#[derive(Copy, Clone, PartialEq, Debug)] -#[repr(C)] -pub enum TypeKind { - Void = 0, - Half = 1, - Float = 2, - Double = 3, - X86_FP80 = 4, - FP128 = 5, - PPC_FP128 = 6, - Label = 7, - Integer = 8, - Function = 9, - Struct = 10, - Array = 11, - Pointer = 12, - Vector = 13, - Metadata = 14, - X86_MMX = 15, - Token = 16, - ScalableVector = 17, - BFloat = 18, - X86_AMX = 19, -} - -impl TypeKind { - pub fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind { - match self { - TypeKind::Void => rustc_codegen_ssa::common::TypeKind::Void, - TypeKind::Half => rustc_codegen_ssa::common::TypeKind::Half, - TypeKind::Float => rustc_codegen_ssa::common::TypeKind::Float, - TypeKind::Double => rustc_codegen_ssa::common::TypeKind::Double, - TypeKind::X86_FP80 => rustc_codegen_ssa::common::TypeKind::X86_FP80, - TypeKind::FP128 => rustc_codegen_ssa::common::TypeKind::FP128, - TypeKind::PPC_FP128 => rustc_codegen_ssa::common::TypeKind::PPC_FP128, - TypeKind::Label => rustc_codegen_ssa::common::TypeKind::Label, - TypeKind::Integer => rustc_codegen_ssa::common::TypeKind::Integer, - TypeKind::Function => rustc_codegen_ssa::common::TypeKind::Function, - TypeKind::Struct => rustc_codegen_ssa::common::TypeKind::Struct, - TypeKind::Array => rustc_codegen_ssa::common::TypeKind::Array, - TypeKind::Pointer => rustc_codegen_ssa::common::TypeKind::Pointer, - TypeKind::Vector => rustc_codegen_ssa::common::TypeKind::Vector, - TypeKind::Metadata => rustc_codegen_ssa::common::TypeKind::Metadata, - TypeKind::X86_MMX => rustc_codegen_ssa::common::TypeKind::X86_MMX, - TypeKind::Token => rustc_codegen_ssa::common::TypeKind::Token, - TypeKind::ScalableVector => rustc_codegen_ssa::common::TypeKind::ScalableVector, - TypeKind::BFloat => rustc_codegen_ssa::common::TypeKind::BFloat, - TypeKind::X86_AMX => rustc_codegen_ssa::common::TypeKind::X86_AMX, - } - } -} - -/// LLVMAtomicRmwBinOp -#[derive(Copy, Clone)] -#[repr(C)] -pub enum AtomicRmwBinOp { - AtomicXchg = 0, - AtomicAdd = 1, - AtomicSub = 2, - AtomicAnd = 3, - AtomicNand = 4, - AtomicOr = 5, - AtomicXor = 6, - AtomicMax = 7, - AtomicMin = 8, - AtomicUMax = 9, - AtomicUMin = 10, -} - -impl AtomicRmwBinOp { - pub fn from_generic(op: rustc_codegen_ssa::common::AtomicRmwBinOp) -> Self { - match op { - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg => AtomicRmwBinOp::AtomicXchg, - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicAdd => AtomicRmwBinOp::AtomicAdd, - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicSub => AtomicRmwBinOp::AtomicSub, - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicAnd => AtomicRmwBinOp::AtomicAnd, - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicNand => AtomicRmwBinOp::AtomicNand, - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicOr => AtomicRmwBinOp::AtomicOr, - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXor => AtomicRmwBinOp::AtomicXor, - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicMax => AtomicRmwBinOp::AtomicMax, - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicMin => AtomicRmwBinOp::AtomicMin, - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicUMax => AtomicRmwBinOp::AtomicUMax, - rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicUMin => AtomicRmwBinOp::AtomicUMin, - } - } -} - -/// LLVMAtomicOrdering -#[derive(Copy, Clone)] -#[repr(C)] -pub enum AtomicOrdering { - #[allow(dead_code)] - NotAtomic = 0, - Unordered = 1, - Monotonic = 2, - // Consume = 3, // Not specified yet. - Acquire = 4, - Release = 5, - AcquireRelease = 6, - SequentiallyConsistent = 7, -} - -impl AtomicOrdering { - pub fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { - match ao { - rustc_codegen_ssa::common::AtomicOrdering::Unordered => AtomicOrdering::Unordered, - rustc_codegen_ssa::common::AtomicOrdering::Relaxed => AtomicOrdering::Monotonic, - rustc_codegen_ssa::common::AtomicOrdering::Acquire => AtomicOrdering::Acquire, - rustc_codegen_ssa::common::AtomicOrdering::Release => AtomicOrdering::Release, - rustc_codegen_ssa::common::AtomicOrdering::AcquireRelease => { - AtomicOrdering::AcquireRelease - } - rustc_codegen_ssa::common::AtomicOrdering::SequentiallyConsistent => { - AtomicOrdering::SequentiallyConsistent - } - } - } -} - -/// LLVMRustSynchronizationScope -#[derive(Copy, Clone)] -#[repr(C)] -pub enum SynchronizationScope { - SingleThread, - CrossThread, -} - -impl SynchronizationScope { - pub fn from_generic(sc: rustc_codegen_ssa::common::SynchronizationScope) -> Self { - match sc { - rustc_codegen_ssa::common::SynchronizationScope::SingleThread => { - SynchronizationScope::SingleThread - } - rustc_codegen_ssa::common::SynchronizationScope::CrossThread => { - SynchronizationScope::CrossThread - } - } - } -} - -/// LLVMRustFileType -#[derive(Copy, Clone)] -#[repr(C)] -pub enum FileType { - AssemblyFile, - ObjectFile, -} - -/// LLVMMetadataType -#[derive(Copy, Clone)] -#[repr(C)] -pub enum MetadataType { - MD_dbg = 0, - MD_tbaa = 1, - MD_prof = 2, - MD_fpmath = 3, - MD_range = 4, - MD_tbaa_struct = 5, - MD_invariant_load = 6, - MD_alias_scope = 7, - MD_noalias = 8, - MD_nontemporal = 9, - MD_mem_parallel_loop_access = 10, - MD_nonnull = 11, - MD_align = 17, - MD_type = 19, - MD_vcall_visibility = 28, - MD_noundef = 29, -} - -/// LLVMRustAsmDialect -#[derive(Copy, Clone, PartialEq)] -#[repr(C)] -pub enum AsmDialect { - Att, - Intel, -} - -/// LLVMRustCodeGenOptLevel -#[derive(Copy, Clone, PartialEq)] -#[repr(C)] -pub enum CodeGenOptLevel { - None, - Less, - Default, - Aggressive, -} - -/// LLVMRustPassBuilderOptLevel -#[repr(C)] -pub enum PassBuilderOptLevel { - O0, - O1, - O2, - O3, - Os, - Oz, -} - -/// LLVMRustOptStage -#[derive(PartialEq)] -#[repr(C)] -pub enum OptStage { - PreLinkNoLTO, - PreLinkThinLTO, - PreLinkFatLTO, - ThinLTO, - FatLTO, -} - -/// LLVMRustSanitizerOptions -#[repr(C)] -pub struct SanitizerOptions { - pub sanitize_address: bool, - pub sanitize_address_recover: bool, - pub sanitize_memory: bool, - pub sanitize_memory_recover: bool, - pub sanitize_memory_track_origins: c_int, - pub sanitize_thread: bool, - pub sanitize_hwaddress: bool, - pub sanitize_hwaddress_recover: bool, -} - -/// LLVMRelocMode -#[derive(Copy, Clone, PartialEq)] -#[repr(C)] -pub enum RelocModel { - Static, - PIC, - DynamicNoPic, - ROPI, - RWPI, - ROPI_RWPI, -} - -/// LLVMRustCodeModel -#[derive(Copy, Clone)] -#[repr(C)] -pub enum CodeModel { - Tiny, - Small, - Kernel, - Medium, - Large, - None, -} - -/// LLVMRustDiagnosticKind -#[derive(Copy, Clone)] -#[repr(C)] -#[allow(dead_code)] // Variants constructed by C++. -pub enum DiagnosticKind { - Other, - InlineAsm, - StackSize, - DebugMetadataVersion, - SampleProfile, - OptimizationRemark, - OptimizationRemarkMissed, - OptimizationRemarkAnalysis, - OptimizationRemarkAnalysisFPCommute, - OptimizationRemarkAnalysisAliasing, - OptimizationRemarkOther, - OptimizationFailure, - PGOProfile, - Linker, - Unsupported, - SrcMgr, -} - -/// LLVMRustDiagnosticLevel -#[derive(Copy, Clone)] -#[repr(C)] -#[allow(dead_code)] // Variants constructed by C++. -pub enum DiagnosticLevel { - Error, - Warning, - Note, - Remark, -} - -/// LLVMRustArchiveKind -#[derive(Copy, Clone)] -#[repr(C)] -pub enum ArchiveKind { - K_GNU, - K_BSD, - K_DARWIN, - K_COFF, -} - -// LLVMRustThinLTOData -extern "C" { - pub type ThinLTOData; -} - -// LLVMRustThinLTOBuffer -extern "C" { - pub type ThinLTOBuffer; -} - -/// LLVMRustThinLTOModule -#[repr(C)] -pub struct ThinLTOModule { - pub identifier: *const c_char, - pub data: *const u8, - pub len: usize, -} - -/// LLVMThreadLocalMode -#[derive(Copy, Clone)] -#[repr(C)] -pub enum ThreadLocalMode { - NotThreadLocal, - GeneralDynamic, - LocalDynamic, - InitialExec, - LocalExec, -} - -/// LLVMRustChecksumKind -#[derive(Copy, Clone)] -#[repr(C)] -pub enum ChecksumKind { - None, - MD5, - SHA1, - SHA256, -} - -extern "C" { - type Opaque; -} -#[repr(C)] -struct InvariantOpaque<'a> { - _marker: PhantomData<&'a mut &'a ()>, - _opaque: Opaque, -} - -// Opaque pointer types -extern "C" { - pub type Module; -} -extern "C" { - pub type Context; -} -extern "C" { - pub type Type; -} -extern "C" { - pub type Value; -} -extern "C" { - pub type ConstantInt; -} -extern "C" { - pub type Attribute; -} -extern "C" { - pub type Metadata; -} -extern "C" { - pub type BasicBlock; -} -#[repr(C)] -pub struct Builder<'a>(InvariantOpaque<'a>); -#[repr(C)] -pub struct PassManager<'a>(InvariantOpaque<'a>); -extern "C" { - pub type PassManagerBuilder; -} -extern "C" { - pub type Pass; -} -extern "C" { - pub type TargetMachine; -} -extern "C" { - pub type Archive; -} -#[repr(C)] -pub struct ArchiveIterator<'a>(InvariantOpaque<'a>); -#[repr(C)] -pub struct ArchiveChild<'a>(InvariantOpaque<'a>); -extern "C" { - pub type Twine; -} -extern "C" { - pub type DiagnosticInfo; -} -extern "C" { - pub type SMDiagnostic; -} -#[repr(C)] -pub struct RustArchiveMember<'a>(InvariantOpaque<'a>); -#[repr(C)] -pub struct OperandBundleDef<'a>(InvariantOpaque<'a>); -#[repr(C)] -pub struct Linker<'a>(InvariantOpaque<'a>); - -extern "C" { - pub type DiagnosticHandler; -} - -pub type DiagnosticHandlerTy = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void); -pub type InlineAsmDiagHandlerTy = unsafe extern "C" fn(&SMDiagnostic, *const c_void, c_uint); - -pub mod coverageinfo { - use super::coverage_map; - - /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L209-L230) - #[derive(Copy, Clone, Debug)] - #[repr(C)] - pub enum RegionKind { - /// A CodeRegion associates some code with a counter - CodeRegion = 0, - - /// An ExpansionRegion represents a file expansion region that associates - /// a source range with the expansion of a virtual source file, such as - /// for a macro instantiation or #include file. - ExpansionRegion = 1, - - /// A SkippedRegion represents a source range with code that was skipped - /// by a preprocessor or similar means. - SkippedRegion = 2, - - /// A GapRegion is like a CodeRegion, but its count is only set as the - /// line execution count when its the only region in the line. - GapRegion = 3, - - /// A BranchRegion represents leaf-level boolean expressions and is - /// associated with two counters, each representing the number of times the - /// expression evaluates to true or false. - BranchRegion = 4, - } - - /// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the - /// coverage map, in accordance with the - /// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). - /// The struct composes fields representing the `Counter` type and value(s) (injected counter - /// ID, or expression type and operands), the source file (an indirect index into a "filenames - /// array", encoded separately), and source location (start and end positions of the represented - /// code region). - /// - /// Matches LLVMRustCounterMappingRegion. - #[derive(Copy, Clone, Debug)] - #[repr(C)] - pub struct CounterMappingRegion { - /// The counter type and type-dependent counter data, if any. - counter: coverage_map::Counter, - - /// If the `RegionKind` is a `BranchRegion`, this represents the counter - /// for the false branch of the region. - false_counter: coverage_map::Counter, - - /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the - /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes - /// that, in turn, are used to look up the filename for this region. - file_id: u32, - - /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find - /// the mapping regions created as a result of macro expansion, by checking if their file id - /// matches the expanded file id. - expanded_file_id: u32, - - /// 1-based starting line of the mapping region. - start_line: u32, - - /// 1-based starting column of the mapping region. - start_col: u32, - - /// 1-based ending line of the mapping region. - end_line: u32, - - /// 1-based ending column of the mapping region. If the high bit is set, the current - /// mapping region is a gap area. - end_col: u32, - - kind: RegionKind, - } - - impl CounterMappingRegion { - pub(crate) fn code_region( - counter: coverage_map::Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: coverage_map::Counter::zero(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::CodeRegion, - } - } - - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn branch_region( - counter: coverage_map::Counter, - false_counter: coverage_map::Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter, - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::BranchRegion, - } - } - - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn expansion_region( - file_id: u32, - expanded_file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: coverage_map::Counter::zero(), - false_counter: coverage_map::Counter::zero(), - file_id, - expanded_file_id, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::ExpansionRegion, - } - } - - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn skipped_region( - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: coverage_map::Counter::zero(), - false_counter: coverage_map::Counter::zero(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::SkippedRegion, - } - } - - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn gap_region( - counter: coverage_map::Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: coverage_map::Counter::zero(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col: (1_u32 << 31) | end_col, - kind: RegionKind::GapRegion, - } - } - } -} - -pub mod debuginfo { - use super::{InvariantOpaque, Metadata}; - use bitflags::bitflags; - - #[repr(C)] - pub struct DIBuilder<'a>(InvariantOpaque<'a>); - - pub type DIDescriptor = Metadata; - pub type DILocation = Metadata; - pub type DIScope = DIDescriptor; - pub type DIFile = DIScope; - pub type DILexicalBlock = DIScope; - pub type DISubprogram = DIScope; - pub type DINameSpace = DIScope; - pub type DIType = DIDescriptor; - pub type DIBasicType = DIType; - pub type DIDerivedType = DIType; - pub type DICompositeType = DIDerivedType; - pub type DIVariable = DIDescriptor; - pub type DIGlobalVariableExpression = DIDescriptor; - pub type DIArray = DIDescriptor; - pub type DISubrange = DIDescriptor; - pub type DIEnumerator = DIDescriptor; - pub type DITemplateTypeParameter = DIDescriptor; - - // These values **must** match with LLVMRustDIFlags!! - bitflags! { - #[repr(transparent)] - #[derive(Default)] - pub struct DIFlags: u32 { - const FlagZero = 0; - const FlagPrivate = 1; - const FlagProtected = 2; - const FlagPublic = 3; - const FlagFwdDecl = (1 << 2); - const FlagAppleBlock = (1 << 3); - const FlagBlockByrefStruct = (1 << 4); - const FlagVirtual = (1 << 5); - const FlagArtificial = (1 << 6); - const FlagExplicit = (1 << 7); - const FlagPrototyped = (1 << 8); - const FlagObjcClassComplete = (1 << 9); - const FlagObjectPointer = (1 << 10); - const FlagVector = (1 << 11); - const FlagStaticMember = (1 << 12); - const FlagLValueReference = (1 << 13); - const FlagRValueReference = (1 << 14); - const FlagExternalTypeRef = (1 << 15); - const FlagIntroducedVirtual = (1 << 18); - const FlagBitField = (1 << 19); - const FlagNoReturn = (1 << 20); - } - } - - // These values **must** match with LLVMRustDISPFlags!! - bitflags! { - #[repr(transparent)] - #[derive(Default)] - pub struct DISPFlags: u32 { - const SPFlagZero = 0; - const SPFlagVirtual = 1; - const SPFlagPureVirtual = 2; - const SPFlagLocalToUnit = (1 << 2); - const SPFlagDefinition = (1 << 3); - const SPFlagOptimized = (1 << 4); - const SPFlagMainSubprogram = (1 << 5); - } - } - - /// LLVMRustDebugEmissionKind - #[derive(Copy, Clone)] - #[repr(C)] - pub enum DebugEmissionKind { - NoDebug, - FullDebug, - LineTablesOnly, - } - - impl DebugEmissionKind { - pub fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { - use rustc_session::config::DebugInfo; - match kind { - DebugInfo::None => DebugEmissionKind::NoDebug, - DebugInfo::Limited => DebugEmissionKind::LineTablesOnly, - DebugInfo::Full => DebugEmissionKind::FullDebug, - } - } - } -} - -use bitflags::bitflags; -// These values **must** match with LLVMRustAllocKindFlags -bitflags! { - #[repr(transparent)] - #[derive(Default)] - pub struct AllocKindFlags : u64 { - const Unknown = 0; - const Alloc = 1; - const Realloc = 1 << 1; - const Free = 1 << 2; - const Uninitialized = 1 << 3; - const Zeroed = 1 << 4; - const Aligned = 1 << 5; - } -} - -extern "C" { - pub type ModuleBuffer; -} - -pub type SelfProfileBeforePassCallback = - unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); -pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void); - -extern "C" { - pub fn LLVMRustInstallFatalErrorHandler(); - pub fn LLVMRustDisableSystemDialogsOnCrash(); - - // Create and destroy contexts. - pub fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; - pub fn LLVMContextDispose(C: &'static mut Context); - pub fn LLVMGetMDKindIDInContext(C: &Context, Name: *const c_char, SLen: c_uint) -> c_uint; - - // Create modules. - pub fn LLVMModuleCreateWithNameInContext(ModuleID: *const c_char, C: &Context) -> &Module; - pub fn LLVMGetModuleContext(M: &Module) -> &Context; - pub fn LLVMCloneModule(M: &Module) -> &Module; - - /// Data layout. See Module::getDataLayout. - pub fn LLVMGetDataLayoutStr(M: &Module) -> *const c_char; - pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); - - /// See Module::setModuleInlineAsm. - pub fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t); - - /// See llvm::LLVMTypeKind::getTypeID. - pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; - - // Operations on integer types - pub fn LLVMInt1TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt8TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt16TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt32TypeInContext(C: &Context) -> &Type; - pub fn LLVMInt64TypeInContext(C: &Context) -> &Type; - pub fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type; - - pub fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; - - // Operations on real types - pub fn LLVMFloatTypeInContext(C: &Context) -> &Type; - pub fn LLVMDoubleTypeInContext(C: &Context) -> &Type; - - // Operations on function types - pub fn LLVMFunctionType<'a>( - ReturnType: &'a Type, - ParamTypes: *const &'a Type, - ParamCount: c_uint, - IsVarArg: Bool, - ) -> &'a Type; - pub fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint; - pub fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type); - - // Operations on struct types - pub fn LLVMStructTypeInContext<'a>( - C: &'a Context, - ElementTypes: *const &'a Type, - ElementCount: c_uint, - Packed: Bool, - ) -> &'a Type; - - // Operations on array, pointer, and vector types (sequence types) - pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type; - pub fn LLVMPointerType(ElementType: &Type, AddressSpace: c_uint) -> &Type; - pub fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; - - pub fn LLVMGetElementType(Ty: &Type) -> &Type; - pub fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint; - - // Operations on other types - pub fn LLVMVoidTypeInContext(C: &Context) -> &Type; - pub fn LLVMRustMetadataTypeInContext(C: &Context) -> &Type; - - // Operations on all values - pub fn LLVMTypeOf(Val: &Value) -> &Type; - pub fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char; - pub fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); - pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); - pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); - pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); - pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); - pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; - - // Operations on constants of any type - pub fn LLVMConstNull(Ty: &Type) -> &Value; - pub fn LLVMGetUndef(Ty: &Type) -> &Value; - - // Operations on metadata - pub fn LLVMMDStringInContext(C: &Context, Str: *const c_char, SLen: c_uint) -> &Value; - pub fn LLVMMDNodeInContext<'a>( - C: &'a Context, - Vals: *const &'a Value, - Count: c_uint, - ) -> &'a Value; - pub fn LLVMMDNodeInContext2<'a>( - C: &'a Context, - Vals: *const &'a Metadata, - Count: size_t, - ) -> &'a Metadata; - pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value); - - // Operations on scalar constants - pub fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; - pub fn LLVMConstIntOfArbitraryPrecision(IntTy: &Type, Wn: c_uint, Ws: *const u64) -> &Value; - pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; - pub fn LLVMConstIntGetZExtValue(ConstantVal: &ConstantInt) -> c_ulonglong; - pub fn LLVMRustConstInt128Get( - ConstantVal: &ConstantInt, - SExt: bool, - high: &mut u64, - low: &mut u64, - ) -> bool; - - // Operations on composite constants - pub fn LLVMConstStringInContext( - C: &Context, - Str: *const c_char, - Length: c_uint, - DontNullTerminate: Bool, - ) -> &Value; - pub fn LLVMConstStructInContext<'a>( - C: &'a Context, - ConstantVals: *const &'a Value, - Count: c_uint, - Packed: Bool, - ) -> &'a Value; - - pub fn LLVMConstArray<'a>( - ElementTy: &'a Type, - ConstantVals: *const &'a Value, - Length: c_uint, - ) -> &'a Value; - pub fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; - - // Constant expressions - pub fn LLVMRustConstInBoundsGEP2<'a>( - ty: &'a Type, - ConstantVal: &'a Value, - ConstantIndices: *const &'a Value, - NumIndices: c_uint, - ) -> &'a Value; - pub fn LLVMConstZExt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; - - // Operations on global variables, functions, and aliases (globals) - pub fn LLVMIsDeclaration(Global: &Value) -> Bool; - pub fn LLVMRustGetLinkage(Global: &Value) -> Linkage; - pub fn LLVMRustSetLinkage(Global: &Value, RustLinkage: Linkage); - pub fn LLVMSetSection(Global: &Value, Section: *const c_char); - pub fn LLVMRustGetVisibility(Global: &Value) -> Visibility; - pub fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility); - pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); - pub fn LLVMGetAlignment(Global: &Value) -> c_uint; - pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); - pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); - - // Operations on global variables - pub fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; - pub fn LLVMAddGlobal<'a>(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value; - pub fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>; - pub fn LLVMRustGetOrInsertGlobal<'a>( - M: &'a Module, - Name: *const c_char, - NameLen: size_t, - T: &'a Type, - ) -> &'a Value; - pub fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value; - pub fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>; - pub fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>; - pub fn LLVMDeleteGlobal(GlobalVar: &Value); - pub fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; - pub fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); - pub fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; - pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); - pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; - pub fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); - pub fn LLVMRustGetNamedValue( - M: &Module, - Name: *const c_char, - NameLen: size_t, - ) -> Option<&Value>; - pub fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); - - // Operations on attributes - pub fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute; - pub fn LLVMCreateStringAttribute( - C: &Context, - Name: *const c_char, - NameLen: c_uint, - Value: *const c_char, - ValueLen: c_uint, - ) -> &Attribute; - pub fn LLVMRustCreateAlignmentAttr(C: &Context, bytes: u64) -> &Attribute; - pub fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute; - pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; - pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; - pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; - pub fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; - pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; - pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; - pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; - - // Operations on functions - pub fn LLVMRustGetOrInsertFunction<'a>( - M: &'a Module, - Name: *const c_char, - NameLen: size_t, - FunctionTy: &'a Type, - ) -> &'a Value; - pub fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); - pub fn LLVMRustAddFunctionAttributes<'a>( - Fn: &'a Value, - index: c_uint, - Attrs: *const &'a Attribute, - AttrsLen: size_t, - ); - - // Operations on parameters - pub fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; - pub fn LLVMCountParams(Fn: &Value) -> c_uint; - pub fn LLVMGetParam(Fn: &Value, Index: c_uint) -> &Value; - - // Operations on basic blocks - pub fn LLVMGetBasicBlockParent(BB: &BasicBlock) -> &Value; - pub fn LLVMAppendBasicBlockInContext<'a>( - C: &'a Context, - Fn: &'a Value, - Name: *const c_char, - ) -> &'a BasicBlock; - - // Operations on instructions - pub fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; - pub fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; - - // Operations on call sites - pub fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); - pub fn LLVMRustAddCallSiteAttributes<'a>( - Instr: &'a Value, - index: c_uint, - Attrs: *const &'a Attribute, - AttrsLen: size_t, - ); - - // Operations on load/store instructions (only) - pub fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool); - - // Operations on phi nodes - pub fn LLVMAddIncoming<'a>( - PhiNode: &'a Value, - IncomingValues: *const &'a Value, - IncomingBlocks: *const &'a BasicBlock, - Count: c_uint, - ); - - // Instruction builders - pub fn LLVMCreateBuilderInContext(C: &Context) -> &mut Builder<'_>; - pub fn LLVMPositionBuilderAtEnd<'a>(Builder: &Builder<'a>, Block: &'a BasicBlock); - pub fn LLVMGetInsertBlock<'a>(Builder: &Builder<'a>) -> &'a BasicBlock; - pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); - - // Metadata - pub fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: &'a Value); - - // Terminators - pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; - pub fn LLVMBuildRet<'a>(B: &Builder<'a>, V: &'a Value) -> &'a Value; - pub fn LLVMBuildBr<'a>(B: &Builder<'a>, Dest: &'a BasicBlock) -> &'a Value; - pub fn LLVMBuildCondBr<'a>( - B: &Builder<'a>, - If: &'a Value, - Then: &'a BasicBlock, - Else: &'a BasicBlock, - ) -> &'a Value; - pub fn LLVMBuildSwitch<'a>( - B: &Builder<'a>, - V: &'a Value, - Else: &'a BasicBlock, - NumCases: c_uint, - ) -> &'a Value; - pub fn LLVMRustBuildInvoke<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Fn: &'a Value, - Args: *const &'a Value, - NumArgs: c_uint, - Then: &'a BasicBlock, - Catch: &'a BasicBlock, - Bundle: Option<&OperandBundleDef<'a>>, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildLandingPad<'a>( - B: &Builder<'a>, - Ty: &'a Type, - PersFn: Option<&'a Value>, - NumClauses: c_uint, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; - pub fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; - - pub fn LLVMRustBuildCleanupPad<'a>( - B: &Builder<'a>, - ParentPad: Option<&'a Value>, - ArgCnt: c_uint, - Args: *const &'a Value, - Name: *const c_char, - ) -> Option<&'a Value>; - pub fn LLVMRustBuildCleanupRet<'a>( - B: &Builder<'a>, - CleanupPad: &'a Value, - UnwindBB: Option<&'a BasicBlock>, - ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchPad<'a>( - B: &Builder<'a>, - ParentPad: &'a Value, - ArgCnt: c_uint, - Args: *const &'a Value, - Name: *const c_char, - ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchRet<'a>( - B: &Builder<'a>, - Pad: &'a Value, - BB: &'a BasicBlock, - ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchSwitch<'a>( - Builder: &Builder<'a>, - ParentPad: Option<&'a Value>, - BB: Option<&'a BasicBlock>, - NumHandlers: c_uint, - Name: *const c_char, - ) -> Option<&'a Value>; - pub fn LLVMRustAddHandler<'a>(CatchSwitch: &'a Value, Handler: &'a BasicBlock); - pub fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); - - // Add a case to the switch instruction - pub fn LLVMAddCase<'a>(Switch: &'a Value, OnVal: &'a Value, Dest: &'a BasicBlock); - - // Add a clause to the landing pad instruction - pub fn LLVMAddClause<'a>(LandingPad: &'a Value, ClauseVal: &'a Value); - - // Set the cleanup on a landing pad instruction - pub fn LLVMSetCleanup(LandingPad: &Value, Val: Bool); - - // Arithmetic - pub fn LLVMBuildAdd<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildFAdd<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildSub<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildFSub<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildMul<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildFMul<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildUDiv<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildExactUDiv<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildSDiv<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildExactSDiv<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildFDiv<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildURem<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildSRem<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildFRem<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildShl<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildLShr<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildAShr<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildNSWAdd<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildNUWAdd<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildNSWSub<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildNUWSub<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildNSWMul<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildNUWMul<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildAnd<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildOr<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildXor<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildFNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; - pub fn LLVMRustSetFastMath(Instr: &Value); - - // Memory - pub fn LLVMBuildAlloca<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; - pub fn LLVMBuildArrayAlloca<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Val: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildLoad2<'a>( - B: &Builder<'a>, - Ty: &'a Type, - PointerVal: &'a Value, - Name: *const c_char, - ) -> &'a Value; - - pub fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value; - - pub fn LLVMBuildGEP2<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Pointer: &'a Value, - Indices: *const &'a Value, - NumIndices: c_uint, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildInBoundsGEP2<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Pointer: &'a Value, - Indices: *const &'a Value, - NumIndices: c_uint, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildStructGEP2<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Pointer: &'a Value, - Idx: c_uint, - Name: *const c_char, - ) -> &'a Value; - - // Casts - pub fn LLVMBuildTrunc<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildZExt<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildSExt<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildFPToUI<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildFPToSI<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildUIToFP<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildSIToFP<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildFPTrunc<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildFPExt<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildPtrToInt<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildIntToPtr<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildBitCast<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildPointerCast<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMRustBuildIntCast<'a>( - B: &Builder<'a>, - Val: &'a Value, - DestTy: &'a Type, - IsSigned: bool, - ) -> &'a Value; - - // Comparisons - pub fn LLVMBuildICmp<'a>( - B: &Builder<'a>, - Op: c_uint, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildFCmp<'a>( - B: &Builder<'a>, - Op: c_uint, - LHS: &'a Value, - RHS: &'a Value, - Name: *const c_char, - ) -> &'a Value; - - // Miscellaneous instructions - pub fn LLVMBuildPhi<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; - pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustBuildCall<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Fn: &'a Value, - Args: *const &'a Value, - NumArgs: c_uint, - Bundle: Option<&OperandBundleDef<'a>>, - ) -> &'a Value; - pub fn LLVMRustBuildMemCpy<'a>( - B: &Builder<'a>, - Dst: &'a Value, - DstAlign: c_uint, - Src: &'a Value, - SrcAlign: c_uint, - Size: &'a Value, - IsVolatile: bool, - ) -> &'a Value; - pub fn LLVMRustBuildMemMove<'a>( - B: &Builder<'a>, - Dst: &'a Value, - DstAlign: c_uint, - Src: &'a Value, - SrcAlign: c_uint, - Size: &'a Value, - IsVolatile: bool, - ) -> &'a Value; - pub fn LLVMRustBuildMemSet<'a>( - B: &Builder<'a>, - Dst: &'a Value, - DstAlign: c_uint, - Val: &'a Value, - Size: &'a Value, - IsVolatile: bool, - ) -> &'a Value; - pub fn LLVMBuildSelect<'a>( - B: &Builder<'a>, - If: &'a Value, - Then: &'a Value, - Else: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildVAArg<'a>( - B: &Builder<'a>, - list: &'a Value, - Ty: &'a Type, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildExtractElement<'a>( - B: &Builder<'a>, - VecVal: &'a Value, - Index: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildInsertElement<'a>( - B: &Builder<'a>, - VecVal: &'a Value, - EltVal: &'a Value, - Index: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildShuffleVector<'a>( - B: &Builder<'a>, - V1: &'a Value, - V2: &'a Value, - Mask: &'a Value, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildExtractValue<'a>( - B: &Builder<'a>, - AggVal: &'a Value, - Index: c_uint, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMBuildInsertValue<'a>( - B: &Builder<'a>, - AggVal: &'a Value, - EltVal: &'a Value, - Index: c_uint, - Name: *const c_char, - ) -> &'a Value; - - pub fn LLVMRustBuildVectorReduceFAdd<'a>( - B: &Builder<'a>, - Acc: &'a Value, - Src: &'a Value, - ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFMul<'a>( - B: &Builder<'a>, - Acc: &'a Value, - Src: &'a Value, - ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceAdd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceMul<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceAnd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceOr<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceXor<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub fn LLVMRustBuildVectorReduceMin<'a>( - B: &Builder<'a>, - Src: &'a Value, - IsSigned: bool, - ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceMax<'a>( - B: &Builder<'a>, - Src: &'a Value, - IsSigned: bool, - ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFMin<'a>( - B: &Builder<'a>, - Src: &'a Value, - IsNaN: bool, - ) -> &'a Value; - pub fn LLVMRustBuildVectorReduceFMax<'a>( - B: &Builder<'a>, - Src: &'a Value, - IsNaN: bool, - ) -> &'a Value; - - pub fn LLVMRustBuildMinNum<'a>(B: &Builder<'a>, LHS: &'a Value, LHS: &'a Value) -> &'a Value; - pub fn LLVMRustBuildMaxNum<'a>(B: &Builder<'a>, LHS: &'a Value, LHS: &'a Value) -> &'a Value; - - // Atomic Operations - pub fn LLVMRustBuildAtomicLoad<'a>( - B: &Builder<'a>, - ElementType: &'a Type, - PointerVal: &'a Value, - Name: *const c_char, - Order: AtomicOrdering, - ) -> &'a Value; - - pub fn LLVMRustBuildAtomicStore<'a>( - B: &Builder<'a>, - Val: &'a Value, - Ptr: &'a Value, - Order: AtomicOrdering, - ) -> &'a Value; - - pub fn LLVMRustBuildAtomicCmpXchg<'a>( - B: &Builder<'a>, - LHS: &'a Value, - CMP: &'a Value, - RHS: &'a Value, - Order: AtomicOrdering, - FailureOrder: AtomicOrdering, - Weak: Bool, - ) -> &'a Value; - - pub fn LLVMBuildAtomicRMW<'a>( - B: &Builder<'a>, - Op: AtomicRmwBinOp, - LHS: &'a Value, - RHS: &'a Value, - Order: AtomicOrdering, - SingleThreaded: Bool, - ) -> &'a Value; - - pub fn LLVMRustBuildAtomicFence( - B: &Builder<'_>, - Order: AtomicOrdering, - Scope: SynchronizationScope, - ); - - /// Writes a module to the specified path. Returns 0 on success. - pub fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int; - - /// Creates a pass manager. - pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; - - /// Creates a function-by-function pass manager - pub fn LLVMCreateFunctionPassManagerForModule(M: &Module) -> &mut PassManager<'_>; - - /// Disposes a pass manager. - pub fn LLVMDisposePassManager<'a>(PM: &'a mut PassManager<'a>); - - /// Runs a pass manager on a module. - pub fn LLVMRunPassManager<'a>(PM: &PassManager<'a>, M: &'a Module) -> Bool; - - pub fn LLVMInitializePasses(); - - pub fn LLVMTimeTraceProfilerInitialize(); - - pub fn LLVMTimeTraceProfilerFinishThread(); - - pub fn LLVMTimeTraceProfilerFinish(FileName: *const c_char); - - pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); - - pub fn LLVMRustPassManagerBuilderCreate() -> &'static mut PassManagerBuilder; - pub fn LLVMRustPassManagerBuilderDispose(PMB: &'static mut PassManagerBuilder); - pub fn LLVMRustPassManagerBuilderUseInlinerWithThreshold( - PMB: &PassManagerBuilder, - threshold: c_uint, - ); - pub fn LLVMRustPassManagerBuilderPopulateModulePassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - ); - - pub fn LLVMRustPassManagerBuilderPopulateFunctionPassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - ); - pub fn LLVMRustPassManagerBuilderPopulateLTOPassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - Internalize: Bool, - RunInliner: Bool, - ); - pub fn LLVMRustPassManagerBuilderPopulateThinLTOPassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - ); - - pub fn LLVMGetHostCPUFeatures() -> *mut c_char; - - pub fn LLVMDisposeMessage(message: *mut c_char); - - pub fn LLVMIsMultithreaded() -> Bool; - - /// Returns a string describing the last error caused by an LLVMRust* call. - pub fn LLVMRustGetLastError() -> *const c_char; - - /// Print the pass timings since static dtors aren't picking them up. - pub fn LLVMRustPrintPassTimings(); - - pub fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type; - - pub fn LLVMStructSetBody<'a>( - StructTy: &'a Type, - ElementTypes: *const &'a Type, - ElementCount: c_uint, - Packed: Bool, - ); - - /// Prepares inline assembly. - pub fn LLVMRustInlineAsm( - Ty: &Type, - AsmString: *const c_char, - AsmStringLen: size_t, - Constraints: *const c_char, - ConstraintsLen: size_t, - SideEffects: Bool, - AlignStack: Bool, - Dialect: AsmDialect, - CanThrow: Bool, - ) -> &Value; - pub fn LLVMRustInlineAsmVerify( - Ty: &Type, - Constraints: *const c_char, - ConstraintsLen: size_t, - ) -> bool; - - #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( - Filenames: *const *const c_char, - FilenamesLen: size_t, - BufferOut: &RustString, - ); - - #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMappingToBuffer( - VirtualFileMappingIDs: *const c_uint, - NumVirtualFileMappingIDs: c_uint, - Expressions: *const coverage_map::CounterExpression, - NumExpressions: c_uint, - MappingRegions: *const coverageinfo::CounterMappingRegion, - NumMappingRegions: c_uint, - BufferOut: &RustString, - ); - - pub fn LLVMRustCoverageCreatePGOFuncNameVar(F: &Value, FuncName: *const c_char) -> &Value; - pub fn LLVMRustCoverageHashCString(StrVal: *const c_char) -> u64; - pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; - - #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); - - #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); - - #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); - - pub fn LLVMRustCoverageMappingVersion() -> u32; - pub fn LLVMRustDebugMetadataVersion() -> u32; - pub fn LLVMRustVersionMajor() -> u32; - pub fn LLVMRustVersionMinor() -> u32; - pub fn LLVMRustVersionPatch() -> u32; - - /// Add LLVM module flags. - /// - /// In order for Rust-C LTO to work, module flags must be compatible with Clang. What - /// "compatible" means depends on the merge behaviors involved. - pub fn LLVMRustAddModuleFlag( - M: &Module, - merge_behavior: LLVMModFlagBehavior, - name: *const c_char, - value: u32, - ); - pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; - - pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; - - pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; - - pub fn LLVMRustDIBuilderDispose<'a>(Builder: &'a mut DIBuilder<'a>); - - pub fn LLVMRustDIBuilderFinalize(Builder: &DIBuilder<'_>); - - pub fn LLVMRustDIBuilderCreateCompileUnit<'a>( - Builder: &DIBuilder<'a>, - Lang: c_uint, - File: &'a DIFile, - Producer: *const c_char, - ProducerLen: size_t, - isOptimized: bool, - Flags: *const c_char, - RuntimeVer: c_uint, - SplitName: *const c_char, - SplitNameLen: size_t, - kind: DebugEmissionKind, - DWOId: u64, - SplitDebugInlining: bool, - ) -> &'a DIDescriptor; - - pub fn LLVMRustDIBuilderCreateFile<'a>( - Builder: &DIBuilder<'a>, - Filename: *const c_char, - FilenameLen: size_t, - Directory: *const c_char, - DirectoryLen: size_t, - CSKind: ChecksumKind, - Checksum: *const c_char, - ChecksumLen: size_t, - ) -> &'a DIFile; - - pub fn LLVMRustDIBuilderCreateSubroutineType<'a>( - Builder: &DIBuilder<'a>, - ParameterTypes: &'a DIArray, - ) -> &'a DICompositeType; - - pub fn LLVMRustDIBuilderCreateFunction<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - LinkageName: *const c_char, - LinkageNameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - ScopeLine: c_uint, - Flags: DIFlags, - SPFlags: DISPFlags, - MaybeFn: Option<&'a Value>, - TParam: &'a DIArray, - Decl: Option<&'a DIDescriptor>, - ) -> &'a DISubprogram; - - pub fn LLVMRustDIBuilderCreateBasicType<'a>( - Builder: &DIBuilder<'a>, - Name: *const c_char, - NameLen: size_t, - SizeInBits: u64, - Encoding: c_uint, - ) -> &'a DIBasicType; - - pub fn LLVMRustDIBuilderCreateTypedef<'a>( - Builder: &DIBuilder<'a>, - Type: &'a DIBasicType, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Scope: Option<&'a DIScope>, - ) -> &'a DIDerivedType; - - pub fn LLVMRustDIBuilderCreatePointerType<'a>( - Builder: &DIBuilder<'a>, - PointeeTy: &'a DIType, - SizeInBits: u64, - AlignInBits: u32, - AddressSpace: c_uint, - Name: *const c_char, - NameLen: size_t, - ) -> &'a DIDerivedType; - - pub fn LLVMRustDIBuilderCreateStructType<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIDescriptor>, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNumber: c_uint, - SizeInBits: u64, - AlignInBits: u32, - Flags: DIFlags, - DerivedFrom: Option<&'a DIType>, - Elements: &'a DIArray, - RunTimeLang: c_uint, - VTableHolder: Option<&'a DIType>, - UniqueId: *const c_char, - UniqueIdLen: size_t, - ) -> &'a DICompositeType; - - pub fn LLVMRustDIBuilderCreateMemberType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - SizeInBits: u64, - AlignInBits: u32, - OffsetInBits: u64, - Flags: DIFlags, - Ty: &'a DIType, - ) -> &'a DIDerivedType; - - pub fn LLVMRustDIBuilderCreateVariantMemberType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIScope, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNumber: c_uint, - SizeInBits: u64, - AlignInBits: u32, - OffsetInBits: u64, - Discriminant: Option<&'a Value>, - Flags: DIFlags, - Ty: &'a DIType, - ) -> &'a DIType; - - pub fn LLVMRustDIBuilderCreateLexicalBlock<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIScope, - File: &'a DIFile, - Line: c_uint, - Col: c_uint, - ) -> &'a DILexicalBlock; - - pub fn LLVMRustDIBuilderCreateLexicalBlockFile<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIScope, - File: &'a DIFile, - ) -> &'a DILexicalBlock; - - pub fn LLVMRustDIBuilderCreateStaticVariable<'a>( - Builder: &DIBuilder<'a>, - Context: Option<&'a DIScope>, - Name: *const c_char, - NameLen: size_t, - LinkageName: *const c_char, - LinkageNameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - isLocalToUnit: bool, - Val: &'a Value, - Decl: Option<&'a DIDescriptor>, - AlignInBits: u32, - ) -> &'a DIGlobalVariableExpression; - - pub fn LLVMRustDIBuilderCreateVariable<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - AlwaysPreserve: bool, - Flags: DIFlags, - ArgNo: c_uint, - AlignInBits: u32, - ) -> &'a DIVariable; - - pub fn LLVMRustDIBuilderCreateArrayType<'a>( - Builder: &DIBuilder<'a>, - Size: u64, - AlignInBits: u32, - Ty: &'a DIType, - Subscripts: &'a DIArray, - ) -> &'a DIType; - - pub fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( - Builder: &DIBuilder<'a>, - Lo: i64, - Count: i64, - ) -> &'a DISubrange; - - pub fn LLVMRustDIBuilderGetOrCreateArray<'a>( - Builder: &DIBuilder<'a>, - Ptr: *const Option<&'a DIDescriptor>, - Count: c_uint, - ) -> &'a DIArray; - - pub fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( - Builder: &DIBuilder<'a>, - Val: &'a Value, - VarInfo: &'a DIVariable, - AddrOps: *const u64, - AddrOpsCount: c_uint, - DL: &'a DILocation, - InsertAtEnd: &'a BasicBlock, - ) -> &'a Value; - - pub fn LLVMRustDIBuilderCreateEnumerator<'a>( - Builder: &DIBuilder<'a>, - Name: *const c_char, - NameLen: size_t, - Value: i64, - IsUnsigned: bool, - ) -> &'a DIEnumerator; - - pub fn LLVMRustDIBuilderCreateEnumerationType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIScope, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNumber: c_uint, - SizeInBits: u64, - AlignInBits: u32, - Elements: &'a DIArray, - ClassType: &'a DIType, - IsScoped: bool, - ) -> &'a DIType; - - pub fn LLVMRustDIBuilderCreateUnionType<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIScope>, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNumber: c_uint, - SizeInBits: u64, - AlignInBits: u32, - Flags: DIFlags, - Elements: Option<&'a DIArray>, - RunTimeLang: c_uint, - UniqueId: *const c_char, - UniqueIdLen: size_t, - ) -> &'a DIType; - - pub fn LLVMRustDIBuilderCreateVariantPart<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIScope, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - SizeInBits: u64, - AlignInBits: u32, - Flags: DIFlags, - Discriminator: Option<&'a DIDerivedType>, - Elements: &'a DIArray, - UniqueId: *const c_char, - UniqueIdLen: size_t, - ) -> &'a DIDerivedType; - - pub fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); - - pub fn LLVMRustDIBuilderCreateTemplateTypeParameter<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIScope>, - Name: *const c_char, - NameLen: size_t, - Ty: &'a DIType, - ) -> &'a DITemplateTypeParameter; - - pub fn LLVMRustDIBuilderCreateNameSpace<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIScope>, - Name: *const c_char, - NameLen: size_t, - ExportSymbols: bool, - ) -> &'a DINameSpace; - - pub fn LLVMRustDICompositeTypeReplaceArrays<'a>( - Builder: &DIBuilder<'a>, - CompositeType: &'a DIType, - Elements: Option<&'a DIArray>, - Params: Option<&'a DIArray>, - ); - - pub fn LLVMRustDIBuilderCreateDebugLocation<'a>( - Line: c_uint, - Column: c_uint, - Scope: &'a DIScope, - InlinedAt: Option<&'a DILocation>, - ) -> &'a DILocation; - pub fn LLVMRustDIBuilderCreateOpDeref() -> u64; - pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> u64; - - #[allow(improper_ctypes)] - pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); - #[allow(improper_ctypes)] - pub fn LLVMRustWriteValueToString(value_ref: &Value, s: &RustString); - - pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; - - pub fn LLVMRustFindAndCreatePass(Pass: *const c_char) -> Option<&'static mut Pass>; - pub fn LLVMRustCreateAddressSanitizerFunctionPass(Recover: bool) -> &'static mut Pass; - pub fn LLVMRustCreateModuleAddressSanitizerPass(Recover: bool) -> &'static mut Pass; - pub fn LLVMRustCreateMemorySanitizerPass( - TrackOrigins: c_int, - Recover: bool, - ) -> &'static mut Pass; - pub fn LLVMRustCreateThreadSanitizerPass() -> &'static mut Pass; - pub fn LLVMRustCreateHWAddressSanitizerPass(Recover: bool) -> &'static mut Pass; - pub fn LLVMRustAddPass(PM: &PassManager<'_>, Pass: &'static mut Pass); - pub fn LLVMRustAddLastExtensionPasses( - PMB: &PassManagerBuilder, - Passes: *const &'static mut Pass, - NumPasses: size_t, - ); - - pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; - - pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine); - pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; - pub fn LLVMRustGetTargetFeature( - T: &TargetMachine, - Index: size_t, - Feature: &mut *const c_char, - Desc: &mut *const c_char, - ); - - pub fn LLVMRustGetHostCPUName(len: *mut usize) -> *const c_char; - pub fn LLVMRustCreateTargetMachine( - Triple: *const c_char, - CPU: *const c_char, - Features: *const c_char, - Abi: *const c_char, - Model: CodeModel, - Reloc: RelocModel, - Level: CodeGenOptLevel, - UseSoftFP: bool, - FunctionSections: bool, - DataSections: bool, - UniqueSectionNames: bool, - TrapUnreachable: bool, - Singlethread: bool, - AsmComments: bool, - EmitStackSizeSection: bool, - RelaxELFRelocations: bool, - UseInitArray: bool, - SplitDwarfFile: *const c_char, - ) -> Option<&'static mut TargetMachine>; - pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine); - pub fn LLVMRustAddBuilderLibraryInfo<'a>( - PMB: &'a PassManagerBuilder, - M: &'a Module, - DisableSimplifyLibCalls: bool, - ); - pub fn LLVMRustConfigurePassManagerBuilder( - PMB: &PassManagerBuilder, - OptLevel: CodeGenOptLevel, - MergeFunctions: bool, - SLPVectorize: bool, - LoopVectorize: bool, - PrepareForThinLTO: bool, - PGOGenPath: *const c_char, - PGOUsePath: *const c_char, - PGOSampleUsePath: *const c_char, - SizeLevel: c_int, - ); - pub fn LLVMRustAddLibraryInfo<'a>( - PM: &PassManager<'a>, - M: &'a Module, - DisableSimplifyLibCalls: bool, - ); - pub fn LLVMRustRunFunctionPassManager<'a>(PM: &PassManager<'a>, M: &'a Module); - pub fn LLVMRustWriteOutputFile<'a>( - T: &'a TargetMachine, - PM: &PassManager<'a>, - M: &'a Module, - Output: *const c_char, - DwoOutput: *const c_char, - FileType: FileType, - ) -> LLVMRustResult; - pub fn LLVMRustOptimizeWithNewPassManager<'a>( - M: &'a Module, - TM: &'a TargetMachine, - OptLevel: PassBuilderOptLevel, - OptStage: OptStage, - NoPrepopulatePasses: bool, - VerifyIR: bool, - UseThinLTOBuffers: bool, - MergeFunctions: bool, - UnrollLoops: bool, - SLPVectorize: bool, - LoopVectorize: bool, - DisableSimplifyLibCalls: bool, - EmitLifetimeMarkers: bool, - SanitizerOptions: Option<&SanitizerOptions>, - PGOGenPath: *const c_char, - PGOUsePath: *const c_char, - InstrumentCoverage: bool, - InstrumentGCOV: bool, - PGOSampleUsePath: *const c_char, - DebugInfoForProfiling: bool, - llvm_selfprofiler: *mut c_void, - begin_callback: SelfProfileBeforePassCallback, - end_callback: SelfProfileAfterPassCallback, - ExtraPasses: *const c_char, - ExtraPassesLen: size_t, - LLVMPlugins: *const c_char, - LLVMPluginsLen: size_t, - ) -> LLVMRustResult; - pub fn LLVMRustPrintModule( - M: &Module, - Output: *const c_char, - Demangle: extern "C" fn(*const c_char, size_t, *mut c_char, size_t) -> size_t, - ) -> LLVMRustResult; - pub fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); - pub fn LLVMRustPrintPasses(); - pub fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); - pub fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool); - pub fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); - - pub fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; - pub fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; - pub fn LLVMRustArchiveIteratorNext<'a>( - AIR: &ArchiveIterator<'a>, - ) -> Option<&'a mut ArchiveChild<'a>>; - pub fn LLVMRustArchiveChildName(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; - pub fn LLVMRustArchiveChildData(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; - pub fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); - pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); - pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); - - #[allow(improper_ctypes)] - pub fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); - - #[allow(improper_ctypes)] - pub fn LLVMRustUnpackOptimizationDiagnostic<'a>( - DI: &'a DiagnosticInfo, - pass_name_out: &RustString, - function_out: &mut Option<&'a Value>, - loc_line_out: &mut c_uint, - loc_column_out: &mut c_uint, - loc_filename_out: &RustString, - message_out: &RustString, - ); - - pub fn LLVMRustUnpackInlineAsmDiagnostic<'a>( - DI: &'a DiagnosticInfo, - level_out: &mut DiagnosticLevel, - cookie_out: &mut c_uint, - message_out: &mut Option<&'a Twine>, - ); - - #[allow(improper_ctypes)] - pub fn LLVMRustWriteDiagnosticInfoToString(DI: &DiagnosticInfo, s: &RustString); - pub fn LLVMRustGetDiagInfoKind(DI: &DiagnosticInfo) -> DiagnosticKind; - - pub fn LLVMRustGetSMDiagnostic<'a>( - DI: &'a DiagnosticInfo, - cookie_out: &mut c_uint, - ) -> &'a SMDiagnostic; - - pub fn LLVMRustSetInlineAsmDiagnosticHandler( - C: &Context, - H: InlineAsmDiagHandlerTy, - CX: *mut c_void, - ); - - #[allow(improper_ctypes)] - pub fn LLVMRustUnpackSMDiagnostic( - d: &SMDiagnostic, - message_out: &RustString, - buffer_out: &RustString, - level_out: &mut DiagnosticLevel, - loc_out: &mut c_uint, - ranges_out: *mut c_uint, - num_ranges: &mut usize, - ) -> bool; - - pub fn LLVMRustWriteArchive( - Dst: *const c_char, - NumMembers: size_t, - Members: *const &RustArchiveMember<'_>, - WriteSymbtab: bool, - Kind: ArchiveKind, - ) -> LLVMRustResult; - pub fn LLVMRustArchiveMemberNew<'a>( - Filename: *const c_char, - Name: *const c_char, - Child: Option<&ArchiveChild<'a>>, - ) -> &'a mut RustArchiveMember<'a>; - pub fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); - - pub fn LLVMRustWriteImportLibrary( - ImportName: *const c_char, - Path: *const c_char, - Exports: *const LLVMRustCOFFShortExport, - NumExports: usize, - Machine: u16, - MinGW: bool, - ) -> LLVMRustResult; - - pub fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); - - pub fn LLVMRustBuildOperandBundleDef<'a>( - Name: *const c_char, - Inputs: *const &'a Value, - NumInputs: c_uint, - ) -> &'a mut OperandBundleDef<'a>; - pub fn LLVMRustFreeOperandBundleDef<'a>(Bundle: &'a mut OperandBundleDef<'a>); - - pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); - - pub fn LLVMRustSetComdat<'a>(M: &'a Module, V: &'a Value, Name: *const c_char, NameLen: size_t); - pub fn LLVMRustSetModulePICLevel(M: &Module); - pub fn LLVMRustSetModulePIELevel(M: &Module); - pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); - pub fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer; - pub fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8; - pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; - pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); - pub fn LLVMRustModuleCost(M: &Module) -> u64; - - pub fn LLVMRustThinLTOBufferCreate(M: &Module, is_thin: bool) -> &'static mut ThinLTOBuffer; - pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); - pub fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; - pub fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; - pub fn LLVMRustCreateThinLTOData( - Modules: *const ThinLTOModule, - NumModules: c_uint, - PreservedSymbols: *const *const c_char, - PreservedSymbolsLen: c_uint, - ) -> Option<&'static mut ThinLTOData>; - pub fn LLVMRustPrepareThinLTORename( - Data: &ThinLTOData, - Module: &Module, - Target: &TargetMachine, - ) -> bool; - pub fn LLVMRustPrepareThinLTOResolveWeak(Data: &ThinLTOData, Module: &Module) -> bool; - pub fn LLVMRustPrepareThinLTOInternalize(Data: &ThinLTOData, Module: &Module) -> bool; - pub fn LLVMRustPrepareThinLTOImport( - Data: &ThinLTOData, - Module: &Module, - Target: &TargetMachine, - ) -> bool; - pub fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); - pub fn LLVMRustParseBitcodeForLTO( - Context: &Context, - Data: *const u8, - len: usize, - Identifier: *const c_char, - ) -> Option<&Module>; - pub fn LLVMRustGetBitcodeSliceFromObjectData( - Data: *const u8, - len: usize, - out_len: &mut usize, - ) -> *const u8; - pub fn LLVMRustThinLTOGetDICompileUnit( - M: &Module, - CU1: &mut *mut c_void, - CU2: &mut *mut c_void, - ); - pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); - - pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; - pub fn LLVMRustLinkerAdd( - linker: &Linker<'_>, - bytecode: *const c_char, - bytecode_len: usize, - ) -> bool; - pub fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>); - #[allow(improper_ctypes)] - pub fn LLVMRustComputeLTOCacheKey( - key_out: &RustString, - mod_id: *const c_char, - data: &ThinLTOData, - ); - - pub fn LLVMRustContextGetDiagnosticHandler(Context: &Context) -> Option<&DiagnosticHandler>; - pub fn LLVMRustContextSetDiagnosticHandler( - context: &Context, - diagnostic_handler: Option<&DiagnosticHandler>, - ); - pub fn LLVMRustContextConfigureDiagnosticHandler( - context: &Context, - diagnostic_handler_callback: DiagnosticHandlerTy, - diagnostic_handler_context: *mut c_void, - remark_all_passes: bool, - remark_passes: *const *const c_char, - remark_passes_len: usize, - ); - - #[allow(improper_ctypes)] - pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); - - pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; -} diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs deleted file mode 100644 index 6602a4ab863..00000000000 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ /dev/null @@ -1,318 +0,0 @@ -#![allow(non_snake_case)] - -pub use self::AtomicRmwBinOp::*; -pub use self::CallConv::*; -pub use self::CodeGenOptSize::*; -pub use self::IntPredicate::*; -pub use self::Linkage::*; -pub use self::MetadataType::*; -pub use self::RealPredicate::*; - -use libc::c_uint; -use rustc_data_structures::small_c_str::SmallCStr; -use rustc_llvm::RustString; -use std::cell::RefCell; -use std::ffi::{CStr, CString}; -use std::str::FromStr; -use std::string::FromUtf8Error; - -pub mod archive_ro; -pub mod diagnostic; -mod ffi; - -pub use self::ffi::*; - -impl LLVMRustResult { - pub fn into_result(self) -> Result<(), ()> { - match self { - LLVMRustResult::Success => Ok(()), - LLVMRustResult::Failure => Err(()), - } - } -} - -pub fn AddFunctionAttributes<'ll>(llfn: &'ll Value, idx: AttributePlace, attrs: &[&'ll Attribute]) { - unsafe { - LLVMRustAddFunctionAttributes(llfn, idx.as_uint(), attrs.as_ptr(), attrs.len()); - } -} - -pub fn AddCallSiteAttributes<'ll>( - callsite: &'ll Value, - idx: AttributePlace, - attrs: &[&'ll Attribute], -) { - unsafe { - LLVMRustAddCallSiteAttributes(callsite, idx.as_uint(), attrs.as_ptr(), attrs.len()); - } -} - -pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &str, value: &str) -> &'ll Attribute { - unsafe { - LLVMCreateStringAttribute( - llcx, - attr.as_ptr().cast(), - attr.len().try_into().unwrap(), - value.as_ptr().cast(), - value.len().try_into().unwrap(), - ) - } -} - -pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { - unsafe { - LLVMCreateStringAttribute( - llcx, - attr.as_ptr().cast(), - attr.len().try_into().unwrap(), - std::ptr::null(), - 0, - ) - } -} - -pub fn CreateAlignmentAttr(llcx: &Context, bytes: u64) -> &Attribute { - unsafe { LLVMRustCreateAlignmentAttr(llcx, bytes) } -} - -pub fn CreateDereferenceableAttr(llcx: &Context, bytes: u64) -> &Attribute { - unsafe { LLVMRustCreateDereferenceableAttr(llcx, bytes) } -} - -pub fn CreateDereferenceableOrNullAttr(llcx: &Context, bytes: u64) -> &Attribute { - unsafe { LLVMRustCreateDereferenceableOrNullAttr(llcx, bytes) } -} - -pub fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { - unsafe { LLVMRustCreateByValAttr(llcx, ty) } -} - -pub fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { - unsafe { LLVMRustCreateStructRetAttr(llcx, ty) } -} - -pub fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { - unsafe { LLVMRustCreateUWTableAttr(llcx, async_) } -} - -pub fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute { - unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) } -} - -pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute { - unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } -} - -#[derive(Copy, Clone)] -pub enum AttributePlace { - ReturnValue, - Argument(u32), - Function, -} - -impl AttributePlace { - pub fn as_uint(self) -> c_uint { - match self { - AttributePlace::ReturnValue => 0, - AttributePlace::Argument(i) => 1 + i, - AttributePlace::Function => !0, - } - } -} - -#[derive(Copy, Clone, PartialEq)] -#[repr(C)] -pub enum CodeGenOptSize { - CodeGenOptSizeNone = 0, - CodeGenOptSizeDefault = 1, - CodeGenOptSizeAggressive = 2, -} - -impl FromStr for ArchiveKind { - type Err = (); - - fn from_str(s: &str) -> Result { - match s { - "gnu" => Ok(ArchiveKind::K_GNU), - "bsd" => Ok(ArchiveKind::K_BSD), - "darwin" => Ok(ArchiveKind::K_DARWIN), - "coff" => Ok(ArchiveKind::K_COFF), - _ => Err(()), - } - } -} - -pub fn SetInstructionCallConv(instr: &Value, cc: CallConv) { - unsafe { - LLVMSetInstructionCallConv(instr, cc as c_uint); - } -} -pub fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { - unsafe { - LLVMSetFunctionCallConv(fn_, cc as c_uint); - } -} - -// Externally visible symbols that might appear in multiple codegen units need to appear in -// their own comdat section so that the duplicates can be discarded at link time. This can for -// example happen for generics when using multiple codegen units. This function simply uses the -// value's name as the comdat value to make sure that it is in a 1-to-1 relationship to the -// function. -// For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52 -pub fn SetUniqueComdat(llmod: &Module, val: &Value) { - unsafe { - let name = get_value_name(val); - LLVMRustSetComdat(llmod, val, name.as_ptr().cast(), name.len()); - } -} - -pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { - unsafe { - LLVMSetUnnamedAddress(global, unnamed); - } -} - -pub fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { - unsafe { - LLVMSetThreadLocalMode(global, mode); - } -} - -impl AttributeKind { - /// Create an LLVM Attribute with no associated value. - pub fn create_attr(self, llcx: &Context) -> &Attribute { - unsafe { LLVMRustCreateAttrNoValue(llcx, self) } - } -} - -pub fn set_section(llglobal: &Value, section_name: &str) { - let section_name_cstr = CString::new(section_name).expect("unexpected CString error"); - unsafe { - LLVMSetSection(llglobal, section_name_cstr.as_ptr()); - } -} - -pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name: &str) -> &'a Value { - let name_cstr = CString::new(name).expect("unexpected CString error"); - unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) } -} - -pub fn set_initializer(llglobal: &Value, constant_val: &Value) { - unsafe { - LLVMSetInitializer(llglobal, constant_val); - } -} - -pub fn set_global_constant(llglobal: &Value, is_constant: bool) { - unsafe { - LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); - } -} - -pub fn set_linkage(llglobal: &Value, linkage: Linkage) { - unsafe { - LLVMRustSetLinkage(llglobal, linkage); - } -} - -pub fn set_visibility(llglobal: &Value, visibility: Visibility) { - unsafe { - LLVMRustSetVisibility(llglobal, visibility); - } -} - -pub fn set_alignment(llglobal: &Value, bytes: usize) { - unsafe { - ffi::LLVMSetAlignment(llglobal, bytes as c_uint); - } -} - -pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &str) { - unsafe { - LLVMRustSetComdat(llmod, llglobal, name.as_ptr().cast(), name.len()); - } -} - -/// Safe wrapper around `LLVMGetParam`, because segfaults are no fun. -pub fn get_param(llfn: &Value, index: c_uint) -> &Value { - unsafe { - assert!( - index < LLVMCountParams(llfn), - "out of bounds argument access: {} out of {} arguments", - index, - LLVMCountParams(llfn) - ); - LLVMGetParam(llfn, index) - } -} - -/// Safe wrapper for `LLVMGetValueName2` into a byte slice -pub fn get_value_name(value: &Value) -> &[u8] { - unsafe { - let mut len = 0; - let data = LLVMGetValueName2(value, &mut len); - std::slice::from_raw_parts(data.cast(), len) - } -} - -/// Safe wrapper for `LLVMSetValueName2` from a byte slice -pub fn set_value_name(value: &Value, name: &[u8]) { - unsafe { - let data = name.as_ptr().cast(); - LLVMSetValueName2(value, data, name.len()); - } -} - -pub fn build_string(f: impl FnOnce(&RustString)) -> Result { - let sr = RustString { bytes: RefCell::new(Vec::new()) }; - f(&sr); - String::from_utf8(sr.bytes.into_inner()) -} - -pub fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec { - let sr = RustString { bytes: RefCell::new(Vec::new()) }; - f(&sr); - sr.bytes.into_inner() -} - -pub fn twine_to_string(tr: &Twine) -> String { - unsafe { - build_string(|s| LLVMRustWriteTwineToString(tr, s)).expect("got a non-UTF8 Twine from LLVM") - } -} - -pub fn last_error() -> Option { - unsafe { - let cstr = LLVMRustGetLastError(); - if cstr.is_null() { - None - } else { - let err = CStr::from_ptr(cstr).to_bytes(); - let err = String::from_utf8_lossy(err).to_string(); - libc::free(cstr as *mut _); - Some(err) - } - } -} - -pub struct OperandBundleDef<'a> { - pub raw: &'a mut ffi::OperandBundleDef<'a>, -} - -impl<'a> OperandBundleDef<'a> { - pub fn new(name: &str, vals: &[&'a Value]) -> Self { - let name = SmallCStr::new(name); - let def = unsafe { - LLVMRustBuildOperandBundleDef(name.as_ptr(), vals.as_ptr(), vals.len() as c_uint) - }; - OperandBundleDef { raw: def } - } -} - -impl Drop for OperandBundleDef<'_> { - fn drop(&mut self) { - unsafe { - LLVMRustFreeOperandBundleDef(&mut *(self.raw as *mut _)); - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs deleted file mode 100644 index a0a640473eb..00000000000 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ /dev/null @@ -1,562 +0,0 @@ -use crate::back::write::create_informational_target_machine; -use crate::{llvm, llvm_util}; -use libc::c_int; -use libloading::Library; -use rustc_codegen_ssa::target_features::{ - supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES, -}; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; -use rustc_data_structures::small_c_str::SmallCStr; -use rustc_fs_util::path_to_c_string; -use rustc_middle::bug; -use rustc_session::config::PrintRequest; -use rustc_session::Session; -use rustc_span::symbol::Symbol; -use rustc_target::spec::{MergeFunctions, PanicStrategy}; -use smallvec::{smallvec, SmallVec}; -use std::ffi::{CStr, CString}; -use tracing::debug; - -use std::mem; -use std::path::Path; -use std::ptr; -use std::slice; -use std::str; -use std::sync::Once; - -static INIT: Once = Once::new(); - -pub(crate) fn init(sess: &Session) { - unsafe { - // Before we touch LLVM, make sure that multithreading is enabled. - if llvm::LLVMIsMultithreaded() != 1 { - bug!("LLVM compiled without support for threads"); - } - INIT.call_once(|| { - configure_llvm(sess); - }); - } -} - -fn require_inited() { - if !INIT.is_completed() { - bug!("LLVM is not initialized"); - } -} - -unsafe fn configure_llvm(sess: &Session) { - let n_args = sess.opts.cg.llvm_args.len() + sess.target.llvm_args.len(); - let mut llvm_c_strs = Vec::with_capacity(n_args + 1); - let mut llvm_args = Vec::with_capacity(n_args + 1); - - llvm::LLVMRustInstallFatalErrorHandler(); - // On Windows, an LLVM assertion will open an Abort/Retry/Ignore dialog - // box for the purpose of launching a debugger. However, on CI this will - // cause it to hang until it times out, which can take several hours. - if std::env::var_os("CI").is_some() { - llvm::LLVMRustDisableSystemDialogsOnCrash(); - } - - fn llvm_arg_to_arg_name(full_arg: &str) -> &str { - full_arg.trim().split(|c: char| c == '=' || c.is_whitespace()).next().unwrap_or("") - } - - let cg_opts = sess.opts.cg.llvm_args.iter().map(AsRef::as_ref); - let tg_opts = sess.target.llvm_args.iter().map(AsRef::as_ref); - let sess_args = cg_opts.chain(tg_opts); - - let user_specified_args: FxHashSet<_> = - sess_args.clone().map(|s| llvm_arg_to_arg_name(s)).filter(|s| !s.is_empty()).collect(); - - { - // This adds the given argument to LLVM. Unless `force` is true - // user specified arguments are *not* overridden. - let mut add = |arg: &str, force: bool| { - if force || !user_specified_args.contains(llvm_arg_to_arg_name(arg)) { - let s = CString::new(arg).unwrap(); - llvm_args.push(s.as_ptr()); - llvm_c_strs.push(s); - } - }; - // Set the llvm "program name" to make usage and invalid argument messages more clear. - add("rustc -Cllvm-args=\"...\" with", true); - if sess.time_llvm_passes() { - add("-time-passes", false); - } - if sess.print_llvm_passes() { - add("-debug-pass=Structure", false); - } - if sess.target.generate_arange_section - && !sess.opts.unstable_opts.no_generate_arange_section - { - add("-generate-arange-section", false); - } - - // Disable the machine outliner by default in LLVM versions 11 and LLVM - // version 12, where it leads to miscompilation. - // - // Ref: - // - https://github.com/rust-lang/rust/issues/85351 - // - https://reviews.llvm.org/D103167 - if llvm_util::get_version() < (13, 0, 0) { - add("-enable-machine-outliner=never", false); - } - - match sess.opts.unstable_opts.merge_functions.unwrap_or(sess.target.merge_functions) { - MergeFunctions::Disabled | MergeFunctions::Trampolines => {} - MergeFunctions::Aliases => { - add("-mergefunc-use-aliases", false); - } - } - - if sess.target.os == "emscripten" && sess.panic_strategy() == PanicStrategy::Unwind { - add("-enable-emscripten-cxx-exceptions", false); - } - - // HACK(eddyb) LLVM inserts `llvm.assume` calls to preserve align attributes - // during inlining. Unfortunately these may block other optimizations. - add("-preserve-alignment-assumptions-during-inlining=false", false); - - // Use non-zero `import-instr-limit` multiplier for cold callsites. - add("-import-cold-multiplier=0.1", false); - - for arg in sess_args { - add(&(*arg), true); - } - } - - if sess.opts.unstable_opts.llvm_time_trace { - llvm::LLVMTimeTraceProfilerInitialize(); - } - - llvm::LLVMInitializePasses(); - - // Use the legacy plugin registration if we don't use the new pass manager - if !should_use_new_llvm_pass_manager( - &sess.opts.unstable_opts.new_llvm_pass_manager, - &sess.target.arch, - ) { - // Register LLVM plugins by loading them into the compiler process. - for plugin in &sess.opts.unstable_opts.llvm_plugins { - let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); - debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); - - // Intentionally leak the dynamic library. We can't ever unload it - // since the library can make things that will live arbitrarily long. - mem::forget(lib); - } - } - - rustc_llvm::initialize_available_targets(); - - llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()); -} - -pub fn time_trace_profiler_finish(file_name: &Path) { - unsafe { - let file_name = path_to_c_string(file_name); - llvm::LLVMTimeTraceProfilerFinish(file_name.as_ptr()); - } -} - -// WARNING: the features after applying `to_llvm_features` must be known -// to LLVM or the feature detection code will walk past the end of the feature -// array, leading to crashes. -// -// To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def -// where the * matches the architecture's name -// Beware to not use the llvm github project for this, but check the git submodule -// found in src/llvm-project -// Though note that Rust can also be build with an external precompiled version of LLVM -// which might lead to failures if the oldest tested / supported LLVM version -// doesn't yet support the relevant intrinsics -pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> { - let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; - match (arch, s) { - ("x86", "sse4.2") => { - if get_version() >= (14, 0, 0) { - smallvec!["sse4.2", "crc32"] - } else { - smallvec!["sse4.2"] - } - } - ("x86", "pclmulqdq") => smallvec!["pclmul"], - ("x86", "rdrand") => smallvec!["rdrnd"], - ("x86", "bmi1") => smallvec!["bmi"], - ("x86", "cmpxchg16b") => smallvec!["cx16"], - ("x86", "avx512vaes") => smallvec!["vaes"], - ("x86", "avx512gfni") => smallvec!["gfni"], - ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"], - ("aarch64", "rcpc2") => smallvec!["rcpc-immo"], - ("aarch64", "dpb") => smallvec!["ccpp"], - ("aarch64", "dpb2") => smallvec!["ccdp"], - ("aarch64", "frintts") => smallvec!["fptoint"], - ("aarch64", "fcma") => smallvec!["complxnum"], - ("aarch64", "pmuv3") => smallvec!["perfmon"], - ("aarch64", "paca") => smallvec!["pauth"], - ("aarch64", "pacg") => smallvec!["pauth"], - // Rust ties fp and neon together. In LLVM neon implicitly enables fp, - // but we manually enable neon when a feature only implicitly enables fp - ("aarch64", "f32mm") => smallvec!["f32mm", "neon"], - ("aarch64", "f64mm") => smallvec!["f64mm", "neon"], - ("aarch64", "fhm") => smallvec!["fp16fml", "neon"], - ("aarch64", "fp16") => smallvec!["fullfp16", "neon"], - ("aarch64", "jsconv") => smallvec!["jsconv", "neon"], - ("aarch64", "sve") => smallvec!["sve", "neon"], - ("aarch64", "sve2") => smallvec!["sve2", "neon"], - ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"], - ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"], - ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"], - ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"], - (_, s) => smallvec![s], - } -} - -// Given a map from target_features to whether they are enabled or disabled, -// ensure only valid combinations are allowed. -pub fn check_tied_features( - sess: &Session, - features: &FxHashMap<&str, bool>, -) -> Option<&'static [&'static str]> { - if !features.is_empty() { - for tied in tied_target_features(sess) { - // Tied features must be set to the same value, or not set at all - let mut tied_iter = tied.iter(); - let enabled = features.get(tied_iter.next().unwrap()); - if tied_iter.any(|f| enabled != features.get(f)) { - return Some(tied); - } - } - } - return None; -} - -// Used to generate cfg variables and apply features -// Must express features in the way Rust understands them -pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec { - let target_machine = create_informational_target_machine(sess); - let mut features: Vec = supported_target_features(sess) - .iter() - .filter_map(|&(feature, gate)| { - if sess.is_nightly_build() || allow_unstable || gate.is_none() { - Some(feature) - } else { - None - } - }) - .filter(|feature| { - // check that all features in a given smallvec are enabled - for llvm_feature in to_llvm_features(sess, feature) { - let cstr = SmallCStr::new(llvm_feature); - if !unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { - return false; - } - } - true - }) - .map(|feature| Symbol::intern(feature)) - .collect(); - - // LLVM 14 changed the ABI for i128 arguments to __float/__fix builtins on Win64 - // (see https://reviews.llvm.org/D110413). This unstable target feature is intended for use - // by compiler-builtins, to export the builtins with the expected, LLVM-version-dependent ABI. - // The target feature can be dropped once we no longer support older LLVM versions. - if sess.is_nightly_build() && get_version() >= (14, 0, 0) { - features.push(Symbol::intern("llvm14-builtins-abi")); - } - features -} - -pub fn print_version() { - let (major, minor, patch) = get_version(); - println!("LLVM version: {}.{}.{}", major, minor, patch); -} - -pub fn get_version() -> (u32, u32, u32) { - // Can be called without initializing LLVM - unsafe { - (llvm::LLVMRustVersionMajor(), llvm::LLVMRustVersionMinor(), llvm::LLVMRustVersionPatch()) - } -} - -pub fn print_passes() { - // Can be called without initializing LLVM - unsafe { - llvm::LLVMRustPrintPasses(); - } -} - -fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> { - let len = unsafe { llvm::LLVMRustGetTargetFeaturesCount(tm) }; - let mut ret = Vec::with_capacity(len); - for i in 0..len { - unsafe { - let mut feature = ptr::null(); - let mut desc = ptr::null(); - llvm::LLVMRustGetTargetFeature(tm, i, &mut feature, &mut desc); - if feature.is_null() || desc.is_null() { - bug!("LLVM returned a `null` target feature string"); - } - let feature = CStr::from_ptr(feature).to_str().unwrap_or_else(|e| { - bug!("LLVM returned a non-utf8 feature string: {}", e); - }); - let desc = CStr::from_ptr(desc).to_str().unwrap_or_else(|e| { - bug!("LLVM returned a non-utf8 feature string: {}", e); - }); - ret.push((feature, desc)); - } - } - ret -} - -fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { - let mut target_features = llvm_target_features(tm); - let mut rustc_target_features = supported_target_features(sess) - .iter() - .filter_map(|(feature, _gate)| { - for llvm_feature in to_llvm_features(sess, *feature) { - // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. - match target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok().map( - |index| { - let (_f, desc) = target_features.remove(index); - (*feature, desc) - }, - ) { - Some(v) => return Some(v), - None => {} - } - } - None - }) - .collect::>(); - rustc_target_features.extend_from_slice(&[( - "crt-static", - "Enables C Run-time Libraries to be statically linked", - )]); - let max_feature_len = target_features - .iter() - .chain(rustc_target_features.iter()) - .map(|(feature, _desc)| feature.len()) - .max() - .unwrap_or(0); - - println!("Features supported by rustc for this target:"); - for (feature, desc) in &rustc_target_features { - println!(" {1:0$} - {2}.", max_feature_len, feature, desc); - } - println!("\nCode-generation features supported by LLVM for this target:"); - for (feature, desc) in &target_features { - println!(" {1:0$} - {2}.", max_feature_len, feature, desc); - } - if target_features.is_empty() { - println!(" Target features listing is not supported by this LLVM version."); - } - println!("\nUse +feature to enable a feature, or -feature to disable it."); - println!("For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n"); - println!("Code-generation features cannot be used in cfg or #[target_feature],"); - println!("and may be renamed or removed in a future version of LLVM or rustc.\n"); -} - -pub(crate) fn print(req: PrintRequest, sess: &Session) { - require_inited(); - let tm = create_informational_target_machine(sess); - match req { - PrintRequest::TargetCPUs => unsafe { llvm::LLVMRustPrintTargetCPUs(tm) }, - PrintRequest::TargetFeatures => print_target_features(sess, tm), - _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), - } -} - -fn handle_native(name: &str) -> &str { - if name != "native" { - return name; - } - - unsafe { - let mut len = 0; - let ptr = llvm::LLVMRustGetHostCPUName(&mut len); - str::from_utf8(slice::from_raw_parts(ptr as *const u8, len)).unwrap() - } -} - -pub fn target_cpu(sess: &Session) -> &str { - match sess.opts.cg.target_cpu { - Some(ref name) => handle_native(name), - None => handle_native(sess.target.cpu.as_ref()), - } -} - -/// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`, -/// `--target` and similar). -pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec { - // Features that come earlier are overridden by conflicting features later in the string. - // Typically we'll want more explicit settings to override the implicit ones, so: - // - // * Features from -Ctarget-cpu=*; are overridden by [^1] - // * Features implied by --target; are overridden by - // * Features from -Ctarget-feature; are overridden by - // * function specific features. - // - // [^1]: target-cpu=native is handled here, other target-cpu values are handled implicitly - // through LLVM TargetMachine implementation. - // - // FIXME(nagisa): it isn't clear what's the best interaction between features implied by - // `-Ctarget-cpu` and `--target` are. On one hand, you'd expect CLI arguments to always - // override anything that's implicit, so e.g. when there's no `--target` flag, features implied - // the host target are overridden by `-Ctarget-cpu=*`. On the other hand, what about when both - // `--target` and `-Ctarget-cpu=*` are specified? Both then imply some target features and both - // flags are specified by the user on the CLI. It isn't as clear-cut which order of precedence - // should be taken in cases like these. - let mut features = vec![]; - - // -Ctarget-cpu=native - match sess.opts.cg.target_cpu { - Some(ref s) if s == "native" => { - let features_string = unsafe { - let ptr = llvm::LLVMGetHostCPUFeatures(); - let features_string = if !ptr.is_null() { - CStr::from_ptr(ptr) - .to_str() - .unwrap_or_else(|e| { - bug!("LLVM returned a non-utf8 features string: {}", e); - }) - .to_owned() - } else { - bug!("could not allocate host CPU features, LLVM returned a `null` string"); - }; - - llvm::LLVMDisposeMessage(ptr); - - features_string - }; - features.extend(features_string.split(',').map(String::from)); - } - Some(_) | None => {} - }; - - // Features implied by an implicit or explicit `--target`. - features.extend( - sess.target - .features - .split(',') - .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) - .map(String::from), - ); - - // -Ctarget-features - let supported_features = supported_target_features(sess); - let mut featsmap = FxHashMap::default(); - let feats = sess - .opts - .cg - .target_feature - .split(',') - .filter_map(|s| { - let enable_disable = match s.chars().next() { - None => return None, - Some(c @ '+' | c @ '-') => c, - Some(_) => { - if diagnostics { - let mut diag = sess.struct_warn(&format!( - "unknown feature specified for `-Ctarget-feature`: `{}`", - s - )); - diag.note("features must begin with a `+` to enable or `-` to disable it"); - diag.emit(); - } - return None; - } - }; - - let feature = backend_feature_name(s)?; - // Warn against use of LLVM specific feature names on the CLI. - if diagnostics && !supported_features.iter().any(|&(v, _)| v == feature) { - let rust_feature = supported_features.iter().find_map(|&(rust_feature, _)| { - let llvm_features = to_llvm_features(sess, rust_feature); - if llvm_features.contains(&feature) && !llvm_features.contains(&rust_feature) { - Some(rust_feature) - } else { - None - } - }); - let mut diag = sess.struct_warn(&format!( - "unknown feature specified for `-Ctarget-feature`: `{}`", - feature - )); - diag.note("it is still passed through to the codegen backend"); - if let Some(rust_feature) = rust_feature { - diag.help(&format!("you might have meant: `{}`", rust_feature)); - } else { - diag.note("consider filing a feature request"); - } - diag.emit(); - } - - if diagnostics { - // FIXME(nagisa): figure out how to not allocate a full hashset here. - featsmap.insert(feature, enable_disable == '+'); - } - - // rustc-specific features do not get passed down to LLVM… - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return None; - } - // ... otherwise though we run through `to_llvm_features` when - // passing requests down to LLVM. This means that all in-language - // features also work on the command line instead of having two - // different names when the LLVM name and the Rust name differ. - Some( - to_llvm_features(sess, feature) - .into_iter() - .map(move |f| format!("{}{}", enable_disable, f)), - ) - }) - .flatten(); - features.extend(feats); - - if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { - sess.err(&format!( - "target features {} must all be enabled or disabled together", - f.join(", ") - )); - } - - features -} - -/// Returns a feature name for the given `+feature` or `-feature` string. -/// -/// Only allows features that are backend specific (i.e. not [`RUSTC_SPECIFIC_FEATURES`].) -fn backend_feature_name(s: &str) -> Option<&str> { - // features must start with a `+` or `-`. - let feature = s.strip_prefix(&['+', '-'][..]).unwrap_or_else(|| { - bug!("target feature `{}` must begin with a `+` or `-`", s); - }); - // Rustc-specific feature requests like `+crt-static` or `-crt-static` - // are not passed down to LLVM. - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return None; - } - Some(feature) -} - -pub fn tune_cpu(sess: &Session) -> Option<&str> { - let name = sess.opts.unstable_opts.tune_cpu.as_ref()?; - Some(handle_native(name)) -} - -pub(crate) fn should_use_new_llvm_pass_manager(user_opt: &Option, target_arch: &str) -> bool { - // The new pass manager is enabled by default for LLVM >= 13. - // This matches Clang, which also enables it since Clang 13. - - // Since LLVM 15, the legacy pass manager is no longer supported. - if llvm_util::get_version() >= (15, 0, 0) { - return true; - } - - // There are some perf issues with the new pass manager when targeting - // s390x with LLVM 13, so enable the new pass manager only with LLVM 14. - // See https://github.com/rust-lang/rust/issues/89609. - let min_version = if target_arch == "s390x" { 14 } else { 13 }; - user_opt.unwrap_or_else(|| llvm_util::get_version() >= (min_version, 0, 0)) -} diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs deleted file mode 100644 index 6e94284852f..00000000000 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ /dev/null @@ -1,150 +0,0 @@ -use crate::attributes; -use crate::base; -use crate::context::CodegenCx; -use crate::llvm; -use crate::type_of::LayoutLlvmExt; -use rustc_codegen_ssa::traits::*; -use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -pub use rustc_middle::mir::mono::MonoItem; -use rustc_middle::mir::mono::{Linkage, Visibility}; -use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; -use rustc_middle::ty::{self, Instance, TypeVisitable}; -use rustc_session::config::CrateType; -use rustc_target::spec::RelocModel; -use tracing::debug; - -impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { - fn predefine_static( - &self, - def_id: DefId, - linkage: Linkage, - visibility: Visibility, - symbol_name: &str, - ) { - let instance = Instance::mono(self.tcx, def_id); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); - let llty = self.layout_of(ty).llvm_type(self); - - let g = self.define_global(symbol_name, llty).unwrap_or_else(|| { - self.sess().span_fatal( - self.tcx.def_span(def_id), - &format!("symbol `{}` is already defined", symbol_name), - ) - }); - - unsafe { - llvm::LLVMRustSetLinkage(g, base::linkage_to_llvm(linkage)); - llvm::LLVMRustSetVisibility(g, base::visibility_to_llvm(visibility)); - if self.should_assume_dso_local(g, false) { - llvm::LLVMRustSetDSOLocal(g, true); - } - } - - self.instances.borrow_mut().insert(instance, g); - } - - fn predefine_fn( - &self, - instance: Instance<'tcx>, - linkage: Linkage, - visibility: Visibility, - symbol_name: &str, - ) { - assert!(!instance.substs.needs_infer()); - - let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - let lldecl = self.declare_fn(symbol_name, fn_abi); - unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; - let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); - base::set_link_section(lldecl, attrs); - if linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR { - llvm::SetUniqueComdat(self.llmod, lldecl); - } - - // If we're compiling the compiler-builtins crate, e.g., the equivalent of - // compiler-rt, then we want to implicitly compile everything with hidden - // visibility as we're going to link this object all over the place but - // don't want the symbols to get exported. - if linkage != Linkage::Internal - && linkage != Linkage::Private - && self.tcx.is_compiler_builtins(LOCAL_CRATE) - { - unsafe { - llvm::LLVMRustSetVisibility(lldecl, llvm::Visibility::Hidden); - } - } else { - unsafe { - llvm::LLVMRustSetVisibility(lldecl, base::visibility_to_llvm(visibility)); - } - } - - debug!("predefine_fn: instance = {:?}", instance); - - attributes::from_fn_attrs(self, lldecl, instance); - - unsafe { - if self.should_assume_dso_local(lldecl, false) { - llvm::LLVMRustSetDSOLocal(lldecl, true); - } - } - - self.instances.borrow_mut().insert(instance, lldecl); - } -} - -impl CodegenCx<'_, '_> { - /// Whether a definition or declaration can be assumed to be local to a group of - /// libraries that form a single DSO or executable. - pub(crate) unsafe fn should_assume_dso_local( - &self, - llval: &llvm::Value, - is_declaration: bool, - ) -> bool { - let linkage = llvm::LLVMRustGetLinkage(llval); - let visibility = llvm::LLVMRustGetVisibility(llval); - - if matches!(linkage, llvm::Linkage::InternalLinkage | llvm::Linkage::PrivateLinkage) { - return true; - } - - if visibility != llvm::Visibility::Default && linkage != llvm::Linkage::ExternalWeakLinkage - { - return true; - } - - // Symbols from executables can't really be imported any further. - let all_exe = self.tcx.sess.crate_types().iter().all(|ty| *ty == CrateType::Executable); - let is_declaration_for_linker = - is_declaration || linkage == llvm::Linkage::AvailableExternallyLinkage; - if all_exe && !is_declaration_for_linker { - return true; - } - - // PowerPC64 prefers TOC indirection to avoid copy relocations. - if matches!(&*self.tcx.sess.target.arch, "powerpc64" | "powerpc64le") { - return false; - } - - // Thread-local variables generally don't support copy relocations. - let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) - .map(|v| llvm::LLVMIsThreadLocal(v) == llvm::True) - .unwrap_or(false); - if is_thread_local_var { - return false; - } - - // Match clang by only supporting COFF and ELF for now. - if self.tcx.sess.target.is_like_osx { - return false; - } - - // Static relocation model should force copy relocations everywhere. - if self.tcx.sess.relocation_model() == RelocModel::Static { - return true; - } - - // With pie relocation model calls of functions defined in the translation - // unit can use copy relocations. - self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration - } -} diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs deleted file mode 100644 index eeb38d4ecf5..00000000000 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ /dev/null @@ -1,319 +0,0 @@ -pub use crate::llvm::Type; - -use crate::abi::{FnAbiLlvmExt, LlvmType}; -use crate::common; -use crate::context::CodegenCx; -use crate::llvm; -use crate::llvm::{Bool, False, True}; -use crate::type_of::LayoutLlvmExt; -use crate::value::Value; -use rustc_codegen_ssa::common::TypeKind; -use rustc_codegen_ssa::traits::*; -use rustc_data_structures::small_c_str::SmallCStr; -use rustc_middle::bug; -use rustc_middle::ty::layout::TyAndLayout; -use rustc_middle::ty::{self, Ty}; -use rustc_target::abi::call::{CastTarget, FnAbi, Reg}; -use rustc_target::abi::{AddressSpace, Align, Integer, Size}; - -use std::fmt; -use std::ptr; - -use libc::{c_char, c_uint}; - -impl PartialEq for Type { - fn eq(&self, other: &Self) -> bool { - ptr::eq(self, other) - } -} - -impl fmt::Debug for Type { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str( - &llvm::build_string(|s| unsafe { - llvm::LLVMRustWriteTypeToString(self, s); - }) - .expect("non-UTF8 type description from LLVM"), - ) - } -} - -impl<'ll> CodegenCx<'ll, '_> { - pub(crate) fn type_named_struct(&self, name: &str) -> &'ll Type { - let name = SmallCStr::new(name); - unsafe { llvm::LLVMStructCreateNamed(self.llcx, name.as_ptr()) } - } - - pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { - unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) } - } - - pub(crate) fn type_void(&self) -> &'ll Type { - unsafe { llvm::LLVMVoidTypeInContext(self.llcx) } - } - - pub(crate) fn type_metadata(&self) -> &'ll Type { - unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } - } - - ///x Creates an integer type with the given number of bits, e.g., i24 - pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type { - unsafe { llvm::LLVMIntTypeInContext(self.llcx, num_bits as c_uint) } - } - - pub(crate) fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { - unsafe { llvm::LLVMVectorType(ty, len as c_uint) } - } - - pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { - unsafe { - let n_args = llvm::LLVMCountParamTypes(ty) as usize; - let mut args = Vec::with_capacity(n_args); - llvm::LLVMGetParamTypes(ty, args.as_mut_ptr()); - args.set_len(n_args); - args - } - } - - pub(crate) fn type_bool(&self) -> &'ll Type { - self.type_i8() - } - - pub(crate) fn type_int_from_ty(&self, t: ty::IntTy) -> &'ll Type { - match t { - ty::IntTy::Isize => self.type_isize(), - ty::IntTy::I8 => self.type_i8(), - ty::IntTy::I16 => self.type_i16(), - ty::IntTy::I32 => self.type_i32(), - ty::IntTy::I64 => self.type_i64(), - ty::IntTy::I128 => self.type_i128(), - } - } - - pub(crate) fn type_uint_from_ty(&self, t: ty::UintTy) -> &'ll Type { - match t { - ty::UintTy::Usize => self.type_isize(), - ty::UintTy::U8 => self.type_i8(), - ty::UintTy::U16 => self.type_i16(), - ty::UintTy::U32 => self.type_i32(), - ty::UintTy::U64 => self.type_i64(), - ty::UintTy::U128 => self.type_i128(), - } - } - - pub(crate) fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { - match t { - ty::FloatTy::F32 => self.type_f32(), - ty::FloatTy::F64 => self.type_f64(), - } - } - - pub(crate) fn type_pointee_for_align(&self, align: Align) -> &'ll Type { - // FIXME(eddyb) We could find a better approximation if ity.align < align. - let ity = Integer::approximate_align(self, align); - self.type_from_integer(ity) - } - - /// Return a LLVM type that has at most the required alignment, - /// and exactly the required size, as a best-effort padding array. - pub(crate) fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { - let unit = Integer::approximate_align(self, align); - let size = size.bytes(); - let unit_size = unit.size().bytes(); - assert_eq!(size % unit_size, 0); - self.type_array(self.type_from_integer(unit), size / unit_size) - } - - pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { - unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } - } - - pub(crate) fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { - unsafe { llvm::LLVMRustArrayType(ty, len) } - } -} - -impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn type_i1(&self) -> &'ll Type { - unsafe { llvm::LLVMInt1TypeInContext(self.llcx) } - } - - fn type_i8(&self) -> &'ll Type { - unsafe { llvm::LLVMInt8TypeInContext(self.llcx) } - } - - fn type_i16(&self) -> &'ll Type { - unsafe { llvm::LLVMInt16TypeInContext(self.llcx) } - } - - fn type_i32(&self) -> &'ll Type { - unsafe { llvm::LLVMInt32TypeInContext(self.llcx) } - } - - fn type_i64(&self) -> &'ll Type { - unsafe { llvm::LLVMInt64TypeInContext(self.llcx) } - } - - fn type_i128(&self) -> &'ll Type { - unsafe { llvm::LLVMIntTypeInContext(self.llcx, 128) } - } - - fn type_isize(&self) -> &'ll Type { - self.isize_ty - } - - fn type_f32(&self) -> &'ll Type { - unsafe { llvm::LLVMFloatTypeInContext(self.llcx) } - } - - fn type_f64(&self) -> &'ll Type { - unsafe { llvm::LLVMDoubleTypeInContext(self.llcx) } - } - - fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { - unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) } - } - - fn type_struct(&self, els: &[&'ll Type], packed: bool) -> &'ll Type { - unsafe { - llvm::LLVMStructTypeInContext( - self.llcx, - els.as_ptr(), - els.len() as c_uint, - packed as Bool, - ) - } - } - - fn type_kind(&self, ty: &'ll Type) -> TypeKind { - unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() } - } - - fn type_ptr_to(&self, ty: &'ll Type) -> &'ll Type { - assert_ne!( - self.type_kind(ty), - TypeKind::Function, - "don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead or explicitly specify an address space if it makes sense" - ); - ty.ptr_to(AddressSpace::DATA) - } - - fn type_ptr_to_ext(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type { - ty.ptr_to(address_space) - } - - fn element_type(&self, ty: &'ll Type) -> &'ll Type { - match self.type_kind(ty) { - TypeKind::Array | TypeKind::Vector => unsafe { llvm::LLVMGetElementType(ty) }, - TypeKind::Pointer => bug!("element_type is not supported for opaque pointers"), - other => bug!("element_type called on unsupported type {:?}", other), - } - } - - fn vector_length(&self, ty: &'ll Type) -> usize { - unsafe { llvm::LLVMGetVectorSize(ty) as usize } - } - - fn float_width(&self, ty: &'ll Type) -> usize { - match self.type_kind(ty) { - TypeKind::Float => 32, - TypeKind::Double => 64, - TypeKind::X86_FP80 => 80, - TypeKind::FP128 | TypeKind::PPC_FP128 => 128, - _ => bug!("llvm_float_width called on a non-float type"), - } - } - - fn int_width(&self, ty: &'ll Type) -> u64 { - unsafe { llvm::LLVMGetIntTypeWidth(ty) as u64 } - } - - fn val_ty(&self, v: &'ll Value) -> &'ll Type { - common::val_ty(v) - } -} - -impl Type { - pub fn i8_llcx(llcx: &llvm::Context) -> &Type { - unsafe { llvm::LLVMInt8TypeInContext(llcx) } - } - - // Creates an integer type with the given number of bits, e.g., i24 - pub fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { - unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) } - } - - pub fn i8p_llcx(llcx: &llvm::Context) -> &Type { - Type::i8_llcx(llcx).ptr_to(AddressSpace::DATA) - } - - fn ptr_to(&self, address_space: AddressSpace) -> &Type { - unsafe { llvm::LLVMPointerType(self, address_space.0) } - } -} - -impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type { - layout.llvm_type(self) - } - fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type { - layout.immediate_llvm_type(self) - } - fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool { - layout.is_llvm_immediate() - } - fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool { - layout.is_llvm_scalar_pair() - } - fn backend_field_index(&self, layout: TyAndLayout<'tcx>, index: usize) -> u64 { - layout.llvm_field_index(self, index) - } - fn scalar_pair_element_backend_type( - &self, - layout: TyAndLayout<'tcx>, - index: usize, - immediate: bool, - ) -> &'ll Type { - layout.scalar_pair_element_llvm_type(self, index, immediate) - } - fn cast_backend_type(&self, ty: &CastTarget) -> &'ll Type { - ty.llvm_type(self) - } - fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { - fn_abi.llvm_type(self) - } - fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { - fn_abi.ptr_to_llvm_type(self) - } - fn reg_backend_type(&self, ty: &Reg) -> &'ll Type { - ty.llvm_type(self) - } -} - -impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn set_type_metadata(&self, function: &'ll Value, typeid: String) { - let typeid_metadata = self.typeid_metadata(typeid); - let v = [self.const_usize(0), typeid_metadata]; - unsafe { - llvm::LLVMGlobalSetMetadata( - function, - llvm::MD_type as c_uint, - llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( - self.llcx, - v.as_ptr(), - v.len() as c_uint, - )), - ) - } - } - - fn typeid_metadata(&self, typeid: String) -> &'ll Value { - unsafe { - llvm::LLVMMDStringInContext( - self.llcx, - typeid.as_ptr() as *const c_char, - typeid.len() as c_uint, - ) - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs deleted file mode 100644 index 9f0e6c80b19..00000000000 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ /dev/null @@ -1,418 +0,0 @@ -use crate::common::*; -use crate::context::TypeLowering; -use crate::llvm_util::get_version; -use crate::type_::Type; -use rustc_codegen_ssa::traits::*; -use rustc_middle::bug; -use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; -use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; -use rustc_middle::ty::{self, Ty, TypeVisitable}; -use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape}; -use rustc_target::abi::{Int, Pointer, F32, F64}; -use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants}; -use smallvec::{smallvec, SmallVec}; -use tracing::debug; - -use std::fmt::Write; - -fn uncached_llvm_type<'a, 'tcx>( - cx: &CodegenCx<'a, 'tcx>, - layout: TyAndLayout<'tcx>, - defer: &mut Option<(&'a Type, TyAndLayout<'tcx>)>, - field_remapping: &mut Option>, -) -> &'a Type { - match layout.abi { - Abi::Scalar(_) => bug!("handled elsewhere"), - Abi::Vector { element, count } => { - let element = layout.scalar_llvm_type_at(cx, element, Size::ZERO); - return cx.type_vector(element, count); - } - Abi::ScalarPair(..) => { - return cx.type_struct( - &[ - layout.scalar_pair_element_llvm_type(cx, 0, false), - layout.scalar_pair_element_llvm_type(cx, 1, false), - ], - false, - ); - } - Abi::Uninhabited | Abi::Aggregate { .. } => {} - } - - let name = match layout.ty.kind() { - // FIXME(eddyb) producing readable type names for trait objects can result - // in problematically distinct types due to HRTB and subtyping (see #47638). - // ty::Dynamic(..) | - ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str - // For performance reasons we use names only when emitting LLVM IR. Unless we are on - // LLVM < 14, where the use of unnamed types resulted in various issues, e.g., #76213, - // #79564, and #79246. - if get_version() < (14, 0, 0) || !cx.sess().fewer_names() => - { - let mut name = with_no_visible_paths!(with_no_trimmed_paths!(layout.ty.to_string())); - if let (&ty::Adt(def, _), &Variants::Single { index }) = - (layout.ty.kind(), &layout.variants) - { - if def.is_enum() && !def.variants().is_empty() { - write!(&mut name, "::{}", def.variant(index).name).unwrap(); - } - } - if let (&ty::Generator(_, _, _), &Variants::Single { index }) = - (layout.ty.kind(), &layout.variants) - { - write!(&mut name, "::{}", ty::GeneratorSubsts::variant_name(index)).unwrap(); - } - Some(name) - } - // Use identified structure types for ADT. Due to pointee types in LLVM IR their definition - // might be recursive. Other cases are non-recursive and we can use literal structure types. - ty::Adt(..) => Some(String::new()), - _ => None, - }; - - match layout.fields { - FieldsShape::Primitive | FieldsShape::Union(_) => { - let fill = cx.type_padding_filler(layout.size, layout.align.abi); - let packed = false; - match name { - None => cx.type_struct(&[fill], packed), - Some(ref name) => { - let llty = cx.type_named_struct(name); - cx.set_struct_body(llty, &[fill], packed); - llty - } - } - } - FieldsShape::Array { count, .. } => cx.type_array(layout.field(cx, 0).llvm_type(cx), count), - FieldsShape::Arbitrary { .. } => match name { - None => { - let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); - *field_remapping = new_field_remapping; - cx.type_struct(&llfields, packed) - } - Some(ref name) => { - let llty = cx.type_named_struct(name); - *defer = Some((llty, layout)); - llty - } - }, - } -} - -fn struct_llfields<'a, 'tcx>( - cx: &CodegenCx<'a, 'tcx>, - layout: TyAndLayout<'tcx>, -) -> (Vec<&'a Type>, bool, Option>) { - debug!("struct_llfields: {:#?}", layout); - let field_count = layout.fields.count(); - - let mut packed = false; - let mut offset = Size::ZERO; - let mut prev_effective_align = layout.align.abi; - let mut result: Vec<_> = Vec::with_capacity(1 + field_count * 2); - let mut field_remapping = smallvec![0; field_count]; - for i in layout.fields.index_by_increasing_offset() { - let target_offset = layout.fields.offset(i as usize); - let field = layout.field(cx, i); - let effective_field_align = - layout.align.abi.min(field.align.abi).restrict_for_offset(target_offset); - packed |= effective_field_align < field.align.abi; - - debug!( - "struct_llfields: {}: {:?} offset: {:?} target_offset: {:?} \ - effective_field_align: {}", - i, - field, - offset, - target_offset, - effective_field_align.bytes() - ); - assert!(target_offset >= offset); - let padding = target_offset - offset; - if padding != Size::ZERO { - let padding_align = prev_effective_align.min(effective_field_align); - assert_eq!(offset.align_to(padding_align) + padding, target_offset); - result.push(cx.type_padding_filler(padding, padding_align)); - debug!(" padding before: {:?}", padding); - } - field_remapping[i] = result.len() as u32; - result.push(field.llvm_type(cx)); - offset = target_offset + field.size; - prev_effective_align = effective_field_align; - } - let padding_used = result.len() > field_count; - if !layout.is_unsized() && field_count > 0 { - if offset > layout.size { - bug!("layout: {:#?} stride: {:?} offset: {:?}", layout, layout.size, offset); - } - let padding = layout.size - offset; - if padding != Size::ZERO { - let padding_align = prev_effective_align; - assert_eq!(offset.align_to(padding_align) + padding, layout.size); - debug!( - "struct_llfields: pad_bytes: {:?} offset: {:?} stride: {:?}", - padding, offset, layout.size - ); - result.push(cx.type_padding_filler(padding, padding_align)); - } - } else { - debug!("struct_llfields: offset: {:?} stride: {:?}", offset, layout.size); - } - let field_remapping = if padding_used { Some(field_remapping) } else { None }; - (result, packed, field_remapping) -} - -impl<'a, 'tcx> CodegenCx<'a, 'tcx> { - pub fn align_of(&self, ty: Ty<'tcx>) -> Align { - self.layout_of(ty).align.abi - } - - pub fn size_of(&self, ty: Ty<'tcx>) -> Size { - self.layout_of(ty).size - } - - pub fn size_and_align_of(&self, ty: Ty<'tcx>) -> (Size, Align) { - let layout = self.layout_of(ty); - (layout.size, layout.align.abi) - } -} - -pub trait LayoutLlvmExt<'tcx> { - fn is_llvm_immediate(&self) -> bool; - fn is_llvm_scalar_pair(&self) -> bool; - fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; - fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; - fn scalar_llvm_type_at<'a>( - &self, - cx: &CodegenCx<'a, 'tcx>, - scalar: Scalar, - offset: Size, - ) -> &'a Type; - fn scalar_pair_element_llvm_type<'a>( - &self, - cx: &CodegenCx<'a, 'tcx>, - index: usize, - immediate: bool, - ) -> &'a Type; - fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64; - fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option; -} - -impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { - fn is_llvm_immediate(&self) -> bool { - match self.abi { - Abi::Scalar(_) | Abi::Vector { .. } => true, - Abi::ScalarPair(..) => false, - Abi::Uninhabited | Abi::Aggregate { .. } => self.is_zst(), - } - } - - fn is_llvm_scalar_pair(&self) -> bool { - match self.abi { - Abi::ScalarPair(..) => true, - Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false, - } - } - - /// Gets the LLVM type corresponding to a Rust type, i.e., `rustc_middle::ty::Ty`. - /// The pointee type of the pointer in `PlaceRef` is always this type. - /// For sized types, it is also the right LLVM type for an `alloca` - /// containing a value of that type, and most immediates (except `bool`). - /// Unsized types, however, are represented by a "minimal unit", e.g. - /// `[T]` becomes `T`, while `str` and `Trait` turn into `i8` - this - /// is useful for indexing slices, as `&[T]`'s data pointer is `T*`. - /// If the type is an unsized struct, the regular layout is generated, - /// with the inner-most trailing unsized field using the "minimal unit" - /// of that field's type - this is useful for taking the address of - /// that field and ensuring the struct has the right alignment. - fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - if let Abi::Scalar(scalar) = self.abi { - // Use a different cache for scalars because pointers to DSTs - // can be either fat or thin (data pointers of fat pointers). - if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { - return llty; - } - let llty = match *self.ty.kind() { - ty::Ref(_, ty, _) | ty::RawPtr(ty::TypeAndMut { ty, .. }) => { - cx.type_ptr_to(cx.layout_of(ty).llvm_type(cx)) - } - ty::Adt(def, _) if def.is_box() => { - cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).llvm_type(cx)) - } - ty::FnPtr(sig) => { - cx.fn_ptr_backend_type(cx.fn_abi_of_fn_ptr(sig, ty::List::empty())) - } - _ => self.scalar_llvm_type_at(cx, scalar, Size::ZERO), - }; - cx.scalar_lltypes.borrow_mut().insert(self.ty, llty); - return llty; - } - - // Check the cache. - let variant_index = match self.variants { - Variants::Single { index } => Some(index), - _ => None, - }; - if let Some(llty) = cx.type_lowering.borrow().get(&(self.ty, variant_index)) { - return llty.lltype; - } - - debug!("llvm_type({:#?})", self); - - assert!(!self.ty.has_escaping_bound_vars(), "{:?} has escaping bound vars", self.ty); - - // Make sure lifetimes are erased, to avoid generating distinct LLVM - // types for Rust types that only differ in the choice of lifetimes. - let normal_ty = cx.tcx.erase_regions(self.ty); - - let mut defer = None; - let mut field_remapping = None; - let llty = if self.ty != normal_ty { - let mut layout = cx.layout_of(normal_ty); - if let Some(v) = variant_index { - layout = layout.for_variant(cx, v); - } - layout.llvm_type(cx) - } else { - uncached_llvm_type(cx, *self, &mut defer, &mut field_remapping) - }; - debug!("--> mapped {:#?} to llty={:?}", self, llty); - - cx.type_lowering - .borrow_mut() - .insert((self.ty, variant_index), TypeLowering { lltype: llty, field_remapping }); - - if let Some((llty, layout)) = defer { - let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); - cx.set_struct_body(llty, &llfields, packed); - cx.type_lowering - .borrow_mut() - .get_mut(&(self.ty, variant_index)) - .unwrap() - .field_remapping = new_field_remapping; - } - llty - } - - fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - if let Abi::Scalar(scalar) = self.abi { - if scalar.is_bool() { - return cx.type_i1(); - } - } - self.llvm_type(cx) - } - - fn scalar_llvm_type_at<'a>( - &self, - cx: &CodegenCx<'a, 'tcx>, - scalar: Scalar, - offset: Size, - ) -> &'a Type { - match scalar.primitive() { - Int(i, _) => cx.type_from_integer(i), - F32 => cx.type_f32(), - F64 => cx.type_f64(), - Pointer => { - // If we know the alignment, pick something better than i8. - let (pointee, address_space) = - if let Some(pointee) = self.pointee_info_at(cx, offset) { - (cx.type_pointee_for_align(pointee.align), pointee.address_space) - } else { - (cx.type_i8(), AddressSpace::DATA) - }; - cx.type_ptr_to_ext(pointee, address_space) - } - } - } - - fn scalar_pair_element_llvm_type<'a>( - &self, - cx: &CodegenCx<'a, 'tcx>, - index: usize, - immediate: bool, - ) -> &'a Type { - // HACK(eddyb) special-case fat pointers until LLVM removes - // pointee types, to avoid bitcasting every `OperandRef::deref`. - match self.ty.kind() { - ty::Ref(..) | ty::RawPtr(_) => { - return self.field(cx, index).llvm_type(cx); - } - // only wide pointer boxes are handled as pointers - // thin pointer boxes with scalar allocators are handled by the general logic below - ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { - let ptr_ty = cx.tcx.mk_mut_ptr(self.ty.boxed_ty()); - return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); - } - _ => {} - } - - let Abi::ScalarPair(a, b) = self.abi else { - bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self); - }; - let scalar = [a, b][index]; - - // Make sure to return the same type `immediate_llvm_type` would when - // dealing with an immediate pair. This means that `(bool, bool)` is - // effectively represented as `{i8, i8}` in memory and two `i1`s as an - // immediate, just like `bool` is typically `i8` in memory and only `i1` - // when immediate. We need to load/store `bool` as `i8` to avoid - // crippling LLVM optimizations or triggering other LLVM bugs with `i1`. - if immediate && scalar.is_bool() { - return cx.type_i1(); - } - - let offset = if index == 0 { Size::ZERO } else { a.size(cx).align_to(b.align(cx).abi) }; - self.scalar_llvm_type_at(cx, scalar, offset) - } - - fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64 { - match self.abi { - Abi::Scalar(_) | Abi::ScalarPair(..) => { - bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self) - } - _ => {} - } - match self.fields { - FieldsShape::Primitive | FieldsShape::Union(_) => { - bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self) - } - - FieldsShape::Array { .. } => index as u64, - - FieldsShape::Arbitrary { .. } => { - let variant_index = match self.variants { - Variants::Single { index } => Some(index), - _ => None, - }; - - // Look up llvm field if indexes do not match memory order due to padding. If - // `field_remapping` is `None` no padding was used and the llvm field index - // matches the memory index. - match cx.type_lowering.borrow().get(&(self.ty, variant_index)) { - Some(TypeLowering { field_remapping: Some(ref remap), .. }) => { - remap[index] as u64 - } - Some(_) => self.fields.memory_index(index) as u64, - None => { - bug!("TyAndLayout::llvm_field_index({:?}): type info not found", self) - } - } - } - } - } - - // FIXME(eddyb) this having the same name as `TyAndLayout::pointee_info_at` - // (the inherent method, which is lacking this caching logic) can result in - // the uncached version being called - not wrong, but potentially inefficient. - fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option { - if let Some(&pointee) = cx.pointee_infos.borrow().get(&(self.ty, offset)) { - return pointee; - } - - let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset); - - cx.pointee_infos.borrow_mut().insert((self.ty, offset), result); - result - } -} diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs deleted file mode 100644 index ceb3d5a84ab..00000000000 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ /dev/null @@ -1,214 +0,0 @@ -use crate::builder::Builder; -use crate::type_::Type; -use crate::type_of::LayoutLlvmExt; -use crate::value::Value; -use rustc_codegen_ssa::mir::operand::OperandRef; -use rustc_codegen_ssa::{ - common::IntPredicate, - traits::{BaseTypeMethods, BuilderMethods, ConstMethods, DerivedTypeMethods}, -}; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; -use rustc_middle::ty::Ty; -use rustc_target::abi::{Align, Endian, HasDataLayout, Size}; - -fn round_pointer_up_to_alignment<'ll>( - bx: &mut Builder<'_, 'll, '_>, - addr: &'ll Value, - align: Align, - ptr_ty: &'ll Type, -) -> &'ll Value { - let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize()); - ptr_as_int = bx.add(ptr_as_int, bx.cx().const_i32(align.bytes() as i32 - 1)); - ptr_as_int = bx.and(ptr_as_int, bx.cx().const_i32(-(align.bytes() as i32))); - bx.inttoptr(ptr_as_int, ptr_ty) -} - -fn emit_direct_ptr_va_arg<'ll, 'tcx>( - bx: &mut Builder<'_, 'll, 'tcx>, - list: OperandRef<'tcx, &'ll Value>, - llty: &'ll Type, - size: Size, - align: Align, - slot_size: Align, - allow_higher_align: bool, -) -> (&'ll Value, Align) { - let va_list_ty = bx.type_i8p(); - let va_list_ptr_ty = bx.type_ptr_to(va_list_ty); - let va_list_addr = if list.layout.llvm_type(bx.cx) != va_list_ptr_ty { - bx.bitcast(list.immediate(), va_list_ptr_ty) - } else { - list.immediate() - }; - - let ptr = bx.load(va_list_ty, va_list_addr, bx.tcx().data_layout.pointer_align.abi); - - let (addr, addr_align) = if allow_higher_align && align > slot_size { - (round_pointer_up_to_alignment(bx, ptr, align, bx.cx().type_i8p()), align) - } else { - (ptr, slot_size) - }; - - let aligned_size = size.align_to(slot_size).bytes() as i32; - let full_direct_size = bx.cx().const_i32(aligned_size); - let next = bx.inbounds_gep(bx.type_i8(), addr, &[full_direct_size]); - bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi); - - if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big { - let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32); - let adjusted = bx.inbounds_gep(bx.type_i8(), addr, &[adjusted_size]); - (bx.bitcast(adjusted, bx.cx().type_ptr_to(llty)), addr_align) - } else { - (bx.bitcast(addr, bx.cx().type_ptr_to(llty)), addr_align) - } -} - -fn emit_ptr_va_arg<'ll, 'tcx>( - bx: &mut Builder<'_, 'll, 'tcx>, - list: OperandRef<'tcx, &'ll Value>, - target_ty: Ty<'tcx>, - indirect: bool, - slot_size: Align, - allow_higher_align: bool, -) -> &'ll Value { - let layout = bx.cx.layout_of(target_ty); - let (llty, size, align) = if indirect { - ( - bx.cx.layout_of(bx.cx.tcx.mk_imm_ptr(target_ty)).llvm_type(bx.cx), - bx.cx.data_layout().pointer_size, - bx.cx.data_layout().pointer_align, - ) - } else { - (layout.llvm_type(bx.cx), layout.size, layout.align) - }; - let (addr, addr_align) = - emit_direct_ptr_va_arg(bx, list, llty, size, align.abi, slot_size, allow_higher_align); - if indirect { - let tmp_ret = bx.load(llty, addr, addr_align); - bx.load(bx.cx.layout_of(target_ty).llvm_type(bx.cx), tmp_ret, align.abi) - } else { - bx.load(llty, addr, addr_align) - } -} - -fn emit_aapcs_va_arg<'ll, 'tcx>( - bx: &mut Builder<'_, 'll, 'tcx>, - list: OperandRef<'tcx, &'ll Value>, - target_ty: Ty<'tcx>, -) -> &'ll Value { - // Implementation of the AAPCS64 calling convention for va_args see - // https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst - let va_list_addr = list.immediate(); - let va_list_layout = list.deref(bx.cx).layout; - let va_list_ty = va_list_layout.llvm_type(bx); - let layout = bx.cx.layout_of(target_ty); - - let maybe_reg = bx.append_sibling_block("va_arg.maybe_reg"); - let in_reg = bx.append_sibling_block("va_arg.in_reg"); - let on_stack = bx.append_sibling_block("va_arg.on_stack"); - let end = bx.append_sibling_block("va_arg.end"); - let zero = bx.const_i32(0); - let offset_align = Align::from_bytes(4).unwrap(); - - let gr_type = target_ty.is_any_ptr() || target_ty.is_integral(); - let (reg_off, reg_top_index, slot_size) = if gr_type { - let gr_offs = - bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); - let nreg = (layout.size.bytes() + 7) / 8; - (gr_offs, va_list_layout.llvm_field_index(bx.cx, 1), nreg * 8) - } else { - let vr_off = - bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 4)); - let nreg = (layout.size.bytes() + 15) / 16; - (vr_off, va_list_layout.llvm_field_index(bx.cx, 2), nreg * 16) - }; - - // if the offset >= 0 then the value will be on the stack - let mut reg_off_v = bx.load(bx.type_i32(), reg_off, offset_align); - let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero); - bx.cond_br(use_stack, on_stack, maybe_reg); - - // The value at this point might be in a register, but there is a chance that - // it could be on the stack so we have to update the offset and then check - // the offset again. - - bx.switch_to_block(maybe_reg); - if gr_type && layout.align.abi.bytes() > 8 { - reg_off_v = bx.add(reg_off_v, bx.const_i32(15)); - reg_off_v = bx.and(reg_off_v, bx.const_i32(-16)); - } - let new_reg_off_v = bx.add(reg_off_v, bx.const_i32(slot_size as i32)); - - bx.store(new_reg_off_v, reg_off, offset_align); - - // Check to see if we have overflowed the registers as a result of this. - // If we have then we need to use the stack for this value - let use_stack = bx.icmp(IntPredicate::IntSGT, new_reg_off_v, zero); - bx.cond_br(use_stack, on_stack, in_reg); - - bx.switch_to_block(in_reg); - let top_type = bx.type_i8p(); - let top = bx.struct_gep(va_list_ty, va_list_addr, reg_top_index); - let top = bx.load(top_type, top, bx.tcx().data_layout.pointer_align.abi); - - // reg_value = *(@top + reg_off_v); - let mut reg_addr = bx.gep(bx.type_i8(), top, &[reg_off_v]); - if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size { - // On big-endian systems the value is right-aligned in its slot. - let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32); - reg_addr = bx.gep(bx.type_i8(), reg_addr, &[offset]); - } - let reg_type = layout.llvm_type(bx); - let reg_addr = bx.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type)); - let reg_value = bx.load(reg_type, reg_addr, layout.align.abi); - bx.br(end); - - // On Stack block - bx.switch_to_block(on_stack); - let stack_value = - emit_ptr_va_arg(bx, list, target_ty, false, Align::from_bytes(8).unwrap(), true); - bx.br(end); - - bx.switch_to_block(end); - let val = - bx.phi(layout.immediate_llvm_type(bx), &[reg_value, stack_value], &[in_reg, on_stack]); - - val -} - -pub(super) fn emit_va_arg<'ll, 'tcx>( - bx: &mut Builder<'_, 'll, 'tcx>, - addr: OperandRef<'tcx, &'ll Value>, - target_ty: Ty<'tcx>, -) -> &'ll Value { - // Determine the va_arg implementation to use. The LLVM va_arg instruction - // is lacking in some instances, so we should only use it as a fallback. - let target = &bx.cx.tcx.sess.target; - let arch = &bx.cx.tcx.sess.target.arch; - match &**arch { - // Windows x86 - "x86" if target.is_like_windows => { - emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), false) - } - // Generic x86 - "x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true), - // Windows AArch64 - "aarch64" if target.is_like_windows => { - emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false) - } - // macOS / iOS AArch64 - "aarch64" if target.is_like_osx => { - emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true) - } - "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), - // Windows x86_64 - "x86_64" if target.is_like_windows => { - let target_ty_size = bx.cx.size_of(target_ty).bytes(); - let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two(); - emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false) - } - // For all other architecture/OS combinations fall back to using - // the LLVM va_arg instruction. - // https://llvm.org/docs/LangRef.html#va-arg-instruction - _ => bx.va_arg(addr.immediate(), bx.cx.layout_of(target_ty).llvm_type(bx.cx)), - } -} diff --git a/compiler/rustc_codegen_llvm/src/value.rs b/compiler/rustc_codegen_llvm/src/value.rs deleted file mode 100644 index 1338a229566..00000000000 --- a/compiler/rustc_codegen_llvm/src/value.rs +++ /dev/null @@ -1,32 +0,0 @@ -pub use crate::llvm::Value; - -use crate::llvm; - -use std::fmt; -use std::hash::{Hash, Hasher}; -use std::ptr; - -impl PartialEq for Value { - fn eq(&self, other: &Self) -> bool { - ptr::eq(self, other) - } -} - -impl Eq for Value {} - -impl Hash for Value { - fn hash(&self, hasher: &mut H) { - (self as *const Self).hash(hasher); - } -} - -impl fmt::Debug for Value { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str( - &llvm::build_string(|s| unsafe { - llvm::LLVMRustWriteValueToString(self, s); - }) - .expect("non-UTF8 value description from LLVM"), - ) - } -} diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml deleted file mode 100644 index 46d6344dbb2..00000000000 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ /dev/null @@ -1,48 +0,0 @@ -[package] -name = "rustc_codegen_ssa" -version = "0.0.0" -edition = "2021" - -[lib] -test = false - -[dependencies] -bitflags = "1.2.1" -cc = "1.0.69" -itertools = "0.10.1" -tracing = "0.1" -libc = "0.2.50" -jobserver = "0.1.22" -tempfile = "3.2" -thorin-dwp = "0.3" -pathdiff = "0.2.0" -serde_json = "1.0.59" -snap = "1" -smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } -regex = "1.4" - -rustc_serialize = { path = "../rustc_serialize" } -rustc_arena = { path = "../rustc_arena" } -rustc_ast = { path = "../rustc_ast" } -rustc_span = { path = "../rustc_span" } -rustc_middle = { path = "../rustc_middle" } -rustc_apfloat = { path = "../rustc_apfloat" } -rustc_attr = { path = "../rustc_attr" } -rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } -rustc_data_structures = { path = "../rustc_data_structures" } -rustc_errors = { path = "../rustc_errors" } -rustc_fs_util = { path = "../rustc_fs_util" } -rustc_hir = { path = "../rustc_hir" } -rustc_incremental = { path = "../rustc_incremental" } -rustc_index = { path = "../rustc_index" } -rustc_macros = { path = "../rustc_macros" } -rustc_metadata = { path = "../rustc_metadata" } -rustc_query_system = { path = "../rustc_query_system" } -rustc_target = { path = "../rustc_target" } -rustc_session = { path = "../rustc_session" } -rustc_const_eval = { path = "../rustc_const_eval" } - -[dependencies.object] -version = "0.29.0" -default-features = false -features = ["read_core", "elf", "macho", "pe", "unaligned", "archive", "write"] diff --git a/compiler/rustc_codegen_ssa/README.md b/compiler/rustc_codegen_ssa/README.md deleted file mode 100644 index 7b770187b75..00000000000 --- a/compiler/rustc_codegen_ssa/README.md +++ /dev/null @@ -1,3 +0,0 @@ -Please read the rustc-dev-guide chapter on [Backend Agnostic Codegen][bac]. - -[bac]: https://rustc-dev-guide.rust-lang.org/backend/backend-agnostic.html diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs deleted file mode 100644 index 0d2aa483d3d..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/archive.rs +++ /dev/null @@ -1,69 +0,0 @@ -use rustc_session::cstore::DllImport; -use rustc_session::Session; - -use std::io; -use std::path::{Path, PathBuf}; - -pub(super) fn find_library( - name: &str, - verbatim: bool, - search_paths: &[PathBuf], - sess: &Session, -) -> PathBuf { - // On Windows, static libraries sometimes show up as libfoo.a and other - // times show up as foo.lib - let oslibname = if verbatim { - name.to_string() - } else { - format!("{}{}{}", sess.target.staticlib_prefix, name, sess.target.staticlib_suffix) - }; - let unixlibname = format!("lib{}.a", name); - - for path in search_paths { - debug!("looking for {} inside {:?}", name, path); - let test = path.join(&oslibname); - if test.exists() { - return test; - } - if oslibname != unixlibname { - let test = path.join(&unixlibname); - if test.exists() { - return test; - } - } - } - sess.fatal(&format!( - "could not find native static library `{}`, \ - perhaps an -L flag is missing?", - name - )); -} - -pub trait ArchiveBuilderBuilder { - fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box + 'a>; - - /// Creates a DLL Import Library . - /// and returns the path on disk to that import library. - /// This functions doesn't take `self` so that it can be called from - /// `linker_with_args`, which is specialized on `ArchiveBuilder` but - /// doesn't take or create an instance of that type. - fn create_dll_import_lib( - &self, - sess: &Session, - lib_name: &str, - dll_imports: &[DllImport], - tmpdir: &Path, - ) -> PathBuf; -} - -pub trait ArchiveBuilder<'a> { - fn add_file(&mut self, path: &Path); - - fn add_archive( - &mut self, - archive: &Path, - skip: Box bool + 'static>, - ) -> io::Result<()>; - - fn build(self: Box, output: &Path) -> bool; -} diff --git a/compiler/rustc_codegen_ssa/src/back/command.rs b/compiler/rustc_codegen_ssa/src/back/command.rs deleted file mode 100644 index 9b0ba34135c..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/command.rs +++ /dev/null @@ -1,178 +0,0 @@ -//! A thin wrapper around `Command` in the standard library which allows us to -//! read the arguments that are built up. - -use std::ffi::{OsStr, OsString}; -use std::fmt; -use std::io; -use std::mem; -use std::process::{self, Output}; - -use rustc_target::spec::LldFlavor; - -#[derive(Clone)] -pub struct Command { - program: Program, - args: Vec, - env: Vec<(OsString, OsString)>, - env_remove: Vec, -} - -#[derive(Clone)] -enum Program { - Normal(OsString), - CmdBatScript(OsString), - Lld(OsString, LldFlavor), -} - -impl Command { - pub fn new>(program: P) -> Command { - Command::_new(Program::Normal(program.as_ref().to_owned())) - } - - pub fn bat_script>(program: P) -> Command { - Command::_new(Program::CmdBatScript(program.as_ref().to_owned())) - } - - pub fn lld>(program: P, flavor: LldFlavor) -> Command { - Command::_new(Program::Lld(program.as_ref().to_owned(), flavor)) - } - - fn _new(program: Program) -> Command { - Command { program, args: Vec::new(), env: Vec::new(), env_remove: Vec::new() } - } - - pub fn arg>(&mut self, arg: P) -> &mut Command { - self._arg(arg.as_ref()); - self - } - - pub fn args(&mut self, args: I) -> &mut Command - where - I: IntoIterator>, - { - for arg in args { - self._arg(arg.as_ref()); - } - self - } - - fn _arg(&mut self, arg: &OsStr) { - self.args.push(arg.to_owned()); - } - - pub fn env(&mut self, key: K, value: V) -> &mut Command - where - K: AsRef, - V: AsRef, - { - self._env(key.as_ref(), value.as_ref()); - self - } - - fn _env(&mut self, key: &OsStr, value: &OsStr) { - self.env.push((key.to_owned(), value.to_owned())); - } - - pub fn env_remove(&mut self, key: K) -> &mut Command - where - K: AsRef, - { - self._env_remove(key.as_ref()); - self - } - - fn _env_remove(&mut self, key: &OsStr) { - self.env_remove.push(key.to_owned()); - } - - pub fn output(&mut self) -> io::Result { - self.command().output() - } - - pub fn command(&self) -> process::Command { - let mut ret = match self.program { - Program::Normal(ref p) => process::Command::new(p), - Program::CmdBatScript(ref p) => { - let mut c = process::Command::new("cmd"); - c.arg("/c").arg(p); - c - } - Program::Lld(ref p, flavor) => { - let mut c = process::Command::new(p); - c.arg("-flavor").arg(flavor.as_str()); - if let LldFlavor::Wasm = flavor { - // LLVM expects host-specific formatting for @file - // arguments, but we always generate posix formatted files - // at this time. Indicate as such. - c.arg("--rsp-quoting=posix"); - } - c - } - }; - ret.args(&self.args); - ret.envs(self.env.clone()); - for k in &self.env_remove { - ret.env_remove(k); - } - ret - } - - // extensions - - pub fn get_args(&self) -> &[OsString] { - &self.args - } - - pub fn take_args(&mut self) -> Vec { - mem::take(&mut self.args) - } - - /// Returns a `true` if we're pretty sure that this'll blow OS spawn limits, - /// or `false` if we should attempt to spawn and see what the OS says. - pub fn very_likely_to_exceed_some_spawn_limit(&self) -> bool { - // We mostly only care about Windows in this method, on Unix the limits - // can be gargantuan anyway so we're pretty unlikely to hit them - if cfg!(unix) { - return false; - } - - // Right now LLD doesn't support the `@` syntax of passing an argument - // through files, so regardless of the platform we try to go to the OS - // on this one. - if let Program::Lld(..) = self.program { - return false; - } - - // Ok so on Windows to spawn a process is 32,768 characters in its - // command line [1]. Unfortunately we don't actually have access to that - // as it's calculated just before spawning. Instead we perform a - // poor-man's guess as to how long our command line will be. We're - // assuming here that we don't have to escape every character... - // - // Turns out though that `cmd.exe` has even smaller limits, 8192 - // characters [2]. Linkers can often be batch scripts (for example - // Emscripten, Gecko's current build system) which means that we're - // running through batch scripts. These linkers often just forward - // arguments elsewhere (and maybe tack on more), so if we blow 8192 - // bytes we'll typically cause them to blow as well. - // - // Basically as a result just perform an inflated estimate of what our - // command line will look like and test if it's > 8192 (we actually - // test against 6k to artificially inflate our estimate). If all else - // fails we'll fall back to the normal unix logic of testing the OS - // error code if we fail to spawn and automatically re-spawning the - // linker with smaller arguments. - // - // [1]: https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessa - // [2]: https://devblogs.microsoft.com/oldnewthing/?p=41553 - - let estimated_command_line_len = self.args.iter().map(|a| a.len()).sum::(); - estimated_command_line_len > 1024 * 6 - } -} - -impl fmt::Debug for Command { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.command().fmt(f) - } -} diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs deleted file mode 100644 index 63207803e32..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ /dev/null @@ -1,2800 +0,0 @@ -use rustc_arena::TypedArena; -use rustc_ast::CRATE_NODE_ID; -use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; -use rustc_data_structures::memmap::Mmap; -use rustc_data_structures::temp_dir::MaybeTempDir; -use rustc_errors::{ErrorGuaranteed, Handler}; -use rustc_fs_util::fix_windows_verbatim_for_gcc; -use rustc_hir::def_id::CrateNum; -use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME}; -use rustc_middle::middle::dependency_format::Linkage; -use rustc_middle::middle::exported_symbols::SymbolExportKind; -use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip}; -use rustc_session::config::{OutputFilenames, OutputType, PrintRequest, SplitDwarfKind}; -use rustc_session::cstore::DllImport; -use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename}; -use rustc_session::search_paths::PathKind; -use rustc_session::utils::NativeLibKind; -/// For all the linkers we support, and information they might -/// need out of the shared crate context before we get rid of it. -use rustc_session::{filesearch, Session}; -use rustc_span::symbol::Symbol; -use rustc_span::DebuggerVisualizerFile; -use rustc_target::spec::crt_objects::{CrtObjects, CrtObjectsFallback}; -use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor, SplitDebuginfo}; -use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, SanitizerSet, Target}; - -use super::archive::{find_library, ArchiveBuilder, ArchiveBuilderBuilder}; -use super::command::Command; -use super::linker::{self, Linker}; -use super::metadata::{create_rmeta_file, MetadataPosition}; -use super::rpath::{self, RPathConfig}; -use crate::{looks_like_rust_object_file, CodegenResults, CompiledModule, CrateInfo, NativeLib}; - -use cc::windows_registry; -use regex::Regex; -use tempfile::Builder as TempFileBuilder; - -use std::borrow::Borrow; -use std::cell::OnceCell; -use std::collections::BTreeSet; -use std::ffi::OsString; -use std::fs::{File, OpenOptions}; -use std::io::{BufWriter, Write}; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::process::{ExitStatus, Output, Stdio}; -use std::{ascii, char, env, fmt, fs, io, mem, str}; - -pub fn ensure_removed(diag_handler: &Handler, path: &Path) { - if let Err(e) = fs::remove_file(path) { - if e.kind() != io::ErrorKind::NotFound { - diag_handler.err(&format!("failed to remove {}: {}", path.display(), e)); - } - } -} - -/// Performs the linkage portion of the compilation phase. This will generate all -/// of the requested outputs for this compilation session. -pub fn link_binary<'a>( - sess: &'a Session, - archive_builder_builder: &dyn ArchiveBuilderBuilder, - codegen_results: &CodegenResults, - outputs: &OutputFilenames, -) -> Result<(), ErrorGuaranteed> { - let _timer = sess.timer("link_binary"); - let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata); - for &crate_type in sess.crate_types().iter() { - // Ignore executable crates if we have -Z no-codegen, as they will error. - if (sess.opts.unstable_opts.no_codegen || !sess.opts.output_types.should_codegen()) - && !output_metadata - && crate_type == CrateType::Executable - { - continue; - } - - if invalid_output_for_target(sess, crate_type) { - bug!( - "invalid output type `{:?}` for target os `{}`", - crate_type, - sess.opts.target_triple - ); - } - - sess.time("link_binary_check_files_are_writeable", || { - for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) { - check_file_is_writeable(obj, sess); - } - }); - - if outputs.outputs.should_link() { - let tmpdir = TempFileBuilder::new() - .prefix("rustc") - .tempdir() - .unwrap_or_else(|err| sess.fatal(&format!("couldn't create a temp dir: {}", err))); - let path = MaybeTempDir::new(tmpdir, sess.opts.cg.save_temps); - let out_filename = out_filename( - sess, - crate_type, - outputs, - codegen_results.crate_info.local_crate_name.as_str(), - ); - match crate_type { - CrateType::Rlib => { - let _timer = sess.timer("link_rlib"); - info!("preparing rlib to {:?}", out_filename); - link_rlib( - sess, - archive_builder_builder, - codegen_results, - RlibFlavor::Normal, - &path, - )? - .build(&out_filename); - } - CrateType::Staticlib => { - link_staticlib( - sess, - archive_builder_builder, - codegen_results, - &out_filename, - &path, - )?; - } - _ => { - link_natively( - sess, - archive_builder_builder, - crate_type, - &out_filename, - codegen_results, - path.as_ref(), - )?; - } - } - if sess.opts.json_artifact_notifications { - sess.parse_sess.span_diagnostic.emit_artifact_notification(&out_filename, "link"); - } - - if sess.prof.enabled() { - if let Some(artifact_name) = out_filename.file_name() { - // Record size for self-profiling - let file_size = std::fs::metadata(&out_filename).map(|m| m.len()).unwrap_or(0); - - sess.prof.artifact_size( - "linked_artifact", - artifact_name.to_string_lossy(), - file_size, - ); - } - } - } - } - - // Remove the temporary object file and metadata if we aren't saving temps. - sess.time("link_binary_remove_temps", || { - // If the user requests that temporaries are saved, don't delete any. - if sess.opts.cg.save_temps { - return; - } - - let maybe_remove_temps_from_module = - |preserve_objects: bool, preserve_dwarf_objects: bool, module: &CompiledModule| { - if !preserve_objects { - if let Some(ref obj) = module.object { - ensure_removed(sess.diagnostic(), obj); - } - } - - if !preserve_dwarf_objects { - if let Some(ref dwo_obj) = module.dwarf_object { - ensure_removed(sess.diagnostic(), dwo_obj); - } - } - }; - - let remove_temps_from_module = - |module: &CompiledModule| maybe_remove_temps_from_module(false, false, module); - - // Otherwise, always remove the metadata and allocator module temporaries. - if let Some(ref metadata_module) = codegen_results.metadata_module { - remove_temps_from_module(metadata_module); - } - - if let Some(ref allocator_module) = codegen_results.allocator_module { - remove_temps_from_module(allocator_module); - } - - // If no requested outputs require linking, then the object temporaries should - // be kept. - if !sess.opts.output_types.should_link() { - return; - } - - // Potentially keep objects for their debuginfo. - let (preserve_objects, preserve_dwarf_objects) = preserve_objects_for_their_debuginfo(sess); - debug!(?preserve_objects, ?preserve_dwarf_objects); - - for module in &codegen_results.modules { - maybe_remove_temps_from_module(preserve_objects, preserve_dwarf_objects, module); - } - }); - - Ok(()) -} - -pub fn each_linked_rlib( - info: &CrateInfo, - f: &mut dyn FnMut(CrateNum, &Path), -) -> Result<(), String> { - let crates = info.used_crates.iter(); - let mut fmts = None; - for (ty, list) in info.dependency_formats.iter() { - match ty { - CrateType::Executable - | CrateType::Staticlib - | CrateType::Cdylib - | CrateType::ProcMacro => { - fmts = Some(list); - break; - } - _ => {} - } - } - let Some(fmts) = fmts else { - return Err("could not find formats for rlibs".to_string()); - }; - for &cnum in crates { - match fmts.get(cnum.as_usize() - 1) { - Some(&Linkage::NotLinked | &Linkage::IncludedFromDylib) => continue, - Some(_) => {} - None => return Err("could not find formats for rlibs".to_string()), - } - let name = info.crate_name[&cnum]; - let used_crate_source = &info.used_crate_source[&cnum]; - if let Some((path, _)) = &used_crate_source.rlib { - f(cnum, &path); - } else { - if used_crate_source.rmeta.is_some() { - return Err(format!( - "could not find rlib for: `{}`, found rmeta (metadata) file", - name - )); - } else { - return Err(format!("could not find rlib for: `{}`", name)); - } - } - } - Ok(()) -} - -/// Create an 'rlib'. -/// -/// An rlib in its current incarnation is essentially a renamed .a file. The rlib primarily contains -/// the object file of the crate, but it also contains all of the object files from native -/// libraries. This is done by unzipping native libraries and inserting all of the contents into -/// this archive. -fn link_rlib<'a>( - sess: &'a Session, - archive_builder_builder: &dyn ArchiveBuilderBuilder, - codegen_results: &CodegenResults, - flavor: RlibFlavor, - tmpdir: &MaybeTempDir, -) -> Result + 'a>, ErrorGuaranteed> { - let lib_search_paths = archive_search_paths(sess); - - let mut ab = archive_builder_builder.new_archive_builder(sess); - - let trailing_metadata = match flavor { - RlibFlavor::Normal => { - let (metadata, metadata_position) = - create_rmeta_file(sess, codegen_results.metadata.raw_data()); - let metadata = emit_metadata(sess, &metadata, tmpdir); - match metadata_position { - MetadataPosition::First => { - // Most of the time metadata in rlib files is wrapped in a "dummy" object - // file for the target platform so the rlib can be processed entirely by - // normal linkers for the platform. Sometimes this is not possible however. - // If it is possible however, placing the metadata object first improves - // performance of getting metadata from rlibs. - ab.add_file(&metadata); - None - } - MetadataPosition::Last => Some(metadata), - } - } - - RlibFlavor::StaticlibBase => None, - }; - - for m in &codegen_results.modules { - if let Some(obj) = m.object.as_ref() { - ab.add_file(obj); - } - - if let Some(dwarf_obj) = m.dwarf_object.as_ref() { - ab.add_file(dwarf_obj); - } - } - - match flavor { - RlibFlavor::Normal => {} - RlibFlavor::StaticlibBase => { - let obj = codegen_results.allocator_module.as_ref().and_then(|m| m.object.as_ref()); - if let Some(obj) = obj { - ab.add_file(obj); - } - } - } - - // Note that in this loop we are ignoring the value of `lib.cfg`. That is, - // we may not be configured to actually include a static library if we're - // adding it here. That's because later when we consume this rlib we'll - // decide whether we actually needed the static library or not. - // - // To do this "correctly" we'd need to keep track of which libraries added - // which object files to the archive. We don't do that here, however. The - // #[link(cfg(..))] feature is unstable, though, and only intended to get - // liblibc working. In that sense the check below just indicates that if - // there are any libraries we want to omit object files for at link time we - // just exclude all custom object files. - // - // Eventually if we want to stabilize or flesh out the #[link(cfg(..))] - // feature then we'll need to figure out how to record what objects were - // loaded from the libraries found here and then encode that into the - // metadata of the rlib we're generating somehow. - for lib in codegen_results.crate_info.used_libraries.iter() { - match lib.kind { - NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) } - if flavor == RlibFlavor::Normal => - { - // Don't allow mixing +bundle with +whole_archive since an rlib may contain - // multiple native libs, some of which are +whole-archive and some of which are - // -whole-archive and it isn't clear how we can currently handle such a - // situation correctly. - // See https://github.com/rust-lang/rust/issues/88085#issuecomment-901050897 - sess.err( - "the linking modifiers `+bundle` and `+whole-archive` are not compatible \ - with each other when generating rlibs", - ); - } - NativeLibKind::Static { bundle: None | Some(true), .. } => {} - NativeLibKind::Static { bundle: Some(false), .. } - | NativeLibKind::Dylib { .. } - | NativeLibKind::Framework { .. } - | NativeLibKind::RawDylib - | NativeLibKind::LinkArg - | NativeLibKind::Unspecified => continue, - } - if let Some(name) = lib.name { - let location = - find_library(name.as_str(), lib.verbatim.unwrap_or(false), &lib_search_paths, sess); - ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|e| { - sess.fatal(&format!( - "failed to add native library {}: {}", - location.to_string_lossy(), - e - )); - }); - } - } - - for (raw_dylib_name, raw_dylib_imports) in - collate_raw_dylibs(sess, &codegen_results.crate_info.used_libraries)? - { - let output_path = archive_builder_builder.create_dll_import_lib( - sess, - &raw_dylib_name, - &raw_dylib_imports, - tmpdir.as_ref(), - ); - - ab.add_archive(&output_path, Box::new(|_| false)).unwrap_or_else(|e| { - sess.fatal(&format!("failed to add native library {}: {}", output_path.display(), e)); - }); - } - - if let Some(trailing_metadata) = trailing_metadata { - // Note that it is important that we add all of our non-object "magical - // files" *after* all of the object files in the archive. The reason for - // this is as follows: - // - // * When performing LTO, this archive will be modified to remove - // objects from above. The reason for this is described below. - // - // * When the system linker looks at an archive, it will attempt to - // determine the architecture of the archive in order to see whether its - // linkable. - // - // The algorithm for this detection is: iterate over the files in the - // archive. Skip magical SYMDEF names. Interpret the first file as an - // object file. Read architecture from the object file. - // - // * As one can probably see, if "metadata" and "foo.bc" were placed - // before all of the objects, then the architecture of this archive would - // not be correctly inferred once 'foo.o' is removed. - // - // * Most of the time metadata in rlib files is wrapped in a "dummy" object - // file for the target platform so the rlib can be processed entirely by - // normal linkers for the platform. Sometimes this is not possible however. - // - // Basically, all this means is that this code should not move above the - // code above. - ab.add_file(&trailing_metadata); - } - - return Ok(ab); -} - -/// Extract all symbols defined in raw-dylib libraries, collated by library name. -/// -/// If we have multiple extern blocks that specify symbols defined in the same raw-dylib library, -/// then the CodegenResults value contains one NativeLib instance for each block. However, the -/// linker appears to expect only a single import library for each library used, so we need to -/// collate the symbols together by library name before generating the import libraries. -fn collate_raw_dylibs( - sess: &Session, - used_libraries: &[NativeLib], -) -> Result)>, ErrorGuaranteed> { - // Use index maps to preserve original order of imports and libraries. - let mut dylib_table = FxIndexMap::>::default(); - - for lib in used_libraries { - if lib.kind == NativeLibKind::RawDylib { - let ext = if matches!(lib.verbatim, Some(true)) { "" } else { ".dll" }; - let name = format!("{}{}", lib.name.expect("unnamed raw-dylib library"), ext); - let imports = dylib_table.entry(name.clone()).or_default(); - for import in &lib.dll_imports { - if let Some(old_import) = imports.insert(import.name, import) { - // FIXME: when we add support for ordinals, figure out if we need to do anything - // if we have two DllImport values with the same name but different ordinals. - if import.calling_convention != old_import.calling_convention { - sess.span_err( - import.span, - &format!( - "multiple declarations of external function `{}` from \ - library `{}` have different calling conventions", - import.name, name, - ), - ); - } - } - } - } - } - sess.compile_status()?; - Ok(dylib_table - .into_iter() - .map(|(name, imports)| { - (name, imports.into_iter().map(|(_, import)| import.clone()).collect()) - }) - .collect()) -} - -/// Create a static archive. -/// -/// This is essentially the same thing as an rlib, but it also involves adding all of the upstream -/// crates' objects into the archive. This will slurp in all of the native libraries of upstream -/// dependencies as well. -/// -/// Additionally, there's no way for us to link dynamic libraries, so we warn about all dynamic -/// library dependencies that they're not linked in. -/// -/// There's no need to include metadata in a static archive, so ensure to not link in the metadata -/// object file (and also don't prepare the archive with a metadata file). -fn link_staticlib<'a>( - sess: &'a Session, - archive_builder_builder: &dyn ArchiveBuilderBuilder, - codegen_results: &CodegenResults, - out_filename: &Path, - tempdir: &MaybeTempDir, -) -> Result<(), ErrorGuaranteed> { - info!("preparing staticlib to {:?}", out_filename); - let mut ab = link_rlib( - sess, - archive_builder_builder, - codegen_results, - RlibFlavor::StaticlibBase, - tempdir, - )?; - let mut all_native_libs = vec![]; - - let res = each_linked_rlib(&codegen_results.crate_info, &mut |cnum, path| { - let name = codegen_results.crate_info.crate_name[&cnum]; - let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; - - // Here when we include the rlib into our staticlib we need to make a - // decision whether to include the extra object files along the way. - // These extra object files come from statically included native - // libraries, but they may be cfg'd away with #[link(cfg(..))]. - // - // This unstable feature, though, only needs liblibc to work. The only - // use case there is where musl is statically included in liblibc.rlib, - // so if we don't want the included version we just need to skip it. As - // a result the logic here is that if *any* linked library is cfg'd away - // we just skip all object files. - // - // Clearly this is not sufficient for a general purpose feature, and - // we'd want to read from the library's metadata to determine which - // object files come from where and selectively skip them. - let skip_object_files = native_libs.iter().any(|lib| { - matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. }) - && !relevant_lib(sess, lib) - }); - - let lto = are_upstream_rust_objects_already_included(sess) - && !ignored_for_lto(sess, &codegen_results.crate_info, cnum); - - // Ignoring obj file starting with the crate name - // as simple comparison is not enough - there - // might be also an extra name suffix - let obj_start = name.as_str().to_owned(); - - ab.add_archive( - path, - Box::new(move |fname: &str| { - // Ignore metadata files, no matter the name. - if fname == METADATA_FILENAME { - return true; - } - - // Don't include Rust objects if LTO is enabled - if lto && looks_like_rust_object_file(fname) { - return true; - } - - // Otherwise if this is *not* a rust object and we're skipping - // objects then skip this file - if skip_object_files && (!fname.starts_with(&obj_start) || !fname.ends_with(".o")) { - return true; - } - - // ok, don't skip this - false - }), - ) - .unwrap(); - - all_native_libs.extend(codegen_results.crate_info.native_libraries[&cnum].iter().cloned()); - }); - if let Err(e) = res { - sess.fatal(&e); - } - - ab.build(out_filename); - - if !all_native_libs.is_empty() { - if sess.opts.prints.contains(&PrintRequest::NativeStaticLibs) { - print_native_static_libs(sess, &all_native_libs); - } - } - - Ok(()) -} - -fn escape_stdout_stderr_string(s: &[u8]) -> String { - str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| { - let mut x = "Non-UTF-8 output: ".to_string(); - x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from)); - x - }) -} - -/// Use `thorin` (rust implementation of a dwarf packaging utility) to link DWARF objects into a -/// DWARF package. -fn link_dwarf_object<'a>( - sess: &'a Session, - cg_results: &CodegenResults, - executable_out_filename: &Path, -) { - let dwp_out_filename = executable_out_filename.with_extension("dwp"); - debug!(?dwp_out_filename, ?executable_out_filename); - - #[derive(Default)] - struct ThorinSession { - arena_data: TypedArena>, - arena_mmap: TypedArena, - arena_relocations: TypedArena, - } - - impl ThorinSession { - fn alloc_mmap<'arena>(&'arena self, data: Mmap) -> &'arena Mmap { - (*self.arena_mmap.alloc(data)).borrow() - } - } - - impl thorin::Session for ThorinSession { - fn alloc_data<'arena>(&'arena self, data: Vec) -> &'arena [u8] { - (*self.arena_data.alloc(data)).borrow() - } - - fn alloc_relocation<'arena>(&'arena self, data: Relocations) -> &'arena Relocations { - (*self.arena_relocations.alloc(data)).borrow() - } - - fn read_input<'arena>(&'arena self, path: &Path) -> std::io::Result<&'arena [u8]> { - let file = File::open(&path)?; - let mmap = (unsafe { Mmap::map(file) })?; - Ok(self.alloc_mmap(mmap)) - } - } - - match sess.time("run_thorin", || -> Result<(), thorin::Error> { - let thorin_sess = ThorinSession::default(); - let mut package = thorin::DwarfPackage::new(&thorin_sess); - - // Input objs contain .o/.dwo files from the current crate. - match sess.opts.unstable_opts.split_dwarf_kind { - SplitDwarfKind::Single => { - for input_obj in cg_results.modules.iter().filter_map(|m| m.object.as_ref()) { - package.add_input_object(input_obj)?; - } - } - SplitDwarfKind::Split => { - for input_obj in cg_results.modules.iter().filter_map(|m| m.dwarf_object.as_ref()) { - package.add_input_object(input_obj)?; - } - } - } - - // Input rlibs contain .o/.dwo files from dependencies. - let input_rlibs = cg_results - .crate_info - .used_crate_source - .values() - .filter_map(|csource| csource.rlib.as_ref()) - .map(|(path, _)| path); - for input_rlib in input_rlibs { - debug!(?input_rlib); - package.add_input_object(input_rlib)?; - } - - // Failing to read the referenced objects is expected for dependencies where the path in the - // executable will have been cleaned by Cargo, but the referenced objects will be contained - // within rlibs provided as inputs. - // - // If paths have been remapped, then .o/.dwo files from the current crate also won't be - // found, but are provided explicitly above. - // - // Adding an executable is primarily done to make `thorin` check that all the referenced - // dwarf objects are found in the end. - package.add_executable( - &executable_out_filename, - thorin::MissingReferencedObjectBehaviour::Skip, - )?; - - let output = package.finish()?.write()?; - let mut output_stream = BufWriter::new( - OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(true) - .open(dwp_out_filename)?, - ); - output_stream.write_all(&output)?; - output_stream.flush()?; - - Ok(()) - }) { - Ok(()) => {} - Err(e) => { - sess.struct_err("linking dwarf objects with thorin failed") - .note(&format!("{:?}", e)) - .emit(); - sess.abort_if_errors(); - } - } -} - -/// Create a dynamic library or executable. -/// -/// This will invoke the system linker/cc to create the resulting file. This links to all upstream -/// files as well. -fn link_natively<'a>( - sess: &'a Session, - archive_builder_builder: &dyn ArchiveBuilderBuilder, - crate_type: CrateType, - out_filename: &Path, - codegen_results: &CodegenResults, - tmpdir: &Path, -) -> Result<(), ErrorGuaranteed> { - info!("preparing {:?} to {:?}", crate_type, out_filename); - let (linker_path, flavor) = linker_and_flavor(sess); - let mut cmd = linker_with_args( - &linker_path, - flavor, - sess, - archive_builder_builder, - crate_type, - tmpdir, - out_filename, - codegen_results, - )?; - - linker::disable_localization(&mut cmd); - - for &(ref k, ref v) in sess.target.link_env.as_ref() { - cmd.env(k.as_ref(), v.as_ref()); - } - for k in sess.target.link_env_remove.as_ref() { - cmd.env_remove(k.as_ref()); - } - - if sess.opts.prints.contains(&PrintRequest::LinkArgs) { - println!("{:?}", &cmd); - } - - // May have not found libraries in the right formats. - sess.abort_if_errors(); - - // Invoke the system linker - info!("{:?}", &cmd); - let retry_on_segfault = env::var("RUSTC_RETRY_LINKER_ON_SEGFAULT").is_ok(); - let unknown_arg_regex = - Regex::new(r"(unknown|unrecognized) (command line )?(option|argument)").unwrap(); - let mut prog; - let mut i = 0; - loop { - i += 1; - prog = sess.time("run_linker", || exec_linker(sess, &cmd, out_filename, tmpdir)); - let Ok(ref output) = prog else { - break; - }; - if output.status.success() { - break; - } - let mut out = output.stderr.clone(); - out.extend(&output.stdout); - let out = String::from_utf8_lossy(&out); - - // Check to see if the link failed with an error message that indicates it - // doesn't recognize the -no-pie option. If so, re-perform the link step - // without it. This is safe because if the linker doesn't support -no-pie - // then it should not default to linking executables as pie. Different - // versions of gcc seem to use different quotes in the error message so - // don't check for them. - if sess.target.linker_is_gnu - && flavor != LinkerFlavor::Ld - && unknown_arg_regex.is_match(&out) - && out.contains("-no-pie") - && cmd.get_args().iter().any(|e| e.to_string_lossy() == "-no-pie") - { - info!("linker output: {:?}", out); - warn!("Linker does not support -no-pie command line option. Retrying without."); - for arg in cmd.take_args() { - if arg.to_string_lossy() != "-no-pie" { - cmd.arg(arg); - } - } - info!("{:?}", &cmd); - continue; - } - - // Detect '-static-pie' used with an older version of gcc or clang not supporting it. - // Fallback from '-static-pie' to '-static' in that case. - if sess.target.linker_is_gnu - && flavor != LinkerFlavor::Ld - && unknown_arg_regex.is_match(&out) - && (out.contains("-static-pie") || out.contains("--no-dynamic-linker")) - && cmd.get_args().iter().any(|e| e.to_string_lossy() == "-static-pie") - { - info!("linker output: {:?}", out); - warn!( - "Linker does not support -static-pie command line option. Retrying with -static instead." - ); - // Mirror `add_(pre,post)_link_objects` to replace CRT objects. - let self_contained = crt_objects_fallback(sess, crate_type); - let opts = &sess.target; - let pre_objects = if self_contained { - &opts.pre_link_objects_fallback - } else { - &opts.pre_link_objects - }; - let post_objects = if self_contained { - &opts.post_link_objects_fallback - } else { - &opts.post_link_objects - }; - let get_objects = |objects: &CrtObjects, kind| { - objects - .get(&kind) - .iter() - .copied() - .flatten() - .map(|obj| get_object_file_path(sess, obj, self_contained).into_os_string()) - .collect::>() - }; - let pre_objects_static_pie = get_objects(pre_objects, LinkOutputKind::StaticPicExe); - let post_objects_static_pie = get_objects(post_objects, LinkOutputKind::StaticPicExe); - let mut pre_objects_static = get_objects(pre_objects, LinkOutputKind::StaticNoPicExe); - let mut post_objects_static = get_objects(post_objects, LinkOutputKind::StaticNoPicExe); - // Assume that we know insertion positions for the replacement arguments from replaced - // arguments, which is true for all supported targets. - assert!(pre_objects_static.is_empty() || !pre_objects_static_pie.is_empty()); - assert!(post_objects_static.is_empty() || !post_objects_static_pie.is_empty()); - for arg in cmd.take_args() { - if arg.to_string_lossy() == "-static-pie" { - // Replace the output kind. - cmd.arg("-static"); - } else if pre_objects_static_pie.contains(&arg) { - // Replace the pre-link objects (replace the first and remove the rest). - cmd.args(mem::take(&mut pre_objects_static)); - } else if post_objects_static_pie.contains(&arg) { - // Replace the post-link objects (replace the first and remove the rest). - cmd.args(mem::take(&mut post_objects_static)); - } else { - cmd.arg(arg); - } - } - info!("{:?}", &cmd); - continue; - } - - // Here's a terribly awful hack that really shouldn't be present in any - // compiler. Here an environment variable is supported to automatically - // retry the linker invocation if the linker looks like it segfaulted. - // - // Gee that seems odd, normally segfaults are things we want to know - // about! Unfortunately though in rust-lang/rust#38878 we're - // experiencing the linker segfaulting on Travis quite a bit which is - // causing quite a bit of pain to land PRs when they spuriously fail - // due to a segfault. - // - // The issue #38878 has some more debugging information on it as well, - // but this unfortunately looks like it's just a race condition in - // macOS's linker with some thread pool working in the background. It - // seems that no one currently knows a fix for this so in the meantime - // we're left with this... - if !retry_on_segfault || i > 3 { - break; - } - let msg_segv = "clang: error: unable to execute command: Segmentation fault: 11"; - let msg_bus = "clang: error: unable to execute command: Bus error: 10"; - if out.contains(msg_segv) || out.contains(msg_bus) { - warn!( - ?cmd, %out, - "looks like the linker segfaulted when we tried to call it, \ - automatically retrying again", - ); - continue; - } - - if is_illegal_instruction(&output.status) { - warn!( - ?cmd, %out, status = %output.status, - "looks like the linker hit an illegal instruction when we \ - tried to call it, automatically retrying again.", - ); - continue; - } - - #[cfg(unix)] - fn is_illegal_instruction(status: &ExitStatus) -> bool { - use std::os::unix::prelude::*; - status.signal() == Some(libc::SIGILL) - } - - #[cfg(not(unix))] - fn is_illegal_instruction(_status: &ExitStatus) -> bool { - false - } - } - - match prog { - Ok(prog) => { - if !prog.status.success() { - let mut output = prog.stderr.clone(); - output.extend_from_slice(&prog.stdout); - let escaped_output = escape_stdout_stderr_string(&output); - let mut err = sess.struct_err(&format!( - "linking with `{}` failed: {}", - linker_path.display(), - prog.status - )); - err.note(&format!("{:?}", &cmd)).note(&escaped_output); - if escaped_output.contains("undefined reference to") { - err.help( - "some `extern` functions couldn't be found; some native libraries may \ - need to be installed or have their path specified", - ); - err.note("use the `-l` flag to specify native libraries to link"); - err.note("use the `cargo:rustc-link-lib` directive to specify the native \ - libraries to link with Cargo (see https://doc.rust-lang.org/cargo/reference/build-scripts.html#cargorustc-link-libkindname)"); - } - err.emit(); - - // If MSVC's `link.exe` was expected but the return code - // is not a Microsoft LNK error then suggest a way to fix or - // install the Visual Studio build tools. - if let Some(code) = prog.status.code() { - if sess.target.is_like_msvc - && flavor == LinkerFlavor::Msvc - // Respect the command line override - && sess.opts.cg.linker.is_none() - // Match exactly "link.exe" - && linker_path.to_str() == Some("link.exe") - // All Microsoft `link.exe` linking error codes are - // four digit numbers in the range 1000 to 9999 inclusive - && (code < 1000 || code > 9999) - { - let is_vs_installed = windows_registry::find_vs_version().is_ok(); - let has_linker = windows_registry::find_tool( - &sess.opts.target_triple.triple(), - "link.exe", - ) - .is_some(); - - sess.note_without_error("`link.exe` returned an unexpected error"); - if is_vs_installed && has_linker { - // the linker is broken - sess.note_without_error( - "the Visual Studio build tools may need to be repaired \ - using the Visual Studio installer", - ); - sess.note_without_error( - "or a necessary component may be missing from the \ - \"C++ build tools\" workload", - ); - } else if is_vs_installed { - // the linker is not installed - sess.note_without_error( - "in the Visual Studio installer, ensure the \ - \"C++ build tools\" workload is selected", - ); - } else { - // visual studio is not installed - sess.note_without_error( - "you may need to install Visual Studio build tools with the \ - \"C++ build tools\" workload", - ); - } - } - } - - sess.abort_if_errors(); - } - info!("linker stderr:\n{}", escape_stdout_stderr_string(&prog.stderr)); - info!("linker stdout:\n{}", escape_stdout_stderr_string(&prog.stdout)); - } - Err(e) => { - let linker_not_found = e.kind() == io::ErrorKind::NotFound; - - let mut linker_error = { - if linker_not_found { - sess.struct_err(&format!("linker `{}` not found", linker_path.display())) - } else { - sess.struct_err(&format!( - "could not exec the linker `{}`", - linker_path.display() - )) - } - }; - - linker_error.note(&e.to_string()); - - if !linker_not_found { - linker_error.note(&format!("{:?}", &cmd)); - } - - linker_error.emit(); - - if sess.target.is_like_msvc && linker_not_found { - sess.note_without_error( - "the msvc targets depend on the msvc linker \ - but `link.exe` was not found", - ); - sess.note_without_error( - "please ensure that VS 2013, VS 2015, VS 2017, VS 2019 or VS 2022 \ - was installed with the Visual C++ option", - ); - } - sess.abort_if_errors(); - } - } - - match sess.split_debuginfo() { - // If split debug information is disabled or located in individual files - // there's nothing to do here. - SplitDebuginfo::Off | SplitDebuginfo::Unpacked => {} - - // If packed split-debuginfo is requested, but the final compilation - // doesn't actually have any debug information, then we skip this step. - SplitDebuginfo::Packed if sess.opts.debuginfo == DebugInfo::None => {} - - // On macOS the external `dsymutil` tool is used to create the packed - // debug information. Note that this will read debug information from - // the objects on the filesystem which we'll clean up later. - SplitDebuginfo::Packed if sess.target.is_like_osx => { - let prog = Command::new("dsymutil").arg(out_filename).output(); - match prog { - Ok(prog) => { - if !prog.status.success() { - let mut output = prog.stderr.clone(); - output.extend_from_slice(&prog.stdout); - sess.struct_warn(&format!( - "processing debug info with `dsymutil` failed: {}", - prog.status - )) - .note(&escape_string(&output)) - .emit(); - } - } - Err(e) => sess.fatal(&format!("unable to run `dsymutil`: {}", e)), - } - } - - // On MSVC packed debug information is produced by the linker itself so - // there's no need to do anything else here. - SplitDebuginfo::Packed if sess.target.is_like_windows => {} - - // ... and otherwise we're processing a `*.dwp` packed dwarf file. - // - // We cannot rely on the .o paths in the executable because they may have been - // remapped by --remap-path-prefix and therefore invalid, so we need to provide - // the .o/.dwo paths explicitly. - SplitDebuginfo::Packed => link_dwarf_object(sess, codegen_results, out_filename), - } - - let strip = strip_value(sess); - - if sess.target.is_like_osx { - match (strip, crate_type) { - (Strip::Debuginfo, _) => strip_symbols_in_osx(sess, &out_filename, Some("-S")), - // Per the manpage, `-x` is the maximum safe strip level for dynamic libraries. (#93988) - (Strip::Symbols, CrateType::Dylib | CrateType::Cdylib | CrateType::ProcMacro) => { - strip_symbols_in_osx(sess, &out_filename, Some("-x")) - } - (Strip::Symbols, _) => strip_symbols_in_osx(sess, &out_filename, None), - (Strip::None, _) => {} - } - } - - Ok(()) -} - -// Temporarily support both -Z strip and -C strip -fn strip_value(sess: &Session) -> Strip { - match (sess.opts.unstable_opts.strip, sess.opts.cg.strip) { - (s, Strip::None) => s, - (_, s) => s, - } -} - -fn strip_symbols_in_osx<'a>(sess: &'a Session, out_filename: &Path, option: Option<&str>) { - let mut cmd = Command::new("strip"); - if let Some(option) = option { - cmd.arg(option); - } - let prog = cmd.arg(out_filename).output(); - match prog { - Ok(prog) => { - if !prog.status.success() { - let mut output = prog.stderr.clone(); - output.extend_from_slice(&prog.stdout); - sess.struct_warn(&format!( - "stripping debug info with `strip` failed: {}", - prog.status - )) - .note(&escape_string(&output)) - .emit(); - } - } - Err(e) => sess.fatal(&format!("unable to run `strip`: {}", e)), - } -} - -fn escape_string(s: &[u8]) -> String { - str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| { - let mut x = "Non-UTF-8 output: ".to_string(); - x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from)); - x - }) -} - -fn add_sanitizer_libraries(sess: &Session, crate_type: CrateType, linker: &mut dyn Linker) { - // On macOS the runtimes are distributed as dylibs which should be linked to - // both executables and dynamic shared objects. Everywhere else the runtimes - // are currently distributed as static libraries which should be linked to - // executables only. - let needs_runtime = match crate_type { - CrateType::Executable => true, - CrateType::Dylib | CrateType::Cdylib | CrateType::ProcMacro => sess.target.is_like_osx, - CrateType::Rlib | CrateType::Staticlib => false, - }; - - if !needs_runtime { - return; - } - - let sanitizer = sess.opts.unstable_opts.sanitizer; - if sanitizer.contains(SanitizerSet::ADDRESS) { - link_sanitizer_runtime(sess, linker, "asan"); - } - if sanitizer.contains(SanitizerSet::LEAK) { - link_sanitizer_runtime(sess, linker, "lsan"); - } - if sanitizer.contains(SanitizerSet::MEMORY) { - link_sanitizer_runtime(sess, linker, "msan"); - } - if sanitizer.contains(SanitizerSet::THREAD) { - link_sanitizer_runtime(sess, linker, "tsan"); - } - if sanitizer.contains(SanitizerSet::HWADDRESS) { - link_sanitizer_runtime(sess, linker, "hwasan"); - } -} - -fn link_sanitizer_runtime(sess: &Session, linker: &mut dyn Linker, name: &str) { - fn find_sanitizer_runtime(sess: &Session, filename: &str) -> PathBuf { - let session_tlib = - filesearch::make_target_lib_path(&sess.sysroot, sess.opts.target_triple.triple()); - let path = session_tlib.join(filename); - if path.exists() { - return session_tlib; - } else { - let default_sysroot = filesearch::get_or_default_sysroot(); - let default_tlib = filesearch::make_target_lib_path( - &default_sysroot, - sess.opts.target_triple.triple(), - ); - return default_tlib; - } - } - - let channel = option_env!("CFG_RELEASE_CHANNEL") - .map(|channel| format!("-{}", channel)) - .unwrap_or_default(); - - if sess.target.is_like_osx { - // On Apple platforms, the sanitizer is always built as a dylib, and - // LLVM will link to `@rpath/*.dylib`, so we need to specify an - // rpath to the library as well (the rpath should be absolute, see - // PR #41352 for details). - let filename = format!("rustc{}_rt.{}", channel, name); - let path = find_sanitizer_runtime(&sess, &filename); - let rpath = path.to_str().expect("non-utf8 component in path"); - linker.args(&["-Wl,-rpath", "-Xlinker", rpath]); - linker.link_dylib(&filename, false, true); - } else { - let filename = format!("librustc{}_rt.{}.a", channel, name); - let path = find_sanitizer_runtime(&sess, &filename).join(&filename); - linker.link_whole_rlib(&path); - } -} - -/// Returns a boolean indicating whether the specified crate should be ignored -/// during LTO. -/// -/// Crates ignored during LTO are not lumped together in the "massive object -/// file" that we create and are linked in their normal rlib states. See -/// comments below for what crates do not participate in LTO. -/// -/// It's unusual for a crate to not participate in LTO. Typically only -/// compiler-specific and unstable crates have a reason to not participate in -/// LTO. -pub fn ignored_for_lto(sess: &Session, info: &CrateInfo, cnum: CrateNum) -> bool { - // If our target enables builtin function lowering in LLVM then the - // crates providing these functions don't participate in LTO (e.g. - // no_builtins or compiler builtins crates). - !sess.target.no_builtins - && (info.compiler_builtins == Some(cnum) || info.is_no_builtins.contains(&cnum)) -} - -// This functions tries to determine the appropriate linker (and corresponding LinkerFlavor) to use -pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { - fn infer_from( - sess: &Session, - linker: Option, - flavor: Option, - ) -> Option<(PathBuf, LinkerFlavor)> { - match (linker, flavor) { - (Some(linker), Some(flavor)) => Some((linker, flavor)), - // only the linker flavor is known; use the default linker for the selected flavor - (None, Some(flavor)) => Some(( - PathBuf::from(match flavor { - LinkerFlavor::Em => { - if cfg!(windows) { - "emcc.bat" - } else { - "emcc" - } - } - LinkerFlavor::Gcc => { - if cfg!(any(target_os = "solaris", target_os = "illumos")) { - // On historical Solaris systems, "cc" may have - // been Sun Studio, which is not flag-compatible - // with "gcc". This history casts a long shadow, - // and many modern illumos distributions today - // ship GCC as "gcc" without also making it - // available as "cc". - "gcc" - } else { - "cc" - } - } - LinkerFlavor::Ld => "ld", - LinkerFlavor::Msvc => "link.exe", - LinkerFlavor::Lld(_) => "lld", - LinkerFlavor::PtxLinker => "rust-ptx-linker", - LinkerFlavor::BpfLinker => "bpf-linker", - LinkerFlavor::L4Bender => "l4-bender", - }), - flavor, - )), - (Some(linker), None) => { - let stem = linker.file_stem().and_then(|stem| stem.to_str()).unwrap_or_else(|| { - sess.fatal("couldn't extract file stem from specified linker") - }); - - let flavor = if stem == "emcc" { - LinkerFlavor::Em - } else if stem == "gcc" - || stem.ends_with("-gcc") - || stem == "clang" - || stem.ends_with("-clang") - { - LinkerFlavor::Gcc - } else if stem == "wasm-ld" || stem.ends_with("-wasm-ld") { - LinkerFlavor::Lld(LldFlavor::Wasm) - } else if stem == "ld" || stem == "ld.lld" || stem.ends_with("-ld") { - LinkerFlavor::Ld - } else if stem == "link" || stem == "lld-link" { - LinkerFlavor::Msvc - } else if stem == "lld" || stem == "rust-lld" { - LinkerFlavor::Lld(sess.target.lld_flavor) - } else { - // fall back to the value in the target spec - sess.target.linker_flavor - }; - - Some((linker, flavor)) - } - (None, None) => None, - } - } - - // linker and linker flavor specified via command line have precedence over what the target - // specification specifies - if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) { - return ret; - } - - if let Some(ret) = infer_from( - sess, - sess.target.linker.as_deref().map(PathBuf::from), - Some(sess.target.linker_flavor), - ) { - return ret; - } - - bug!("Not enough information provided to determine how to invoke the linker"); -} - -/// Returns a pair of boolean indicating whether we should preserve the object and -/// dwarf object files on the filesystem for their debug information. This is often -/// useful with split-dwarf like schemes. -fn preserve_objects_for_their_debuginfo(sess: &Session) -> (bool, bool) { - // If the objects don't have debuginfo there's nothing to preserve. - if sess.opts.debuginfo == config::DebugInfo::None { - return (false, false); - } - - // If we're only producing artifacts that are archives, no need to preserve - // the objects as they're losslessly contained inside the archives. - if sess.crate_types().iter().all(|&x| x.is_archive()) { - return (false, false); - } - - match (sess.split_debuginfo(), sess.opts.unstable_opts.split_dwarf_kind) { - // If there is no split debuginfo then do not preserve objects. - (SplitDebuginfo::Off, _) => (false, false), - // If there is packed split debuginfo, then the debuginfo in the objects - // has been packaged and the objects can be deleted. - (SplitDebuginfo::Packed, _) => (false, false), - // If there is unpacked split debuginfo and the current target can not use - // split dwarf, then keep objects. - (SplitDebuginfo::Unpacked, _) if !sess.target_can_use_split_dwarf() => (true, false), - // If there is unpacked split debuginfo and the target can use split dwarf, then - // keep the object containing that debuginfo (whether that is an object file or - // dwarf object file depends on the split dwarf kind). - (SplitDebuginfo::Unpacked, SplitDwarfKind::Single) => (true, false), - (SplitDebuginfo::Unpacked, SplitDwarfKind::Split) => (false, true), - } -} - -fn archive_search_paths(sess: &Session) -> Vec { - sess.target_filesearch(PathKind::Native).search_path_dirs() -} - -#[derive(PartialEq)] -enum RlibFlavor { - Normal, - StaticlibBase, -} - -fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) { - let lib_args: Vec<_> = all_native_libs - .iter() - .filter(|l| relevant_lib(sess, l)) - .filter_map(|lib| { - let name = lib.name?; - match lib.kind { - NativeLibKind::Static { bundle: Some(false), .. } - | NativeLibKind::Dylib { .. } - | NativeLibKind::Unspecified => { - let verbatim = lib.verbatim.unwrap_or(false); - if sess.target.is_like_msvc { - Some(format!("{}{}", name, if verbatim { "" } else { ".lib" })) - } else if sess.target.linker_is_gnu { - Some(format!("-l{}{}", if verbatim { ":" } else { "" }, name)) - } else { - Some(format!("-l{}", name)) - } - } - NativeLibKind::Framework { .. } => { - // ld-only syntax, since there are no frameworks in MSVC - Some(format!("-framework {}", name)) - } - // These are included, no need to print them - NativeLibKind::Static { bundle: None | Some(true), .. } - | NativeLibKind::LinkArg - | NativeLibKind::RawDylib => None, - } - }) - .collect(); - if !lib_args.is_empty() { - sess.note_without_error( - "Link against the following native artifacts when linking \ - against this static library. The order and any duplication \ - can be significant on some platforms.", - ); - // Prefix for greppability - sess.note_without_error(&format!("native-static-libs: {}", &lib_args.join(" "))); - } -} - -fn get_object_file_path(sess: &Session, name: &str, self_contained: bool) -> PathBuf { - let fs = sess.target_filesearch(PathKind::Native); - let file_path = fs.get_lib_path().join(name); - if file_path.exists() { - return file_path; - } - // Special directory with objects used only in self-contained linkage mode - if self_contained { - let file_path = fs.get_self_contained_lib_path().join(name); - if file_path.exists() { - return file_path; - } - } - for search_path in fs.search_paths() { - let file_path = search_path.dir.join(name); - if file_path.exists() { - return file_path; - } - } - PathBuf::from(name) -} - -fn exec_linker( - sess: &Session, - cmd: &Command, - out_filename: &Path, - tmpdir: &Path, -) -> io::Result { - // When attempting to spawn the linker we run a risk of blowing out the - // size limits for spawning a new process with respect to the arguments - // we pass on the command line. - // - // Here we attempt to handle errors from the OS saying "your list of - // arguments is too big" by reinvoking the linker again with an `@`-file - // that contains all the arguments. The theory is that this is then - // accepted on all linkers and the linker will read all its options out of - // there instead of looking at the command line. - if !cmd.very_likely_to_exceed_some_spawn_limit() { - match cmd.command().stdout(Stdio::piped()).stderr(Stdio::piped()).spawn() { - Ok(child) => { - let output = child.wait_with_output(); - flush_linked_file(&output, out_filename)?; - return output; - } - Err(ref e) if command_line_too_big(e) => { - info!("command line to linker was too big: {}", e); - } - Err(e) => return Err(e), - } - } - - info!("falling back to passing arguments to linker via an @-file"); - let mut cmd2 = cmd.clone(); - let mut args = String::new(); - for arg in cmd2.take_args() { - args.push_str( - &Escape { arg: arg.to_str().unwrap(), is_like_msvc: sess.target.is_like_msvc } - .to_string(), - ); - args.push('\n'); - } - let file = tmpdir.join("linker-arguments"); - let bytes = if sess.target.is_like_msvc { - let mut out = Vec::with_capacity((1 + args.len()) * 2); - // start the stream with a UTF-16 BOM - for c in std::iter::once(0xFEFF).chain(args.encode_utf16()) { - // encode in little endian - out.push(c as u8); - out.push((c >> 8) as u8); - } - out - } else { - args.into_bytes() - }; - fs::write(&file, &bytes)?; - cmd2.arg(format!("@{}", file.display())); - info!("invoking linker {:?}", cmd2); - let output = cmd2.output(); - flush_linked_file(&output, out_filename)?; - return output; - - #[cfg(not(windows))] - fn flush_linked_file(_: &io::Result, _: &Path) -> io::Result<()> { - Ok(()) - } - - #[cfg(windows)] - fn flush_linked_file( - command_output: &io::Result, - out_filename: &Path, - ) -> io::Result<()> { - // On Windows, under high I/O load, output buffers are sometimes not flushed, - // even long after process exit, causing nasty, non-reproducible output bugs. - // - // File::sync_all() calls FlushFileBuffers() down the line, which solves the problem. - // - // А full writeup of the original Chrome bug can be found at - // randomascii.wordpress.com/2018/02/25/compiler-bug-linker-bug-windows-kernel-bug/amp - - if let &Ok(ref out) = command_output { - if out.status.success() { - if let Ok(of) = fs::OpenOptions::new().write(true).open(out_filename) { - of.sync_all()?; - } - } - } - - Ok(()) - } - - #[cfg(unix)] - fn command_line_too_big(err: &io::Error) -> bool { - err.raw_os_error() == Some(::libc::E2BIG) - } - - #[cfg(windows)] - fn command_line_too_big(err: &io::Error) -> bool { - const ERROR_FILENAME_EXCED_RANGE: i32 = 206; - err.raw_os_error() == Some(ERROR_FILENAME_EXCED_RANGE) - } - - #[cfg(not(any(unix, windows)))] - fn command_line_too_big(_: &io::Error) -> bool { - false - } - - struct Escape<'a> { - arg: &'a str, - is_like_msvc: bool, - } - - impl<'a> fmt::Display for Escape<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.is_like_msvc { - // This is "documented" at - // https://docs.microsoft.com/en-us/cpp/build/reference/at-specify-a-linker-response-file - // - // Unfortunately there's not a great specification of the - // syntax I could find online (at least) but some local - // testing showed that this seemed sufficient-ish to catch - // at least a few edge cases. - write!(f, "\"")?; - for c in self.arg.chars() { - match c { - '"' => write!(f, "\\{}", c)?, - c => write!(f, "{}", c)?, - } - } - write!(f, "\"")?; - } else { - // This is documented at https://linux.die.net/man/1/ld, namely: - // - // > Options in file are separated by whitespace. A whitespace - // > character may be included in an option by surrounding the - // > entire option in either single or double quotes. Any - // > character (including a backslash) may be included by - // > prefixing the character to be included with a backslash. - // - // We put an argument on each line, so all we need to do is - // ensure the line is interpreted as one whole argument. - for c in self.arg.chars() { - match c { - '\\' | ' ' => write!(f, "\\{}", c)?, - c => write!(f, "{}", c)?, - } - } - } - Ok(()) - } - } -} - -fn link_output_kind(sess: &Session, crate_type: CrateType) -> LinkOutputKind { - let kind = match (crate_type, sess.crt_static(Some(crate_type)), sess.relocation_model()) { - (CrateType::Executable, _, _) if sess.is_wasi_reactor() => LinkOutputKind::WasiReactorExe, - (CrateType::Executable, false, RelocModel::Pic | RelocModel::Pie) => { - LinkOutputKind::DynamicPicExe - } - (CrateType::Executable, false, _) => LinkOutputKind::DynamicNoPicExe, - (CrateType::Executable, true, RelocModel::Pic | RelocModel::Pie) => { - LinkOutputKind::StaticPicExe - } - (CrateType::Executable, true, _) => LinkOutputKind::StaticNoPicExe, - (_, true, _) => LinkOutputKind::StaticDylib, - (_, false, _) => LinkOutputKind::DynamicDylib, - }; - - // Adjust the output kind to target capabilities. - let opts = &sess.target; - let pic_exe_supported = opts.position_independent_executables; - let static_pic_exe_supported = opts.static_position_independent_executables; - let static_dylib_supported = opts.crt_static_allows_dylibs; - match kind { - LinkOutputKind::DynamicPicExe if !pic_exe_supported => LinkOutputKind::DynamicNoPicExe, - LinkOutputKind::StaticPicExe if !static_pic_exe_supported => LinkOutputKind::StaticNoPicExe, - LinkOutputKind::StaticDylib if !static_dylib_supported => LinkOutputKind::DynamicDylib, - _ => kind, - } -} - -// Returns true if linker is located within sysroot -fn detect_self_contained_mingw(sess: &Session) -> bool { - let (linker, _) = linker_and_flavor(&sess); - // Assume `-C linker=rust-lld` as self-contained mode - if linker == Path::new("rust-lld") { - return true; - } - let linker_with_extension = if cfg!(windows) && linker.extension().is_none() { - linker.with_extension("exe") - } else { - linker - }; - for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { - let full_path = dir.join(&linker_with_extension); - // If linker comes from sysroot assume self-contained mode - if full_path.is_file() && !full_path.starts_with(&sess.sysroot) { - return false; - } - } - true -} - -/// Whether we link to our own CRT objects instead of relying on gcc to pull them. -/// We only provide such support for a very limited number of targets. -fn crt_objects_fallback(sess: &Session, crate_type: CrateType) -> bool { - if let Some(self_contained) = sess.opts.cg.link_self_contained { - return self_contained; - } - - match sess.target.crt_objects_fallback { - // FIXME: Find a better heuristic for "native musl toolchain is available", - // based on host and linker path, for example. - // (https://github.com/rust-lang/rust/pull/71769#issuecomment-626330237). - Some(CrtObjectsFallback::Musl) => sess.crt_static(Some(crate_type)), - Some(CrtObjectsFallback::Mingw) => { - sess.host == sess.target - && sess.target.vendor != "uwp" - && detect_self_contained_mingw(&sess) - } - // FIXME: Figure out cases in which WASM needs to link with a native toolchain. - Some(CrtObjectsFallback::Wasm) => true, - None => false, - } -} - -/// Add pre-link object files defined by the target spec. -fn add_pre_link_objects( - cmd: &mut dyn Linker, - sess: &Session, - link_output_kind: LinkOutputKind, - self_contained: bool, -) { - let opts = &sess.target; - let objects = - if self_contained { &opts.pre_link_objects_fallback } else { &opts.pre_link_objects }; - for obj in objects.get(&link_output_kind).iter().copied().flatten() { - cmd.add_object(&get_object_file_path(sess, obj, self_contained)); - } -} - -/// Add post-link object files defined by the target spec. -fn add_post_link_objects( - cmd: &mut dyn Linker, - sess: &Session, - link_output_kind: LinkOutputKind, - self_contained: bool, -) { - let opts = &sess.target; - let objects = - if self_contained { &opts.post_link_objects_fallback } else { &opts.post_link_objects }; - for obj in objects.get(&link_output_kind).iter().copied().flatten() { - cmd.add_object(&get_object_file_path(sess, obj, self_contained)); - } -} - -/// Add arbitrary "pre-link" args defined by the target spec or from command line. -/// FIXME: Determine where exactly these args need to be inserted. -fn add_pre_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { - if let Some(args) = sess.target.pre_link_args.get(&flavor) { - cmd.args(args.iter().map(Deref::deref)); - } - cmd.args(&sess.opts.unstable_opts.pre_link_args); -} - -/// Add a link script embedded in the target, if applicable. -fn add_link_script(cmd: &mut dyn Linker, sess: &Session, tmpdir: &Path, crate_type: CrateType) { - match (crate_type, &sess.target.link_script) { - (CrateType::Cdylib | CrateType::Executable, Some(script)) => { - if !sess.target.linker_is_gnu { - sess.fatal("can only use link script when linking with GNU-like linker"); - } - - let file_name = ["rustc", &sess.target.llvm_target, "linkfile.ld"].join("-"); - - let path = tmpdir.join(file_name); - if let Err(e) = fs::write(&path, script.as_ref()) { - sess.fatal(&format!("failed to write link script to {}: {}", path.display(), e)); - } - - cmd.arg("--script"); - cmd.arg(path); - } - _ => {} - } -} - -/// Add arbitrary "user defined" args defined from command line. -/// FIXME: Determine where exactly these args need to be inserted. -fn add_user_defined_link_args(cmd: &mut dyn Linker, sess: &Session) { - cmd.args(&sess.opts.cg.link_args); -} - -/// Add arbitrary "late link" args defined by the target spec. -/// FIXME: Determine where exactly these args need to be inserted. -fn add_late_link_args( - cmd: &mut dyn Linker, - sess: &Session, - flavor: LinkerFlavor, - crate_type: CrateType, - codegen_results: &CodegenResults, -) { - let any_dynamic_crate = crate_type == CrateType::Dylib - || codegen_results.crate_info.dependency_formats.iter().any(|(ty, list)| { - *ty == crate_type && list.iter().any(|&linkage| linkage == Linkage::Dynamic) - }); - if any_dynamic_crate { - if let Some(args) = sess.target.late_link_args_dynamic.get(&flavor) { - cmd.args(args.iter().map(Deref::deref)); - } - } else { - if let Some(args) = sess.target.late_link_args_static.get(&flavor) { - cmd.args(args.iter().map(Deref::deref)); - } - } - if let Some(args) = sess.target.late_link_args.get(&flavor) { - cmd.args(args.iter().map(Deref::deref)); - } -} - -/// Add arbitrary "post-link" args defined by the target spec. -/// FIXME: Determine where exactly these args need to be inserted. -fn add_post_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { - if let Some(args) = sess.target.post_link_args.get(&flavor) { - cmd.args(args.iter().map(Deref::deref)); - } -} - -/// Add a synthetic object file that contains reference to all symbols that we want to expose to -/// the linker. -/// -/// Background: we implement rlibs as static library (archives). Linkers treat archives -/// differently from object files: all object files participate in linking, while archives will -/// only participate in linking if they can satisfy at least one undefined reference (version -/// scripts doesn't count). This causes `#[no_mangle]` or `#[used]` items to be ignored by the -/// linker, and since they never participate in the linking, using `KEEP` in the linker scripts -/// can't keep them either. This causes #47384. -/// -/// To keep them around, we could use `--whole-archive` and equivalents to force rlib to -/// participate in linking like object files, but this proves to be expensive (#93791). Therefore -/// we instead just introduce an undefined reference to them. This could be done by `-u` command -/// line option to the linker or `EXTERN(...)` in linker scripts, however they does not only -/// introduce an undefined reference, but also make them the GC roots, preventing `--gc-sections` -/// from removing them, and this is especially problematic for embedded programming where every -/// byte counts. -/// -/// This method creates a synthetic object file, which contains undefined references to all symbols -/// that are necessary for the linking. They are only present in symbol table but not actually -/// used in any sections, so the linker will therefore pick relevant rlibs for linking, but -/// unused `#[no_mangle]` or `#[used]` can still be discard by GC sections. -fn add_linked_symbol_object( - cmd: &mut dyn Linker, - sess: &Session, - tmpdir: &Path, - symbols: &[(String, SymbolExportKind)], -) { - if symbols.is_empty() { - return; - } - - let Some(mut file) = super::metadata::create_object_file(sess) else { - return; - }; - - // NOTE(nbdd0121): MSVC will hang if the input object file contains no sections, - // so add an empty section. - if file.format() == object::BinaryFormat::Coff { - file.add_section(Vec::new(), ".text".into(), object::SectionKind::Text); - - // We handle the name decoration of COFF targets in `symbol_export.rs`, so disable the - // default mangler in `object` crate. - file.set_mangling(object::write::Mangling::None); - - // Add feature flags to the object file. On MSVC this is optional but LLD will complain if - // not present. - let mut feature = 0; - - if file.architecture() == object::Architecture::I386 { - // Indicate that all SEH handlers are registered in .sxdata section. - // We don't have generate any code, so we don't need .sxdata section but LLD still - // expects us to set this bit (see #96498). - // Reference: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format - feature |= 1; - } - - file.add_symbol(object::write::Symbol { - name: "@feat.00".into(), - value: feature, - size: 0, - kind: object::SymbolKind::Data, - scope: object::SymbolScope::Compilation, - weak: false, - section: object::write::SymbolSection::Absolute, - flags: object::SymbolFlags::None, - }); - } - - for (sym, kind) in symbols.iter() { - file.add_symbol(object::write::Symbol { - name: sym.clone().into(), - value: 0, - size: 0, - kind: match kind { - SymbolExportKind::Text => object::SymbolKind::Text, - SymbolExportKind::Data => object::SymbolKind::Data, - SymbolExportKind::Tls => object::SymbolKind::Tls, - }, - scope: object::SymbolScope::Unknown, - weak: false, - section: object::write::SymbolSection::Undefined, - flags: object::SymbolFlags::None, - }); - } - - let path = tmpdir.join("symbols.o"); - let result = std::fs::write(&path, file.write().unwrap()); - if let Err(e) = result { - sess.fatal(&format!("failed to write {}: {}", path.display(), e)); - } - cmd.add_object(&path); -} - -/// Add object files containing code from the current crate. -fn add_local_crate_regular_objects(cmd: &mut dyn Linker, codegen_results: &CodegenResults) { - for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) { - cmd.add_object(obj); - } -} - -/// Add object files for allocator code linked once for the whole crate tree. -fn add_local_crate_allocator_objects(cmd: &mut dyn Linker, codegen_results: &CodegenResults) { - if let Some(obj) = codegen_results.allocator_module.as_ref().and_then(|m| m.object.as_ref()) { - cmd.add_object(obj); - } -} - -/// Add object files containing metadata for the current crate. -fn add_local_crate_metadata_objects( - cmd: &mut dyn Linker, - crate_type: CrateType, - codegen_results: &CodegenResults, -) { - // When linking a dynamic library, we put the metadata into a section of the - // executable. This metadata is in a separate object file from the main - // object file, so we link that in here. - if crate_type == CrateType::Dylib || crate_type == CrateType::ProcMacro { - if let Some(obj) = codegen_results.metadata_module.as_ref().and_then(|m| m.object.as_ref()) - { - cmd.add_object(obj); - } - } -} - -/// Add sysroot and other globally set directories to the directory search list. -fn add_library_search_dirs(cmd: &mut dyn Linker, sess: &Session, self_contained: bool) { - // The default library location, we need this to find the runtime. - // The location of crates will be determined as needed. - let lib_path = sess.target_filesearch(PathKind::All).get_lib_path(); - cmd.include_path(&fix_windows_verbatim_for_gcc(&lib_path)); - - // Special directory with libraries used only in self-contained linkage mode - if self_contained { - let lib_path = sess.target_filesearch(PathKind::All).get_self_contained_lib_path(); - cmd.include_path(&fix_windows_verbatim_for_gcc(&lib_path)); - } -} - -/// Add options making relocation sections in the produced ELF files read-only -/// and suppressing lazy binding. -fn add_relro_args(cmd: &mut dyn Linker, sess: &Session) { - match sess.opts.unstable_opts.relro_level.unwrap_or(sess.target.relro_level) { - RelroLevel::Full => cmd.full_relro(), - RelroLevel::Partial => cmd.partial_relro(), - RelroLevel::Off => cmd.no_relro(), - RelroLevel::None => {} - } -} - -/// Add library search paths used at runtime by dynamic linkers. -fn add_rpath_args( - cmd: &mut dyn Linker, - sess: &Session, - codegen_results: &CodegenResults, - out_filename: &Path, -) { - // FIXME (#2397): At some point we want to rpath our guesses as to - // where extern libraries might live, based on the - // add_lib_search_paths - if sess.opts.cg.rpath { - let libs = codegen_results - .crate_info - .used_crates - .iter() - .filter_map(|cnum| { - codegen_results.crate_info.used_crate_source[cnum] - .dylib - .as_ref() - .map(|(path, _)| &**path) - }) - .collect::>(); - let mut rpath_config = RPathConfig { - libs: &*libs, - out_filename: out_filename.to_path_buf(), - has_rpath: sess.target.has_rpath, - is_like_osx: sess.target.is_like_osx, - linker_is_gnu: sess.target.linker_is_gnu, - }; - cmd.args(&rpath::get_rpath_flags(&mut rpath_config)); - } -} - -/// Produce the linker command line containing linker path and arguments. -/// -/// When comments in the function say "order-(in)dependent" they mean order-dependence between -/// options and libraries/object files. For example `--whole-archive` (order-dependent) applies -/// to specific libraries passed after it, and `-o` (output file, order-independent) applies -/// to the linking process as a whole. -/// Order-independent options may still override each other in order-dependent fashion, -/// e.g `--foo=yes --foo=no` may be equivalent to `--foo=no`. -fn linker_with_args<'a>( - path: &Path, - flavor: LinkerFlavor, - sess: &'a Session, - archive_builder_builder: &dyn ArchiveBuilderBuilder, - crate_type: CrateType, - tmpdir: &Path, - out_filename: &Path, - codegen_results: &CodegenResults, -) -> Result { - let crt_objects_fallback = crt_objects_fallback(sess, crate_type); - let cmd = &mut *super::linker::get_linker( - sess, - path, - flavor, - crt_objects_fallback, - &codegen_results.crate_info.target_cpu, - ); - let link_output_kind = link_output_kind(sess, crate_type); - - // ------------ Early order-dependent options ------------ - - // If we're building something like a dynamic library then some platforms - // need to make sure that all symbols are exported correctly from the - // dynamic library. - // Must be passed before any libraries to prevent the symbols to export from being thrown away, - // at least on some platforms (e.g. windows-gnu). - cmd.export_symbols( - tmpdir, - crate_type, - &codegen_results.crate_info.exported_symbols[&crate_type], - ); - - // Can be used for adding custom CRT objects or overriding order-dependent options above. - // FIXME: In practice built-in target specs use this for arbitrary order-independent options, - // introduce a target spec option for order-independent linker options and migrate built-in - // specs to it. - add_pre_link_args(cmd, sess, flavor); - - // ------------ Object code and libraries, order-dependent ------------ - - // Pre-link CRT objects. - add_pre_link_objects(cmd, sess, link_output_kind, crt_objects_fallback); - - add_linked_symbol_object( - cmd, - sess, - tmpdir, - &codegen_results.crate_info.linked_symbols[&crate_type], - ); - - // Sanitizer libraries. - add_sanitizer_libraries(sess, crate_type, cmd); - - // Object code from the current crate. - // Take careful note of the ordering of the arguments we pass to the linker - // here. Linkers will assume that things on the left depend on things to the - // right. Things on the right cannot depend on things on the left. This is - // all formally implemented in terms of resolving symbols (libs on the right - // resolve unknown symbols of libs on the left, but not vice versa). - // - // For this reason, we have organized the arguments we pass to the linker as - // such: - // - // 1. The local object that LLVM just generated - // 2. Local native libraries - // 3. Upstream rust libraries - // 4. Upstream native libraries - // - // The rationale behind this ordering is that those items lower down in the - // list can't depend on items higher up in the list. For example nothing can - // depend on what we just generated (e.g., that'd be a circular dependency). - // Upstream rust libraries are not supposed to depend on our local native - // libraries as that would violate the structure of the DAG, in that - // scenario they are required to link to them as well in a shared fashion. - // (The current implementation still doesn't prevent it though, see the FIXME below.) - // - // Note that upstream rust libraries may contain native dependencies as - // well, but they also can't depend on what we just started to add to the - // link line. And finally upstream native libraries can't depend on anything - // in this DAG so far because they can only depend on other native libraries - // and such dependencies are also required to be specified. - add_local_crate_regular_objects(cmd, codegen_results); - add_local_crate_metadata_objects(cmd, crate_type, codegen_results); - add_local_crate_allocator_objects(cmd, codegen_results); - - // Avoid linking to dynamic libraries unless they satisfy some undefined symbols - // at the point at which they are specified on the command line. - // Must be passed before any (dynamic) libraries to have effect on them. - // On Solaris-like systems, `-z ignore` acts as both `--as-needed` and `--gc-sections` - // so it will ignore unreferenced ELF sections from relocatable objects. - // For that reason, we put this flag after metadata objects as they would otherwise be removed. - // FIXME: Support more fine-grained dead code removal on Solaris/illumos - // and move this option back to the top. - cmd.add_as_needed(); - - // FIXME: Move this below to other native libraries - // (or alternatively link all native libraries after their respective crates). - // This change is somewhat breaking in practice due to local static libraries being linked - // as whole-archive (#85144), so removing whole-archive may be a pre-requisite. - if sess.opts.unstable_opts.link_native_libraries { - add_local_native_libraries(cmd, sess, codegen_results); - } - - // Upstream rust libraries and their non-bundled static libraries - add_upstream_rust_crates( - cmd, - sess, - archive_builder_builder, - codegen_results, - crate_type, - tmpdir, - ); - - // Upstream dynamic native libraries linked with `#[link]` attributes at and `-l` - // command line options. - // If -Zlink-native-libraries=false is set, then the assumption is that an - // external build system already has the native dependencies defined, and it - // will provide them to the linker itself. - if sess.opts.unstable_opts.link_native_libraries { - add_upstream_native_libraries(cmd, sess, codegen_results); - } - - // Link with the import library generated for any raw-dylib functions. - for (raw_dylib_name, raw_dylib_imports) in - collate_raw_dylibs(sess, &codegen_results.crate_info.used_libraries)? - { - cmd.add_object(&archive_builder_builder.create_dll_import_lib( - sess, - &raw_dylib_name, - &raw_dylib_imports, - tmpdir, - )); - } - - // Library linking above uses some global state for things like `-Bstatic`/`-Bdynamic` to make - // command line shorter, reset it to default here before adding more libraries. - cmd.reset_per_library_state(); - - // FIXME: Built-in target specs occasionally use this for linking system libraries, - // eliminate all such uses by migrating them to `#[link]` attributes in `lib(std,c,unwind)` - // and remove the option. - add_late_link_args(cmd, sess, flavor, crate_type, codegen_results); - - // ------------ Arbitrary order-independent options ------------ - - // Add order-independent options determined by rustc from its compiler options, - // target properties and source code. - add_order_independent_options( - cmd, - sess, - link_output_kind, - crt_objects_fallback, - flavor, - crate_type, - codegen_results, - out_filename, - tmpdir, - ); - - // Can be used for arbitrary order-independent options. - // In practice may also be occasionally used for linking native libraries. - // Passed after compiler-generated options to support manual overriding when necessary. - add_user_defined_link_args(cmd, sess); - - // ------------ Object code and libraries, order-dependent ------------ - - // Post-link CRT objects. - add_post_link_objects(cmd, sess, link_output_kind, crt_objects_fallback); - - // ------------ Late order-dependent options ------------ - - // Doesn't really make sense. - // FIXME: In practice built-in target specs use this for arbitrary order-independent options, - // introduce a target spec option for order-independent linker options, migrate built-in specs - // to it and remove the option. - add_post_link_args(cmd, sess, flavor); - - Ok(cmd.take_cmd()) -} - -fn add_order_independent_options( - cmd: &mut dyn Linker, - sess: &Session, - link_output_kind: LinkOutputKind, - crt_objects_fallback: bool, - flavor: LinkerFlavor, - crate_type: CrateType, - codegen_results: &CodegenResults, - out_filename: &Path, - tmpdir: &Path, -) { - add_gcc_ld_path(cmd, sess, flavor); - - add_apple_sdk(cmd, sess, flavor); - - add_link_script(cmd, sess, tmpdir, crate_type); - - if sess.target.os == "fuchsia" && crate_type == CrateType::Executable { - let prefix = if sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::ADDRESS) { - "asan/" - } else { - "" - }; - cmd.arg(format!("--dynamic-linker={}ld.so.1", prefix)); - } - - if sess.target.eh_frame_header { - cmd.add_eh_frame_header(); - } - - // Make the binary compatible with data execution prevention schemes. - cmd.add_no_exec(); - - if crt_objects_fallback { - cmd.no_crt_objects(); - } - - if sess.target.os == "emscripten" { - cmd.arg("-s"); - cmd.arg(if sess.panic_strategy() == PanicStrategy::Abort { - "DISABLE_EXCEPTION_CATCHING=1" - } else { - "DISABLE_EXCEPTION_CATCHING=0" - }); - } - - if flavor == LinkerFlavor::PtxLinker { - // Provide the linker with fallback to internal `target-cpu`. - cmd.arg("--fallback-arch"); - cmd.arg(&codegen_results.crate_info.target_cpu); - } else if flavor == LinkerFlavor::BpfLinker { - cmd.arg("--cpu"); - cmd.arg(&codegen_results.crate_info.target_cpu); - cmd.arg("--cpu-features"); - cmd.arg(match &sess.opts.cg.target_feature { - feat if !feat.is_empty() => feat.as_ref(), - _ => sess.target.options.features.as_ref(), - }); - } - - cmd.linker_plugin_lto(); - - add_library_search_dirs(cmd, sess, crt_objects_fallback); - - cmd.output_filename(out_filename); - - if crate_type == CrateType::Executable && sess.target.is_like_windows { - if let Some(ref s) = codegen_results.crate_info.windows_subsystem { - cmd.subsystem(s); - } - } - - // Try to strip as much out of the generated object by removing unused - // sections if possible. See more comments in linker.rs - if !sess.link_dead_code() { - // If PGO is enabled sometimes gc_sections will remove the profile data section - // as it appears to be unused. This can then cause the PGO profile file to lose - // some functions. If we are generating a profile we shouldn't strip those metadata - // sections to ensure we have all the data for PGO. - let keep_metadata = - crate_type == CrateType::Dylib || sess.opts.cg.profile_generate.enabled(); - if crate_type != CrateType::Executable || !sess.opts.unstable_opts.export_executable_symbols - { - cmd.gc_sections(keep_metadata); - } else { - cmd.no_gc_sections(); - } - } - - cmd.set_output_kind(link_output_kind, out_filename); - - add_relro_args(cmd, sess); - - // Pass optimization flags down to the linker. - cmd.optimize(); - - // Gather the set of NatVis files, if any, and write them out to a temp directory. - let natvis_visualizers = collect_natvis_visualizers( - tmpdir, - sess, - &codegen_results.crate_info.local_crate_name, - &codegen_results.crate_info.natvis_debugger_visualizers, - ); - - // Pass debuginfo, NatVis debugger visualizers and strip flags down to the linker. - cmd.debuginfo(strip_value(sess), &natvis_visualizers); - - // We want to prevent the compiler from accidentally leaking in any system libraries, - // so by default we tell linkers not to link to any default libraries. - if !sess.opts.cg.default_linker_libraries && sess.target.no_default_libraries { - cmd.no_default_libraries(); - } - - if sess.opts.cg.profile_generate.enabled() || sess.instrument_coverage() { - cmd.pgo_gen(); - } - - if sess.opts.cg.control_flow_guard != CFGuard::Disabled { - cmd.control_flow_guard(); - } - - add_rpath_args(cmd, sess, codegen_results, out_filename); -} - -// Write the NatVis debugger visualizer files for each crate to the temp directory and gather the file paths. -fn collect_natvis_visualizers( - tmpdir: &Path, - sess: &Session, - crate_name: &Symbol, - natvis_debugger_visualizers: &BTreeSet, -) -> Vec { - let mut visualizer_paths = Vec::with_capacity(natvis_debugger_visualizers.len()); - - for (index, visualizer) in natvis_debugger_visualizers.iter().enumerate() { - let visualizer_out_file = tmpdir.join(format!("{}-{}.natvis", crate_name.as_str(), index)); - - match fs::write(&visualizer_out_file, &visualizer.src) { - Ok(()) => { - visualizer_paths.push(visualizer_out_file); - } - Err(error) => { - sess.warn( - format!( - "Unable to write debugger visualizer file `{}`: {} ", - visualizer_out_file.display(), - error - ) - .as_str(), - ); - } - }; - } - visualizer_paths -} - -/// # Native library linking -/// -/// User-supplied library search paths (-L on the command line). These are the same paths used to -/// find Rust crates, so some of them may have been added already by the previous crate linking -/// code. This only allows them to be found at compile time so it is still entirely up to outside -/// forces to make sure that library can be found at runtime. -/// -/// Also note that the native libraries linked here are only the ones located in the current crate. -/// Upstream crates with native library dependencies may have their native library pulled in above. -fn add_local_native_libraries( - cmd: &mut dyn Linker, - sess: &Session, - codegen_results: &CodegenResults, -) { - let filesearch = sess.target_filesearch(PathKind::All); - for search_path in filesearch.search_paths() { - match search_path.kind { - PathKind::Framework => { - cmd.framework_path(&search_path.dir); - } - _ => { - cmd.include_path(&fix_windows_verbatim_for_gcc(&search_path.dir)); - } - } - } - - let relevant_libs = - codegen_results.crate_info.used_libraries.iter().filter(|l| relevant_lib(sess, l)); - - let search_path = OnceCell::new(); - let mut last = (None, NativeLibKind::Unspecified, None); - for lib in relevant_libs { - let Some(name) = lib.name else { - continue; - }; - let name = name.as_str(); - - // Skip if this library is the same as the last. - last = if (lib.name, lib.kind, lib.verbatim) == last { - continue; - } else { - (lib.name, lib.kind, lib.verbatim) - }; - - let verbatim = lib.verbatim.unwrap_or(false); - match lib.kind { - NativeLibKind::Dylib { as_needed } => { - cmd.link_dylib(name, verbatim, as_needed.unwrap_or(true)) - } - NativeLibKind::Unspecified => cmd.link_dylib(name, verbatim, true), - NativeLibKind::Framework { as_needed } => { - cmd.link_framework(name, as_needed.unwrap_or(true)) - } - NativeLibKind::Static { whole_archive, bundle, .. } => { - if whole_archive == Some(true) - // Backward compatibility case: this can be a rlib (so `+whole-archive` cannot - // be added explicitly if necessary, see the error in `fn link_rlib`) compiled - // as an executable due to `--test`. Use whole-archive implicitly, like before - // the introduction of native lib modifiers. - || (whole_archive == None && bundle != Some(false) && sess.opts.test) - { - cmd.link_whole_staticlib( - name, - verbatim, - &search_path.get_or_init(|| archive_search_paths(sess)), - ); - } else { - cmd.link_staticlib(name, verbatim) - } - } - NativeLibKind::RawDylib => { - // Ignore RawDylib here, they are handled separately in linker_with_args(). - } - NativeLibKind::LinkArg => { - cmd.arg(name); - } - } - } -} - -/// # Linking Rust crates and their non-bundled static libraries -/// -/// Rust crates are not considered at all when creating an rlib output. All dependencies will be -/// linked when producing the final output (instead of the intermediate rlib version). -fn add_upstream_rust_crates<'a>( - cmd: &mut dyn Linker, - sess: &'a Session, - archive_builder_builder: &dyn ArchiveBuilderBuilder, - codegen_results: &CodegenResults, - crate_type: CrateType, - tmpdir: &Path, -) { - // All of the heavy lifting has previously been accomplished by the - // dependency_format module of the compiler. This is just crawling the - // output of that module, adding crates as necessary. - // - // Linking to a rlib involves just passing it to the linker (the linker - // will slurp up the object files inside), and linking to a dynamic library - // involves just passing the right -l flag. - - let (_, data) = codegen_results - .crate_info - .dependency_formats - .iter() - .find(|(ty, _)| *ty == crate_type) - .expect("failed to find crate type in dependency format list"); - - // Invoke get_used_crates to ensure that we get a topological sorting of - // crates. - let deps = &codegen_results.crate_info.used_crates; - - // There's a few internal crates in the standard library (aka libcore and - // libstd) which actually have a circular dependence upon one another. This - // currently arises through "weak lang items" where libcore requires things - // like `rust_begin_unwind` but libstd ends up defining it. To get this - // circular dependence to work correctly in all situations we'll need to be - // sure to correctly apply the `--start-group` and `--end-group` options to - // GNU linkers, otherwise if we don't use any other symbol from the standard - // library it'll get discarded and the whole application won't link. - // - // In this loop we're calculating the `group_end`, after which crate to - // pass `--end-group` and `group_start`, before which crate to pass - // `--start-group`. We currently do this by passing `--end-group` after - // the first crate (when iterating backwards) that requires a lang item - // defined somewhere else. Once that's set then when we've defined all the - // necessary lang items we'll pass `--start-group`. - // - // Note that this isn't amazing logic for now but it should do the trick - // for the current implementation of the standard library. - let mut group_end = None; - let mut group_start = None; - // Crates available for linking thus far. - let mut available = FxHashSet::default(); - // Crates required to satisfy dependencies discovered so far. - let mut required = FxHashSet::default(); - - let info = &codegen_results.crate_info; - for &cnum in deps.iter().rev() { - if let Some(missing) = info.missing_lang_items.get(&cnum) { - let missing_crates = missing.iter().map(|i| info.lang_item_to_crate.get(i).copied()); - required.extend(missing_crates); - } - - required.insert(Some(cnum)); - available.insert(Some(cnum)); - - if required.len() > available.len() && group_end.is_none() { - group_end = Some(cnum); - } - if required.len() == available.len() && group_end.is_some() { - group_start = Some(cnum); - break; - } - } - - // If we didn't end up filling in all lang items from upstream crates then - // we'll be filling it in with our crate. This probably means we're the - // standard library itself, so skip this for now. - if group_end.is_some() && group_start.is_none() { - group_end = None; - } - - let mut compiler_builtins = None; - let search_path = OnceCell::new(); - - for &cnum in deps.iter() { - if group_start == Some(cnum) { - cmd.group_start(); - } - - // We may not pass all crates through to the linker. Some crates may - // appear statically in an existing dylib, meaning we'll pick up all the - // symbols from the dylib. - let src = &codegen_results.crate_info.used_crate_source[&cnum]; - match data[cnum.as_usize() - 1] { - _ if codegen_results.crate_info.profiler_runtime == Some(cnum) => { - add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum); - } - // compiler-builtins are always placed last to ensure that they're - // linked correctly. - _ if codegen_results.crate_info.compiler_builtins == Some(cnum) => { - assert!(compiler_builtins.is_none()); - compiler_builtins = Some(cnum); - } - Linkage::NotLinked | Linkage::IncludedFromDylib => {} - Linkage::Static => { - add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum); - - // Link static native libs with "-bundle" modifier only if the crate they originate from - // is being linked statically to the current crate. If it's linked dynamically - // or is an rlib already included via some other dylib crate, the symbols from - // native libs will have already been included in that dylib. - // - // If -Zlink-native-libraries=false is set, then the assumption is that an - // external build system already has the native dependencies defined, and it - // will provide them to the linker itself. - if sess.opts.unstable_opts.link_native_libraries { - let mut last = (None, NativeLibKind::Unspecified, None); - for lib in &codegen_results.crate_info.native_libraries[&cnum] { - let Some(name) = lib.name else { - continue; - }; - let name = name.as_str(); - if !relevant_lib(sess, lib) { - continue; - } - - // Skip if this library is the same as the last. - last = if (lib.name, lib.kind, lib.verbatim) == last { - continue; - } else { - (lib.name, lib.kind, lib.verbatim) - }; - - match lib.kind { - NativeLibKind::Static { - bundle: Some(false), - whole_archive: Some(true), - } => { - cmd.link_whole_staticlib( - name, - lib.verbatim.unwrap_or(false), - search_path.get_or_init(|| archive_search_paths(sess)), - ); - } - NativeLibKind::Static { - bundle: Some(false), - whole_archive: Some(false) | None, - } => { - cmd.link_staticlib(name, lib.verbatim.unwrap_or(false)); - } - NativeLibKind::LinkArg => { - cmd.arg(name); - } - NativeLibKind::Dylib { .. } - | NativeLibKind::Framework { .. } - | NativeLibKind::Unspecified - | NativeLibKind::RawDylib => {} - NativeLibKind::Static { - bundle: Some(true) | None, - whole_archive: _, - } => {} - } - } - } - } - Linkage::Dynamic => add_dynamic_crate(cmd, sess, &src.dylib.as_ref().unwrap().0), - } - - if group_end == Some(cnum) { - cmd.group_end(); - } - } - - // compiler-builtins are always placed last to ensure that they're - // linked correctly. - // We must always link the `compiler_builtins` crate statically. Even if it - // was already "included" in a dylib (e.g., `libstd` when `-C prefer-dynamic` - // is used) - if let Some(cnum) = compiler_builtins { - add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum); - } - - // Converts a library file-stem into a cc -l argument - fn unlib<'a>(target: &Target, stem: &'a str) -> &'a str { - if stem.starts_with("lib") && !target.is_like_windows { &stem[3..] } else { stem } - } - - // Adds the static "rlib" versions of all crates to the command line. - // There's a bit of magic which happens here specifically related to LTO, - // namely that we remove upstream object files. - // - // When performing LTO, almost(*) all of the bytecode from the upstream - // libraries has already been included in our object file output. As a - // result we need to remove the object files in the upstream libraries so - // the linker doesn't try to include them twice (or whine about duplicate - // symbols). We must continue to include the rest of the rlib, however, as - // it may contain static native libraries which must be linked in. - // - // (*) Crates marked with `#![no_builtins]` don't participate in LTO and - // their bytecode wasn't included. The object files in those libraries must - // still be passed to the linker. - // - // Note, however, that if we're not doing LTO we can just pass the rlib - // blindly to the linker (fast) because it's fine if it's not actually - // included as we're at the end of the dependency chain. - fn add_static_crate<'a>( - cmd: &mut dyn Linker, - sess: &'a Session, - archive_builder_builder: &dyn ArchiveBuilderBuilder, - codegen_results: &CodegenResults, - tmpdir: &Path, - cnum: CrateNum, - ) { - let src = &codegen_results.crate_info.used_crate_source[&cnum]; - let cratepath = &src.rlib.as_ref().unwrap().0; - - let mut link_upstream = |path: &Path| { - cmd.link_rlib(&fix_windows_verbatim_for_gcc(path)); - }; - - // See the comment above in `link_staticlib` and `link_rlib` for why if - // there's a static library that's not relevant we skip all object - // files. - let native_libs = &codegen_results.crate_info.native_libraries[&cnum]; - let skip_native = native_libs.iter().any(|lib| { - matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. }) - && !relevant_lib(sess, lib) - }); - - if (!are_upstream_rust_objects_already_included(sess) - || ignored_for_lto(sess, &codegen_results.crate_info, cnum)) - && !skip_native - { - link_upstream(cratepath); - return; - } - - let dst = tmpdir.join(cratepath.file_name().unwrap()); - let name = cratepath.file_name().unwrap().to_str().unwrap(); - let name = &name[3..name.len() - 5]; // chop off lib/.rlib - - sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| { - let canonical_name = name.replace('-', "_"); - let upstream_rust_objects_already_included = - are_upstream_rust_objects_already_included(sess); - let is_builtins = sess.target.no_builtins - || !codegen_results.crate_info.is_no_builtins.contains(&cnum); - - let mut archive = archive_builder_builder.new_archive_builder(sess); - if let Err(e) = archive.add_archive( - cratepath, - Box::new(move |f| { - if f == METADATA_FILENAME { - return true; - } - - let canonical = f.replace('-', "_"); - - let is_rust_object = - canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f); - - // If we've been requested to skip all native object files - // (those not generated by the rust compiler) then we can skip - // this file. See above for why we may want to do this. - let skip_because_cfg_say_so = skip_native && !is_rust_object; - - // If we're performing LTO and this is a rust-generated object - // file, then we don't need the object file as it's part of the - // LTO module. Note that `#![no_builtins]` is excluded from LTO, - // though, so we let that object file slide. - let skip_because_lto = - upstream_rust_objects_already_included && is_rust_object && is_builtins; - - if skip_because_cfg_say_so || skip_because_lto { - return true; - } - - false - }), - ) { - sess.fatal(&format!("failed to build archive from rlib: {}", e)); - } - if archive.build(&dst) { - link_upstream(&dst); - } - }); - } - - // Same thing as above, but for dynamic crates instead of static crates. - fn add_dynamic_crate(cmd: &mut dyn Linker, sess: &Session, cratepath: &Path) { - // Just need to tell the linker about where the library lives and - // what its name is - let parent = cratepath.parent(); - if let Some(dir) = parent { - cmd.include_path(&fix_windows_verbatim_for_gcc(dir)); - } - let filestem = cratepath.file_stem().unwrap().to_str().unwrap(); - cmd.link_rust_dylib( - &unlib(&sess.target, filestem), - parent.unwrap_or_else(|| Path::new("")), - ); - } -} - -/// Link in all of our upstream crates' native dependencies. Remember that all of these upstream -/// native dependencies are all non-static dependencies. We've got two cases then: -/// -/// 1. The upstream crate is an rlib. In this case we *must* link in the native dependency because -/// the rlib is just an archive. -/// -/// 2. The upstream crate is a dylib. In order to use the dylib, we have to have the dependency -/// present on the system somewhere. Thus, we don't gain a whole lot from not linking in the -/// dynamic dependency to this crate as well. -/// -/// The use case for this is a little subtle. In theory the native dependencies of a crate are -/// purely an implementation detail of the crate itself, but the problem arises with generic and -/// inlined functions. If a generic function calls a native function, then the generic function -/// must be instantiated in the target crate, meaning that the native symbol must also be resolved -/// in the target crate. -fn add_upstream_native_libraries( - cmd: &mut dyn Linker, - sess: &Session, - codegen_results: &CodegenResults, -) { - let mut last = (None, NativeLibKind::Unspecified, None); - for &cnum in &codegen_results.crate_info.used_crates { - for lib in codegen_results.crate_info.native_libraries[&cnum].iter() { - let Some(name) = lib.name else { - continue; - }; - let name = name.as_str(); - if !relevant_lib(sess, &lib) { - continue; - } - - // Skip if this library is the same as the last. - last = if (lib.name, lib.kind, lib.verbatim) == last { - continue; - } else { - (lib.name, lib.kind, lib.verbatim) - }; - - let verbatim = lib.verbatim.unwrap_or(false); - match lib.kind { - NativeLibKind::Dylib { as_needed } => { - cmd.link_dylib(name, verbatim, as_needed.unwrap_or(true)) - } - NativeLibKind::Unspecified => cmd.link_dylib(name, verbatim, true), - NativeLibKind::Framework { as_needed } => { - cmd.link_framework(name, as_needed.unwrap_or(true)) - } - // ignore static native libraries here as we've - // already included them in add_local_native_libraries and - // add_upstream_rust_crates - NativeLibKind::Static { .. } => {} - NativeLibKind::RawDylib | NativeLibKind::LinkArg => {} - } - } - } -} - -fn relevant_lib(sess: &Session, lib: &NativeLib) -> bool { - match lib.cfg { - Some(ref cfg) => rustc_attr::cfg_matches(cfg, &sess.parse_sess, CRATE_NODE_ID, None), - None => true, - } -} - -fn are_upstream_rust_objects_already_included(sess: &Session) -> bool { - match sess.lto() { - config::Lto::Fat => true, - config::Lto::Thin => { - // If we defer LTO to the linker, we haven't run LTO ourselves, so - // any upstream object files have not been copied yet. - !sess.opts.cg.linker_plugin_lto.enabled() - } - config::Lto::No | config::Lto::ThinLocal => false, - } -} - -fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { - let arch = &sess.target.arch; - let os = &sess.target.os; - let llvm_target = &sess.target.llvm_target; - if sess.target.vendor != "apple" - || !matches!(os.as_ref(), "ios" | "tvos" | "watchos") - || (flavor != LinkerFlavor::Gcc && flavor != LinkerFlavor::Lld(LldFlavor::Ld64)) - { - return; - } - let sdk_name = match (arch.as_ref(), os.as_ref()) { - ("aarch64", "tvos") => "appletvos", - ("x86_64", "tvos") => "appletvsimulator", - ("arm", "ios") => "iphoneos", - ("aarch64", "ios") if llvm_target.contains("macabi") => "macosx", - ("aarch64", "ios") if llvm_target.ends_with("-simulator") => "iphonesimulator", - ("aarch64", "ios") => "iphoneos", - ("x86", "ios") => "iphonesimulator", - ("x86_64", "ios") if llvm_target.contains("macabi") => "macosx", - ("x86_64", "ios") => "iphonesimulator", - ("x86_64", "watchos") => "watchsimulator", - ("arm64_32", "watchos") => "watchos", - ("aarch64", "watchos") if llvm_target.ends_with("-simulator") => "watchsimulator", - ("aarch64", "watchos") => "watchos", - ("arm", "watchos") => "watchos", - _ => { - sess.err(&format!("unsupported arch `{}` for os `{}`", arch, os)); - return; - } - }; - let sdk_root = match get_apple_sdk_root(sdk_name) { - Ok(s) => s, - Err(e) => { - sess.err(&e); - return; - } - }; - - match flavor { - LinkerFlavor::Gcc => { - cmd.args(&["-isysroot", &sdk_root, "-Wl,-syslibroot", &sdk_root]); - } - LinkerFlavor::Lld(LldFlavor::Ld64) => { - cmd.args(&["-syslibroot", &sdk_root]); - } - _ => unreachable!(), - } -} - -fn get_apple_sdk_root(sdk_name: &str) -> Result { - // Following what clang does - // (https://github.com/llvm/llvm-project/blob/ - // 296a80102a9b72c3eda80558fb78a3ed8849b341/clang/lib/Driver/ToolChains/Darwin.cpp#L1661-L1678) - // to allow the SDK path to be set. (For clang, xcrun sets - // SDKROOT; for rustc, the user or build system can set it, or we - // can fall back to checking for xcrun on PATH.) - if let Ok(sdkroot) = env::var("SDKROOT") { - let p = Path::new(&sdkroot); - match sdk_name { - // Ignore `SDKROOT` if it's clearly set for the wrong platform. - "appletvos" - if sdkroot.contains("TVSimulator.platform") - || sdkroot.contains("MacOSX.platform") => {} - "appletvsimulator" - if sdkroot.contains("TVOS.platform") || sdkroot.contains("MacOSX.platform") => {} - "iphoneos" - if sdkroot.contains("iPhoneSimulator.platform") - || sdkroot.contains("MacOSX.platform") => {} - "iphonesimulator" - if sdkroot.contains("iPhoneOS.platform") || sdkroot.contains("MacOSX.platform") => { - } - "macosx10.15" - if sdkroot.contains("iPhoneOS.platform") - || sdkroot.contains("iPhoneSimulator.platform") => {} - "watchos" - if sdkroot.contains("WatchSimulator.platform") - || sdkroot.contains("MacOSX.platform") => {} - "watchsimulator" - if sdkroot.contains("WatchOS.platform") || sdkroot.contains("MacOSX.platform") => {} - // Ignore `SDKROOT` if it's not a valid path. - _ if !p.is_absolute() || p == Path::new("/") || !p.exists() => {} - _ => return Ok(sdkroot), - } - } - let res = - Command::new("xcrun").arg("--show-sdk-path").arg("-sdk").arg(sdk_name).output().and_then( - |output| { - if output.status.success() { - Ok(String::from_utf8(output.stdout).unwrap()) - } else { - let error = String::from_utf8(output.stderr); - let error = format!("process exit with error: {}", error.unwrap()); - Err(io::Error::new(io::ErrorKind::Other, &error[..])) - } - }, - ); - - match res { - Ok(output) => Ok(output.trim().to_string()), - Err(e) => Err(format!("failed to get {} SDK path: {}", sdk_name, e)), - } -} - -fn add_gcc_ld_path(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { - if let Some(ld_impl) = sess.opts.unstable_opts.gcc_ld { - if let LinkerFlavor::Gcc = flavor { - match ld_impl { - LdImpl::Lld => { - let tools_path = sess.get_tools_search_paths(false); - let gcc_ld_dir = tools_path - .into_iter() - .map(|p| p.join("gcc-ld")) - .find(|p| { - p.join(if sess.host.is_like_windows { "ld.exe" } else { "ld" }).exists() - }) - .unwrap_or_else(|| sess.fatal("rust-lld (as ld) not found")); - cmd.arg({ - let mut arg = OsString::from("-B"); - arg.push(gcc_ld_dir); - arg - }); - cmd.arg(format!("-Wl,-rustc-lld-flavor={}", sess.target.lld_flavor.as_str())); - } - } - } else { - sess.fatal("option `-Z gcc-ld` is used even though linker flavor is not gcc"); - } - } -} diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs deleted file mode 100644 index ce51b2e9531..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ /dev/null @@ -1,1788 +0,0 @@ -use super::archive; -use super::command::Command; -use super::symbol_export; -use rustc_span::symbol::sym; - -use std::ffi::{OsStr, OsString}; -use std::fs::{self, File}; -use std::io::prelude::*; -use std::io::{self, BufWriter}; -use std::path::{Path, PathBuf}; -use std::{env, mem, str}; - -use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; -use rustc_middle::middle::dependency_format::Linkage; -use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind}; -use rustc_middle::ty::TyCtxt; -use rustc_session::config::{self, CrateType, DebugInfo, LinkerPluginLto, Lto, OptLevel, Strip}; -use rustc_session::Session; -use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor}; - -use cc::windows_registry; - -/// Disables non-English messages from localized linkers. -/// Such messages may cause issues with text encoding on Windows (#35785) -/// and prevent inspection of linker output in case of errors, which we occasionally do. -/// This should be acceptable because other messages from rustc are in English anyway, -/// and may also be desirable to improve searchability of the linker diagnostics. -pub fn disable_localization(linker: &mut Command) { - // No harm in setting both env vars simultaneously. - // Unix-style linkers. - linker.env("LC_ALL", "C"); - // MSVC's `link.exe`. - linker.env("VSLANG", "1033"); -} - -// The third parameter is for env vars, used on windows to set up the -// path for MSVC to find its DLLs, and gcc to find its bundled -// toolchain -pub fn get_linker<'a>( - sess: &'a Session, - linker: &Path, - flavor: LinkerFlavor, - self_contained: bool, - target_cpu: &'a str, -) -> Box { - let msvc_tool = windows_registry::find_tool(&sess.opts.target_triple.triple(), "link.exe"); - - // If our linker looks like a batch script on Windows then to execute this - // we'll need to spawn `cmd` explicitly. This is primarily done to handle - // emscripten where the linker is `emcc.bat` and needs to be spawned as - // `cmd /c emcc.bat ...`. - // - // This worked historically but is needed manually since #42436 (regression - // was tagged as #42791) and some more info can be found on #44443 for - // emscripten itself. - let mut cmd = match linker.to_str() { - Some(linker) if cfg!(windows) && linker.ends_with(".bat") => Command::bat_script(linker), - _ => match flavor { - LinkerFlavor::Lld(f) => Command::lld(linker, f), - LinkerFlavor::Msvc if sess.opts.cg.linker.is_none() && sess.target.linker.is_none() => { - Command::new(msvc_tool.as_ref().map_or(linker, |t| t.path())) - } - _ => Command::new(linker), - }, - }; - - // UWP apps have API restrictions enforced during Store submissions. - // To comply with the Windows App Certification Kit, - // MSVC needs to link with the Store versions of the runtime libraries (vcruntime, msvcrt, etc). - let t = &sess.target; - if (flavor == LinkerFlavor::Msvc || flavor == LinkerFlavor::Lld(LldFlavor::Link)) - && t.vendor == "uwp" - { - if let Some(ref tool) = msvc_tool { - let original_path = tool.path(); - if let Some(ref root_lib_path) = original_path.ancestors().nth(4) { - let arch = match t.arch.as_ref() { - "x86_64" => Some("x64"), - "x86" => Some("x86"), - "aarch64" => Some("arm64"), - "arm" => Some("arm"), - _ => None, - }; - if let Some(ref a) = arch { - // FIXME: Move this to `fn linker_with_args`. - let mut arg = OsString::from("/LIBPATH:"); - arg.push(format!("{}\\lib\\{}\\store", root_lib_path.display(), a)); - cmd.arg(&arg); - } else { - warn!("arch is not supported"); - } - } else { - warn!("MSVC root path lib location not found"); - } - } else { - warn!("link.exe not found"); - } - } - - // The compiler's sysroot often has some bundled tools, so add it to the - // PATH for the child. - let mut new_path = sess.get_tools_search_paths(self_contained); - let mut msvc_changed_path = false; - if sess.target.is_like_msvc { - if let Some(ref tool) = msvc_tool { - cmd.args(tool.args()); - for &(ref k, ref v) in tool.env() { - if k == "PATH" { - new_path.extend(env::split_paths(v)); - msvc_changed_path = true; - } else { - cmd.env(k, v); - } - } - } - } - - if !msvc_changed_path { - if let Some(path) = env::var_os("PATH") { - new_path.extend(env::split_paths(&path)); - } - } - cmd.env("PATH", env::join_paths(new_path).unwrap()); - - // FIXME: Move `/LIBPATH` addition for uwp targets from the linker construction - // to the linker args construction. - assert!(cmd.get_args().is_empty() || sess.target.vendor == "uwp"); - match flavor { - LinkerFlavor::Lld(LldFlavor::Link) | LinkerFlavor::Msvc => { - Box::new(MsvcLinker { cmd, sess }) as Box - } - LinkerFlavor::Em => Box::new(EmLinker { cmd, sess }) as Box, - LinkerFlavor::Gcc => { - Box::new(GccLinker { cmd, sess, target_cpu, hinted_static: false, is_ld: false }) - as Box - } - - LinkerFlavor::Lld(LldFlavor::Ld) - | LinkerFlavor::Lld(LldFlavor::Ld64) - | LinkerFlavor::Ld => { - Box::new(GccLinker { cmd, sess, target_cpu, hinted_static: false, is_ld: true }) - as Box - } - - LinkerFlavor::Lld(LldFlavor::Wasm) => Box::new(WasmLd::new(cmd, sess)) as Box, - - LinkerFlavor::PtxLinker => Box::new(PtxLinker { cmd, sess }) as Box, - - LinkerFlavor::BpfLinker => Box::new(BpfLinker { cmd, sess }) as Box, - - LinkerFlavor::L4Bender => Box::new(L4Bender::new(cmd, sess)) as Box, - } -} - -/// Linker abstraction used by `back::link` to build up the command to invoke a -/// linker. -/// -/// This trait is the total list of requirements needed by `back::link` and -/// represents the meaning of each option being passed down. This trait is then -/// used to dispatch on whether a GNU-like linker (generally `ld.exe`) or an -/// MSVC linker (e.g., `link.exe`) is being used. -pub trait Linker { - fn cmd(&mut self) -> &mut Command; - fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path); - fn link_dylib(&mut self, lib: &str, verbatim: bool, as_needed: bool); - fn link_rust_dylib(&mut self, lib: &str, path: &Path); - fn link_framework(&mut self, framework: &str, as_needed: bool); - fn link_staticlib(&mut self, lib: &str, verbatim: bool); - fn link_rlib(&mut self, lib: &Path); - fn link_whole_rlib(&mut self, lib: &Path); - fn link_whole_staticlib(&mut self, lib: &str, verbatim: bool, search_path: &[PathBuf]); - fn include_path(&mut self, path: &Path); - fn framework_path(&mut self, path: &Path); - fn output_filename(&mut self, path: &Path); - fn add_object(&mut self, path: &Path); - fn gc_sections(&mut self, keep_metadata: bool); - fn no_gc_sections(&mut self); - fn full_relro(&mut self); - fn partial_relro(&mut self); - fn no_relro(&mut self); - fn optimize(&mut self); - fn pgo_gen(&mut self); - fn control_flow_guard(&mut self); - fn debuginfo(&mut self, strip: Strip, natvis_debugger_visualizers: &[PathBuf]); - fn no_crt_objects(&mut self); - fn no_default_libraries(&mut self); - fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]); - fn subsystem(&mut self, subsystem: &str); - fn group_start(&mut self); - fn group_end(&mut self); - fn linker_plugin_lto(&mut self); - fn add_eh_frame_header(&mut self) {} - fn add_no_exec(&mut self) {} - fn add_as_needed(&mut self) {} - fn reset_per_library_state(&mut self) {} -} - -impl dyn Linker + '_ { - pub fn arg(&mut self, arg: impl AsRef) { - self.cmd().arg(arg); - } - - pub fn args(&mut self, args: impl IntoIterator>) { - self.cmd().args(args); - } - - pub fn take_cmd(&mut self) -> Command { - mem::replace(self.cmd(), Command::new("")) - } -} - -pub struct GccLinker<'a> { - cmd: Command, - sess: &'a Session, - target_cpu: &'a str, - hinted_static: bool, // Keeps track of the current hinting mode. - // Link as ld - is_ld: bool, -} - -impl<'a> GccLinker<'a> { - /// Passes an argument directly to the linker. - /// - /// When the linker is not ld-like such as when using a compiler as a linker, the argument is - /// prepended by `-Wl,`. - fn linker_arg(&mut self, arg: impl AsRef) -> &mut Self { - self.linker_args(&[arg]); - self - } - - /// Passes a series of arguments directly to the linker. - /// - /// When the linker is ld-like, the arguments are simply appended to the command. When the - /// linker is not ld-like such as when using a compiler as a linker, the arguments are joined by - /// commas to form an argument that is then prepended with `-Wl`. In this situation, only a - /// single argument is appended to the command to ensure that the order of the arguments is - /// preserved by the compiler. - fn linker_args(&mut self, args: &[impl AsRef]) -> &mut Self { - if self.is_ld { - args.into_iter().for_each(|a| { - self.cmd.arg(a); - }); - } else { - if !args.is_empty() { - let mut s = OsString::from("-Wl"); - for a in args { - s.push(","); - s.push(a); - } - self.cmd.arg(s); - } - } - self - } - - fn takes_hints(&self) -> bool { - // Really this function only returns true if the underlying linker - // configured for a compiler is binutils `ld.bfd` and `ld.gold`. We - // don't really have a foolproof way to detect that, so rule out some - // platforms where currently this is guaranteed to *not* be the case: - // - // * On OSX they have their own linker, not binutils' - // * For WebAssembly the only functional linker is LLD, which doesn't - // support hint flags - !self.sess.target.is_like_osx && !self.sess.target.is_like_wasm - } - - // Some platforms take hints about whether a library is static or dynamic. - // For those that support this, we ensure we pass the option if the library - // was flagged "static" (most defaults are dynamic) to ensure that if - // libfoo.a and libfoo.so both exist that the right one is chosen. - fn hint_static(&mut self) { - if !self.takes_hints() { - return; - } - if !self.hinted_static { - self.linker_arg("-Bstatic"); - self.hinted_static = true; - } - } - - fn hint_dynamic(&mut self) { - if !self.takes_hints() { - return; - } - if self.hinted_static { - self.linker_arg("-Bdynamic"); - self.hinted_static = false; - } - } - - fn push_linker_plugin_lto_args(&mut self, plugin_path: Option<&OsStr>) { - if let Some(plugin_path) = plugin_path { - let mut arg = OsString::from("-plugin="); - arg.push(plugin_path); - self.linker_arg(&arg); - } - - let opt_level = match self.sess.opts.optimize { - config::OptLevel::No => "O0", - config::OptLevel::Less => "O1", - config::OptLevel::Default | config::OptLevel::Size | config::OptLevel::SizeMin => "O2", - config::OptLevel::Aggressive => "O3", - }; - - if let Some(path) = &self.sess.opts.unstable_opts.profile_sample_use { - self.linker_arg(&format!("-plugin-opt=sample-profile={}", path.display())); - }; - self.linker_args(&[ - &format!("-plugin-opt={}", opt_level), - &format!("-plugin-opt=mcpu={}", self.target_cpu), - ]); - } - - fn build_dylib(&mut self, out_filename: &Path) { - // On mac we need to tell the linker to let this library be rpathed - if self.sess.target.is_like_osx { - if !self.is_ld { - self.cmd.arg("-dynamiclib"); - } - - self.linker_arg("-dylib"); - - // Note that the `osx_rpath_install_name` option here is a hack - // purely to support rustbuild right now, we should get a more - // principled solution at some point to force the compiler to pass - // the right `-Wl,-install_name` with an `@rpath` in it. - if self.sess.opts.cg.rpath || self.sess.opts.unstable_opts.osx_rpath_install_name { - let mut rpath = OsString::from("@rpath/"); - rpath.push(out_filename.file_name().unwrap()); - self.linker_args(&[OsString::from("-install_name"), rpath]); - } - } else { - self.cmd.arg("-shared"); - if self.sess.target.is_like_windows { - // The output filename already contains `dll_suffix` so - // the resulting import library will have a name in the - // form of libfoo.dll.a - let implib_name = - out_filename.file_name().and_then(|file| file.to_str()).map(|file| { - format!( - "{}{}{}", - self.sess.target.staticlib_prefix, - file, - self.sess.target.staticlib_suffix - ) - }); - if let Some(implib_name) = implib_name { - let implib = out_filename.parent().map(|dir| dir.join(&implib_name)); - if let Some(implib) = implib { - self.linker_arg(&format!("--out-implib={}", (*implib).to_str().unwrap())); - } - } - } - } - } -} - -impl<'a> Linker for GccLinker<'a> { - fn cmd(&mut self) -> &mut Command { - &mut self.cmd - } - - fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path) { - match output_kind { - LinkOutputKind::DynamicNoPicExe => { - if !self.is_ld && self.sess.target.linker_is_gnu { - self.cmd.arg("-no-pie"); - } - } - LinkOutputKind::DynamicPicExe => { - // noop on windows w/ gcc & ld, error w/ lld - if !self.sess.target.is_like_windows { - // `-pie` works for both gcc wrapper and ld. - self.cmd.arg("-pie"); - } - } - LinkOutputKind::StaticNoPicExe => { - // `-static` works for both gcc wrapper and ld. - self.cmd.arg("-static"); - if !self.is_ld && self.sess.target.linker_is_gnu { - self.cmd.arg("-no-pie"); - } - } - LinkOutputKind::StaticPicExe => { - if !self.is_ld { - // Note that combination `-static -pie` doesn't work as expected - // for the gcc wrapper, `-static` in that case suppresses `-pie`. - self.cmd.arg("-static-pie"); - } else { - // `--no-dynamic-linker` and `-z text` are not strictly necessary for producing - // a static pie, but currently passed because gcc and clang pass them. - // The former suppresses the `INTERP` ELF header specifying dynamic linker, - // which is otherwise implicitly injected by ld (but not lld). - // The latter doesn't change anything, only ensures that everything is pic. - self.cmd.args(&["-static", "-pie", "--no-dynamic-linker", "-z", "text"]); - } - } - LinkOutputKind::DynamicDylib => self.build_dylib(out_filename), - LinkOutputKind::StaticDylib => { - self.cmd.arg("-static"); - self.build_dylib(out_filename); - } - LinkOutputKind::WasiReactorExe => { - self.linker_args(&["--entry", "_initialize"]); - } - } - // VxWorks compiler driver introduced `--static-crt` flag specifically for rustc, - // it switches linking for libc and similar system libraries to static without using - // any `#[link]` attributes in the `libc` crate, see #72782 for details. - // FIXME: Switch to using `#[link]` attributes in the `libc` crate - // similarly to other targets. - if self.sess.target.os == "vxworks" - && matches!( - output_kind, - LinkOutputKind::StaticNoPicExe - | LinkOutputKind::StaticPicExe - | LinkOutputKind::StaticDylib - ) - { - self.cmd.arg("--static-crt"); - } - } - - fn link_dylib(&mut self, lib: &str, verbatim: bool, as_needed: bool) { - if self.sess.target.os == "illumos" && lib == "c" { - // libc will be added via late_link_args on illumos so that it will - // appear last in the library search order. - // FIXME: This should be replaced by a more complete and generic - // mechanism for controlling the order of library arguments passed - // to the linker. - return; - } - if !as_needed { - if self.sess.target.is_like_osx { - // FIXME(81490): ld64 doesn't support these flags but macOS 11 - // has -needed-l{} / -needed_library {} - // but we have no way to detect that here. - self.sess.warn("`as-needed` modifier not implemented yet for ld64"); - } else if self.sess.target.linker_is_gnu && !self.sess.target.is_like_windows { - self.linker_arg("--no-as-needed"); - } else { - self.sess.warn("`as-needed` modifier not supported for current linker"); - } - } - self.hint_dynamic(); - self.cmd.arg(format!("-l{}{}", if verbatim { ":" } else { "" }, lib)); - if !as_needed { - if self.sess.target.is_like_osx { - // See above FIXME comment - } else if self.sess.target.linker_is_gnu && !self.sess.target.is_like_windows { - self.linker_arg("--as-needed"); - } - } - } - fn link_staticlib(&mut self, lib: &str, verbatim: bool) { - self.hint_static(); - self.cmd.arg(format!("-l{}{}", if verbatim { ":" } else { "" }, lib)); - } - fn link_rlib(&mut self, lib: &Path) { - self.hint_static(); - self.cmd.arg(lib); - } - fn include_path(&mut self, path: &Path) { - self.cmd.arg("-L").arg(path); - } - fn framework_path(&mut self, path: &Path) { - self.cmd.arg("-F").arg(path); - } - fn output_filename(&mut self, path: &Path) { - self.cmd.arg("-o").arg(path); - } - fn add_object(&mut self, path: &Path) { - self.cmd.arg(path); - } - fn full_relro(&mut self) { - self.linker_args(&["-zrelro", "-znow"]); - } - fn partial_relro(&mut self) { - self.linker_arg("-zrelro"); - } - fn no_relro(&mut self) { - self.linker_arg("-znorelro"); - } - - fn link_rust_dylib(&mut self, lib: &str, _path: &Path) { - self.hint_dynamic(); - self.cmd.arg(format!("-l{}", lib)); - } - - fn link_framework(&mut self, framework: &str, as_needed: bool) { - self.hint_dynamic(); - if !as_needed { - // FIXME(81490): ld64 as of macOS 11 supports the -needed_framework - // flag but we have no way to detect that here. - // self.cmd.arg("-needed_framework").arg(framework); - self.sess.warn("`as-needed` modifier not implemented yet for ld64"); - } - self.cmd.arg("-framework").arg(framework); - } - - // Here we explicitly ask that the entire archive is included into the - // result artifact. For more details see #15460, but the gist is that - // the linker will strip away any unused objects in the archive if we - // don't otherwise explicitly reference them. This can occur for - // libraries which are just providing bindings, libraries with generic - // functions, etc. - fn link_whole_staticlib(&mut self, lib: &str, verbatim: bool, search_path: &[PathBuf]) { - self.hint_static(); - let target = &self.sess.target; - if !target.is_like_osx { - self.linker_arg("--whole-archive").cmd.arg(format!( - "-l{}{}", - if verbatim { ":" } else { "" }, - lib - )); - self.linker_arg("--no-whole-archive"); - } else { - // -force_load is the macOS equivalent of --whole-archive, but it - // involves passing the full path to the library to link. - self.linker_arg("-force_load"); - let lib = archive::find_library(lib, verbatim, search_path, &self.sess); - self.linker_arg(&lib); - } - } - - fn link_whole_rlib(&mut self, lib: &Path) { - self.hint_static(); - if self.sess.target.is_like_osx { - self.linker_arg("-force_load"); - self.linker_arg(&lib); - } else { - self.linker_arg("--whole-archive").cmd.arg(lib); - self.linker_arg("--no-whole-archive"); - } - } - - fn gc_sections(&mut self, keep_metadata: bool) { - // The dead_strip option to the linker specifies that functions and data - // unreachable by the entry point will be removed. This is quite useful - // with Rust's compilation model of compiling libraries at a time into - // one object file. For example, this brings hello world from 1.7MB to - // 458K. - // - // Note that this is done for both executables and dynamic libraries. We - // won't get much benefit from dylibs because LLVM will have already - // stripped away as much as it could. This has not been seen to impact - // link times negatively. - // - // -dead_strip can't be part of the pre_link_args because it's also used - // for partial linking when using multiple codegen units (-r). So we - // insert it here. - if self.sess.target.is_like_osx { - self.linker_arg("-dead_strip"); - - // If we're building a dylib, we don't use --gc-sections because LLVM - // has already done the best it can do, and we also don't want to - // eliminate the metadata. If we're building an executable, however, - // --gc-sections drops the size of hello world from 1.8MB to 597K, a 67% - // reduction. - } else if (self.sess.target.linker_is_gnu || self.sess.target.is_like_wasm) - && !keep_metadata - { - self.linker_arg("--gc-sections"); - } - } - - fn no_gc_sections(&mut self) { - if self.sess.target.linker_is_gnu || self.sess.target.is_like_wasm { - self.linker_arg("--no-gc-sections"); - } - } - - fn optimize(&mut self) { - if !self.sess.target.linker_is_gnu && !self.sess.target.is_like_wasm { - return; - } - - // GNU-style linkers support optimization with -O. GNU ld doesn't - // need a numeric argument, but other linkers do. - if self.sess.opts.optimize == config::OptLevel::Default - || self.sess.opts.optimize == config::OptLevel::Aggressive - { - self.linker_arg("-O1"); - } - } - - fn pgo_gen(&mut self) { - if !self.sess.target.linker_is_gnu { - return; - } - - // If we're doing PGO generation stuff and on a GNU-like linker, use the - // "-u" flag to properly pull in the profiler runtime bits. - // - // This is because LLVM otherwise won't add the needed initialization - // for us on Linux (though the extra flag should be harmless if it - // does). - // - // See https://reviews.llvm.org/D14033 and https://reviews.llvm.org/D14030. - // - // Though it may be worth to try to revert those changes upstream, since - // the overhead of the initialization should be minor. - self.cmd.arg("-u"); - self.cmd.arg("__llvm_profile_runtime"); - } - - fn control_flow_guard(&mut self) {} - - fn debuginfo(&mut self, strip: Strip, _: &[PathBuf]) { - // MacOS linker doesn't support stripping symbols directly anymore. - if self.sess.target.is_like_osx { - return; - } - - match strip { - Strip::None => {} - Strip::Debuginfo => { - self.linker_arg("--strip-debug"); - } - Strip::Symbols => { - self.linker_arg("--strip-all"); - } - } - } - - fn no_crt_objects(&mut self) { - if !self.is_ld { - self.cmd.arg("-nostartfiles"); - } - } - - fn no_default_libraries(&mut self) { - if !self.is_ld { - self.cmd.arg("-nodefaultlibs"); - } - } - - fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]) { - // Symbol visibility in object files typically takes care of this. - if crate_type == CrateType::Executable { - let should_export_executable_symbols = - self.sess.opts.unstable_opts.export_executable_symbols; - if self.sess.target.override_export_symbols.is_none() - && !should_export_executable_symbols - { - return; - } - } - - // We manually create a list of exported symbols to ensure we don't expose any more. - // The object files have far more public symbols than we actually want to export, - // so we hide them all here. - - if !self.sess.target.limit_rdylib_exports { - return; - } - - // FIXME(#99978) hide #[no_mangle] symbols for proc-macros - - let is_windows = self.sess.target.is_like_windows; - let path = tmpdir.join(if is_windows { "list.def" } else { "list" }); - - debug!("EXPORTED SYMBOLS:"); - - if self.sess.target.is_like_osx { - // Write a plain, newline-separated list of symbols - let res: io::Result<()> = try { - let mut f = BufWriter::new(File::create(&path)?); - for sym in symbols { - debug!(" _{}", sym); - writeln!(f, "_{}", sym)?; - } - }; - if let Err(e) = res { - self.sess.fatal(&format!("failed to write lib.def file: {}", e)); - } - } else if is_windows { - let res: io::Result<()> = try { - let mut f = BufWriter::new(File::create(&path)?); - - // .def file similar to MSVC one but without LIBRARY section - // because LD doesn't like when it's empty - writeln!(f, "EXPORTS")?; - for symbol in symbols { - debug!(" _{}", symbol); - writeln!(f, " {}", symbol)?; - } - }; - if let Err(e) = res { - self.sess.fatal(&format!("failed to write list.def file: {}", e)); - } - } else { - // Write an LD version script - let res: io::Result<()> = try { - let mut f = BufWriter::new(File::create(&path)?); - writeln!(f, "{{")?; - if !symbols.is_empty() { - writeln!(f, " global:")?; - for sym in symbols { - debug!(" {};", sym); - writeln!(f, " {};", sym)?; - } - } - writeln!(f, "\n local:\n *;\n}};")?; - }; - if let Err(e) = res { - self.sess.fatal(&format!("failed to write version script: {}", e)); - } - } - - if self.sess.target.is_like_osx { - self.linker_args(&[OsString::from("-exported_symbols_list"), path.into()]); - } else if self.sess.target.is_like_solaris { - self.linker_args(&[OsString::from("-M"), path.into()]); - } else { - if is_windows { - self.linker_arg(path); - } else { - let mut arg = OsString::from("--version-script="); - arg.push(path); - self.linker_arg(arg); - } - } - } - - fn subsystem(&mut self, subsystem: &str) { - self.linker_arg("--subsystem"); - self.linker_arg(&subsystem); - } - - fn reset_per_library_state(&mut self) { - self.hint_dynamic(); // Reset to default before returning the composed command line. - } - - fn group_start(&mut self) { - if self.takes_hints() { - self.linker_arg("--start-group"); - } - } - - fn group_end(&mut self) { - if self.takes_hints() { - self.linker_arg("--end-group"); - } - } - - fn linker_plugin_lto(&mut self) { - match self.sess.opts.cg.linker_plugin_lto { - LinkerPluginLto::Disabled => { - // Nothing to do - } - LinkerPluginLto::LinkerPluginAuto => { - self.push_linker_plugin_lto_args(None); - } - LinkerPluginLto::LinkerPlugin(ref path) => { - self.push_linker_plugin_lto_args(Some(path.as_os_str())); - } - } - } - - // Add the `GNU_EH_FRAME` program header which is required to locate unwinding information. - // Some versions of `gcc` add it implicitly, some (e.g. `musl-gcc`) don't, - // so we just always add it. - fn add_eh_frame_header(&mut self) { - self.linker_arg("--eh-frame-hdr"); - } - - fn add_no_exec(&mut self) { - if self.sess.target.is_like_windows { - self.linker_arg("--nxcompat"); - } else if self.sess.target.linker_is_gnu { - self.linker_arg("-znoexecstack"); - } - } - - fn add_as_needed(&mut self) { - if self.sess.target.linker_is_gnu && !self.sess.target.is_like_windows { - self.linker_arg("--as-needed"); - } else if self.sess.target.is_like_solaris { - // -z ignore is the Solaris equivalent to the GNU ld --as-needed option - self.linker_args(&["-z", "ignore"]); - } - } -} - -pub struct MsvcLinker<'a> { - cmd: Command, - sess: &'a Session, -} - -impl<'a> Linker for MsvcLinker<'a> { - fn cmd(&mut self) -> &mut Command { - &mut self.cmd - } - - fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path) { - match output_kind { - LinkOutputKind::DynamicNoPicExe - | LinkOutputKind::DynamicPicExe - | LinkOutputKind::StaticNoPicExe - | LinkOutputKind::StaticPicExe => {} - LinkOutputKind::DynamicDylib | LinkOutputKind::StaticDylib => { - self.cmd.arg("/DLL"); - let mut arg: OsString = "/IMPLIB:".into(); - arg.push(out_filename.with_extension("dll.lib")); - self.cmd.arg(arg); - } - LinkOutputKind::WasiReactorExe => { - panic!("can't link as reactor on non-wasi target"); - } - } - } - - fn link_rlib(&mut self, lib: &Path) { - self.cmd.arg(lib); - } - fn add_object(&mut self, path: &Path) { - self.cmd.arg(path); - } - - fn gc_sections(&mut self, _keep_metadata: bool) { - // MSVC's ICF (Identical COMDAT Folding) link optimization is - // slow for Rust and thus we disable it by default when not in - // optimization build. - if self.sess.opts.optimize != config::OptLevel::No { - self.cmd.arg("/OPT:REF,ICF"); - } else { - // It is necessary to specify NOICF here, because /OPT:REF - // implies ICF by default. - self.cmd.arg("/OPT:REF,NOICF"); - } - } - - fn no_gc_sections(&mut self) { - self.cmd.arg("/OPT:NOREF,NOICF"); - } - - fn link_dylib(&mut self, lib: &str, verbatim: bool, _as_needed: bool) { - self.cmd.arg(format!("{}{}", lib, if verbatim { "" } else { ".lib" })); - } - - fn link_rust_dylib(&mut self, lib: &str, path: &Path) { - // When producing a dll, the MSVC linker may not actually emit a - // `foo.lib` file if the dll doesn't actually export any symbols, so we - // check to see if the file is there and just omit linking to it if it's - // not present. - let name = format!("{}.dll.lib", lib); - if path.join(&name).exists() { - self.cmd.arg(name); - } - } - - fn link_staticlib(&mut self, lib: &str, verbatim: bool) { - self.cmd.arg(format!("{}{}", lib, if verbatim { "" } else { ".lib" })); - } - - fn full_relro(&mut self) { - // noop - } - - fn partial_relro(&mut self) { - // noop - } - - fn no_relro(&mut self) { - // noop - } - - fn no_crt_objects(&mut self) { - // noop - } - - fn no_default_libraries(&mut self) { - self.cmd.arg("/NODEFAULTLIB"); - } - - fn include_path(&mut self, path: &Path) { - let mut arg = OsString::from("/LIBPATH:"); - arg.push(path); - self.cmd.arg(&arg); - } - - fn output_filename(&mut self, path: &Path) { - let mut arg = OsString::from("/OUT:"); - arg.push(path); - self.cmd.arg(&arg); - } - - fn framework_path(&mut self, _path: &Path) { - bug!("frameworks are not supported on windows") - } - fn link_framework(&mut self, _framework: &str, _as_needed: bool) { - bug!("frameworks are not supported on windows") - } - - fn link_whole_staticlib(&mut self, lib: &str, verbatim: bool, _search_path: &[PathBuf]) { - self.cmd.arg(format!("/WHOLEARCHIVE:{}{}", lib, if verbatim { "" } else { ".lib" })); - } - fn link_whole_rlib(&mut self, path: &Path) { - let mut arg = OsString::from("/WHOLEARCHIVE:"); - arg.push(path); - self.cmd.arg(arg); - } - fn optimize(&mut self) { - // Needs more investigation of `/OPT` arguments - } - - fn pgo_gen(&mut self) { - // Nothing needed here. - } - - fn control_flow_guard(&mut self) { - self.cmd.arg("/guard:cf"); - } - - fn debuginfo(&mut self, strip: Strip, natvis_debugger_visualizers: &[PathBuf]) { - match strip { - Strip::None => { - // This will cause the Microsoft linker to generate a PDB file - // from the CodeView line tables in the object files. - self.cmd.arg("/DEBUG"); - - // This will cause the Microsoft linker to embed .natvis info into the PDB file - let natvis_dir_path = self.sess.sysroot.join("lib\\rustlib\\etc"); - if let Ok(natvis_dir) = fs::read_dir(&natvis_dir_path) { - for entry in natvis_dir { - match entry { - Ok(entry) => { - let path = entry.path(); - if path.extension() == Some("natvis".as_ref()) { - let mut arg = OsString::from("/NATVIS:"); - arg.push(path); - self.cmd.arg(arg); - } - } - Err(err) => { - self.sess - .warn(&format!("error enumerating natvis directory: {}", err)); - } - } - } - } - - // This will cause the Microsoft linker to embed .natvis info for all crates into the PDB file - for path in natvis_debugger_visualizers { - let mut arg = OsString::from("/NATVIS:"); - arg.push(path); - self.cmd.arg(arg); - } - } - Strip::Debuginfo | Strip::Symbols => { - self.cmd.arg("/DEBUG:NONE"); - } - } - } - - // Currently the compiler doesn't use `dllexport` (an LLVM attribute) to - // export symbols from a dynamic library. When building a dynamic library, - // however, we're going to want some symbols exported, so this function - // generates a DEF file which lists all the symbols. - // - // The linker will read this `*.def` file and export all the symbols from - // the dynamic library. Note that this is not as simple as just exporting - // all the symbols in the current crate (as specified by `codegen.reachable`) - // but rather we also need to possibly export the symbols of upstream - // crates. Upstream rlibs may be linked statically to this dynamic library, - // in which case they may continue to transitively be used and hence need - // their symbols exported. - fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]) { - // Symbol visibility takes care of this typically - if crate_type == CrateType::Executable { - let should_export_executable_symbols = - self.sess.opts.unstable_opts.export_executable_symbols; - if !should_export_executable_symbols { - return; - } - } - - let path = tmpdir.join("lib.def"); - let res: io::Result<()> = try { - let mut f = BufWriter::new(File::create(&path)?); - - // Start off with the standard module name header and then go - // straight to exports. - writeln!(f, "LIBRARY")?; - writeln!(f, "EXPORTS")?; - for symbol in symbols { - debug!(" _{}", symbol); - writeln!(f, " {}", symbol)?; - } - }; - if let Err(e) = res { - self.sess.fatal(&format!("failed to write lib.def file: {}", e)); - } - let mut arg = OsString::from("/DEF:"); - arg.push(path); - self.cmd.arg(&arg); - } - - fn subsystem(&mut self, subsystem: &str) { - // Note that previous passes of the compiler validated this subsystem, - // so we just blindly pass it to the linker. - self.cmd.arg(&format!("/SUBSYSTEM:{}", subsystem)); - - // Windows has two subsystems we're interested in right now, the console - // and windows subsystems. These both implicitly have different entry - // points (starting symbols). The console entry point starts with - // `mainCRTStartup` and the windows entry point starts with - // `WinMainCRTStartup`. These entry points, defined in system libraries, - // will then later probe for either `main` or `WinMain`, respectively to - // start the application. - // - // In Rust we just always generate a `main` function so we want control - // to always start there, so we force the entry point on the windows - // subsystem to be `mainCRTStartup` to get everything booted up - // correctly. - // - // For more information see RFC #1665 - if subsystem == "windows" { - self.cmd.arg("/ENTRY:mainCRTStartup"); - } - } - - // MSVC doesn't need group indicators - fn group_start(&mut self) {} - fn group_end(&mut self) {} - - fn linker_plugin_lto(&mut self) { - // Do nothing - } - - fn add_no_exec(&mut self) { - self.cmd.arg("/NXCOMPAT"); - } -} - -pub struct EmLinker<'a> { - cmd: Command, - sess: &'a Session, -} - -impl<'a> Linker for EmLinker<'a> { - fn cmd(&mut self) -> &mut Command { - &mut self.cmd - } - - fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {} - - fn include_path(&mut self, path: &Path) { - self.cmd.arg("-L").arg(path); - } - - fn link_staticlib(&mut self, lib: &str, _verbatim: bool) { - self.cmd.arg("-l").arg(lib); - } - - fn output_filename(&mut self, path: &Path) { - self.cmd.arg("-o").arg(path); - } - - fn add_object(&mut self, path: &Path) { - self.cmd.arg(path); - } - - fn link_dylib(&mut self, lib: &str, verbatim: bool, _as_needed: bool) { - // Emscripten always links statically - self.link_staticlib(lib, verbatim); - } - - fn link_whole_staticlib(&mut self, lib: &str, verbatim: bool, _search_path: &[PathBuf]) { - // not supported? - self.link_staticlib(lib, verbatim); - } - - fn link_whole_rlib(&mut self, lib: &Path) { - // not supported? - self.link_rlib(lib); - } - - fn link_rust_dylib(&mut self, lib: &str, _path: &Path) { - self.link_dylib(lib, false, true); - } - - fn link_rlib(&mut self, lib: &Path) { - self.add_object(lib); - } - - fn full_relro(&mut self) { - // noop - } - - fn partial_relro(&mut self) { - // noop - } - - fn no_relro(&mut self) { - // noop - } - - fn framework_path(&mut self, _path: &Path) { - bug!("frameworks are not supported on Emscripten") - } - - fn link_framework(&mut self, _framework: &str, _as_needed: bool) { - bug!("frameworks are not supported on Emscripten") - } - - fn gc_sections(&mut self, _keep_metadata: bool) { - // noop - } - - fn no_gc_sections(&mut self) { - // noop - } - - fn optimize(&mut self) { - // Emscripten performs own optimizations - self.cmd.arg(match self.sess.opts.optimize { - OptLevel::No => "-O0", - OptLevel::Less => "-O1", - OptLevel::Default => "-O2", - OptLevel::Aggressive => "-O3", - OptLevel::Size => "-Os", - OptLevel::SizeMin => "-Oz", - }); - } - - fn pgo_gen(&mut self) { - // noop, but maybe we need something like the gnu linker? - } - - fn control_flow_guard(&mut self) {} - - fn debuginfo(&mut self, _strip: Strip, _: &[PathBuf]) { - // Preserve names or generate source maps depending on debug info - self.cmd.arg(match self.sess.opts.debuginfo { - DebugInfo::None => "-g0", - DebugInfo::Limited => "--profiling-funcs", - DebugInfo::Full => "-g", - }); - } - - fn no_crt_objects(&mut self) {} - - fn no_default_libraries(&mut self) { - self.cmd.arg("-nodefaultlibs"); - } - - fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) { - debug!("EXPORTED SYMBOLS:"); - - self.cmd.arg("-s"); - - let mut arg = OsString::from("EXPORTED_FUNCTIONS="); - let encoded = serde_json::to_string( - &symbols.iter().map(|sym| "_".to_owned() + sym).collect::>(), - ) - .unwrap(); - debug!("{}", encoded); - - arg.push(encoded); - - self.cmd.arg(arg); - } - - fn subsystem(&mut self, _subsystem: &str) { - // noop - } - - // Appears not necessary on Emscripten - fn group_start(&mut self) {} - fn group_end(&mut self) {} - - fn linker_plugin_lto(&mut self) { - // Do nothing - } -} - -pub struct WasmLd<'a> { - cmd: Command, - sess: &'a Session, -} - -impl<'a> WasmLd<'a> { - fn new(mut cmd: Command, sess: &'a Session) -> WasmLd<'a> { - // If the atomics feature is enabled for wasm then we need a whole bunch - // of flags: - // - // * `--shared-memory` - the link won't even succeed without this, flags - // the one linear memory as `shared` - // - // * `--max-memory=1G` - when specifying a shared memory this must also - // be specified. We conservatively choose 1GB but users should be able - // to override this with `-C link-arg`. - // - // * `--import-memory` - it doesn't make much sense for memory to be - // exported in a threaded module because typically you're - // sharing memory and instantiating the module multiple times. As a - // result if it were exported then we'd just have no sharing. - // - // * `--export=__wasm_init_memory` - when using `--passive-segments` the - // linker will synthesize this function, and so we need to make sure - // that our usage of `--export` below won't accidentally cause this - // function to get deleted. - // - // * `--export=*tls*` - when `#[thread_local]` symbols are used these - // symbols are how the TLS segments are initialized and configured. - if sess.target_features.contains(&sym::atomics) { - cmd.arg("--shared-memory"); - cmd.arg("--max-memory=1073741824"); - cmd.arg("--import-memory"); - cmd.arg("--export=__wasm_init_memory"); - cmd.arg("--export=__wasm_init_tls"); - cmd.arg("--export=__tls_size"); - cmd.arg("--export=__tls_align"); - cmd.arg("--export=__tls_base"); - } - WasmLd { cmd, sess } - } -} - -impl<'a> Linker for WasmLd<'a> { - fn cmd(&mut self) -> &mut Command { - &mut self.cmd - } - - fn set_output_kind(&mut self, output_kind: LinkOutputKind, _out_filename: &Path) { - match output_kind { - LinkOutputKind::DynamicNoPicExe - | LinkOutputKind::DynamicPicExe - | LinkOutputKind::StaticNoPicExe - | LinkOutputKind::StaticPicExe => {} - LinkOutputKind::DynamicDylib | LinkOutputKind::StaticDylib => { - self.cmd.arg("--no-entry"); - } - LinkOutputKind::WasiReactorExe => { - self.cmd.arg("--entry"); - self.cmd.arg("_initialize"); - } - } - } - - fn link_dylib(&mut self, lib: &str, _verbatim: bool, _as_needed: bool) { - self.cmd.arg("-l").arg(lib); - } - - fn link_staticlib(&mut self, lib: &str, _verbatim: bool) { - self.cmd.arg("-l").arg(lib); - } - - fn link_rlib(&mut self, lib: &Path) { - self.cmd.arg(lib); - } - - fn include_path(&mut self, path: &Path) { - self.cmd.arg("-L").arg(path); - } - - fn framework_path(&mut self, _path: &Path) { - panic!("frameworks not supported") - } - - fn output_filename(&mut self, path: &Path) { - self.cmd.arg("-o").arg(path); - } - - fn add_object(&mut self, path: &Path) { - self.cmd.arg(path); - } - - fn full_relro(&mut self) {} - - fn partial_relro(&mut self) {} - - fn no_relro(&mut self) {} - - fn link_rust_dylib(&mut self, lib: &str, _path: &Path) { - self.cmd.arg("-l").arg(lib); - } - - fn link_framework(&mut self, _framework: &str, _as_needed: bool) { - panic!("frameworks not supported") - } - - fn link_whole_staticlib(&mut self, lib: &str, _verbatim: bool, _search_path: &[PathBuf]) { - self.cmd.arg("-l").arg(lib); - } - - fn link_whole_rlib(&mut self, lib: &Path) { - self.cmd.arg(lib); - } - - fn gc_sections(&mut self, _keep_metadata: bool) { - self.cmd.arg("--gc-sections"); - } - - fn no_gc_sections(&mut self) { - self.cmd.arg("--no-gc-sections"); - } - - fn optimize(&mut self) { - self.cmd.arg(match self.sess.opts.optimize { - OptLevel::No => "-O0", - OptLevel::Less => "-O1", - OptLevel::Default => "-O2", - OptLevel::Aggressive => "-O3", - // Currently LLD doesn't support `Os` and `Oz`, so pass through `O2` - // instead. - OptLevel::Size => "-O2", - OptLevel::SizeMin => "-O2", - }); - } - - fn pgo_gen(&mut self) {} - - fn debuginfo(&mut self, strip: Strip, _: &[PathBuf]) { - match strip { - Strip::None => {} - Strip::Debuginfo => { - self.cmd.arg("--strip-debug"); - } - Strip::Symbols => { - self.cmd.arg("--strip-all"); - } - } - } - - fn control_flow_guard(&mut self) {} - - fn no_crt_objects(&mut self) {} - - fn no_default_libraries(&mut self) {} - - fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) { - for sym in symbols { - self.cmd.arg("--export").arg(&sym); - } - - // LLD will hide these otherwise-internal symbols since it only exports - // symbols explicitly passed via the `--export` flags above and hides all - // others. Various bits and pieces of tooling use this, so be sure these - // symbols make their way out of the linker as well. - self.cmd.arg("--export=__heap_base"); - self.cmd.arg("--export=__data_end"); - } - - fn subsystem(&mut self, _subsystem: &str) {} - - // Not needed for now with LLD - fn group_start(&mut self) {} - fn group_end(&mut self) {} - - fn linker_plugin_lto(&mut self) { - // Do nothing for now - } -} - -/// Linker shepherd script for L4Re (Fiasco) -pub struct L4Bender<'a> { - cmd: Command, - sess: &'a Session, - hinted_static: bool, -} - -impl<'a> Linker for L4Bender<'a> { - fn link_dylib(&mut self, _lib: &str, _verbatim: bool, _as_needed: bool) { - bug!("dylibs are not supported on L4Re"); - } - fn link_staticlib(&mut self, lib: &str, _verbatim: bool) { - self.hint_static(); - self.cmd.arg(format!("-PC{}", lib)); - } - fn link_rlib(&mut self, lib: &Path) { - self.hint_static(); - self.cmd.arg(lib); - } - fn include_path(&mut self, path: &Path) { - self.cmd.arg("-L").arg(path); - } - fn framework_path(&mut self, _: &Path) { - bug!("frameworks are not supported on L4Re"); - } - fn output_filename(&mut self, path: &Path) { - self.cmd.arg("-o").arg(path); - } - - fn add_object(&mut self, path: &Path) { - self.cmd.arg(path); - } - - fn full_relro(&mut self) { - self.cmd.arg("-zrelro"); - self.cmd.arg("-znow"); - } - - fn partial_relro(&mut self) { - self.cmd.arg("-zrelro"); - } - - fn no_relro(&mut self) { - self.cmd.arg("-znorelro"); - } - - fn cmd(&mut self) -> &mut Command { - &mut self.cmd - } - - fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {} - - fn link_rust_dylib(&mut self, _: &str, _: &Path) { - panic!("Rust dylibs not supported"); - } - - fn link_framework(&mut self, _framework: &str, _as_needed: bool) { - bug!("frameworks not supported on L4Re"); - } - - fn link_whole_staticlib(&mut self, lib: &str, _verbatim: bool, _search_path: &[PathBuf]) { - self.hint_static(); - self.cmd.arg("--whole-archive").arg(format!("-l{}", lib)); - self.cmd.arg("--no-whole-archive"); - } - - fn link_whole_rlib(&mut self, lib: &Path) { - self.hint_static(); - self.cmd.arg("--whole-archive").arg(lib).arg("--no-whole-archive"); - } - - fn gc_sections(&mut self, keep_metadata: bool) { - if !keep_metadata { - self.cmd.arg("--gc-sections"); - } - } - - fn no_gc_sections(&mut self) { - self.cmd.arg("--no-gc-sections"); - } - - fn optimize(&mut self) { - // GNU-style linkers support optimization with -O. GNU ld doesn't - // need a numeric argument, but other linkers do. - if self.sess.opts.optimize == config::OptLevel::Default - || self.sess.opts.optimize == config::OptLevel::Aggressive - { - self.cmd.arg("-O1"); - } - } - - fn pgo_gen(&mut self) {} - - fn debuginfo(&mut self, strip: Strip, _: &[PathBuf]) { - match strip { - Strip::None => {} - Strip::Debuginfo => { - self.cmd().arg("--strip-debug"); - } - Strip::Symbols => { - self.cmd().arg("--strip-all"); - } - } - } - - fn no_default_libraries(&mut self) { - self.cmd.arg("-nostdlib"); - } - - fn export_symbols(&mut self, _: &Path, _: CrateType, _: &[String]) { - // ToDo, not implemented, copy from GCC - self.sess.warn("exporting symbols not implemented yet for L4Bender"); - return; - } - - fn subsystem(&mut self, subsystem: &str) { - self.cmd.arg(&format!("--subsystem {}", subsystem)); - } - - fn reset_per_library_state(&mut self) { - self.hint_static(); // Reset to default before returning the composed command line. - } - - fn group_start(&mut self) { - self.cmd.arg("--start-group"); - } - - fn group_end(&mut self) { - self.cmd.arg("--end-group"); - } - - fn linker_plugin_lto(&mut self) {} - - fn control_flow_guard(&mut self) {} - - fn no_crt_objects(&mut self) {} -} - -impl<'a> L4Bender<'a> { - pub fn new(cmd: Command, sess: &'a Session) -> L4Bender<'a> { - L4Bender { cmd: cmd, sess: sess, hinted_static: false } - } - - fn hint_static(&mut self) { - if !self.hinted_static { - self.cmd.arg("-static"); - self.hinted_static = true; - } - } -} - -fn for_each_exported_symbols_include_dep<'tcx>( - tcx: TyCtxt<'tcx>, - crate_type: CrateType, - mut callback: impl FnMut(ExportedSymbol<'tcx>, SymbolExportInfo, CrateNum), -) { - for &(symbol, info) in tcx.exported_symbols(LOCAL_CRATE).iter() { - callback(symbol, info, LOCAL_CRATE); - } - - let formats = tcx.dependency_formats(()); - let deps = formats.iter().find_map(|(t, list)| (*t == crate_type).then_some(list)).unwrap(); - - for (index, dep_format) in deps.iter().enumerate() { - let cnum = CrateNum::new(index + 1); - // For each dependency that we are linking to statically ... - if *dep_format == Linkage::Static { - for &(symbol, info) in tcx.exported_symbols(cnum).iter() { - callback(symbol, info, cnum); - } - } - } -} - -pub(crate) fn exported_symbols(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec { - if let Some(ref exports) = tcx.sess.target.override_export_symbols { - return exports.iter().map(ToString::to_string).collect(); - } - - let mut symbols = Vec::new(); - - let export_threshold = symbol_export::crates_export_threshold(&[crate_type]); - for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| { - if info.level.is_below_threshold(export_threshold) { - symbols.push(symbol_export::symbol_name_for_instance_in_crate(tcx, symbol, cnum)); - } - }); - - symbols -} - -pub(crate) fn linked_symbols( - tcx: TyCtxt<'_>, - crate_type: CrateType, -) -> Vec<(String, SymbolExportKind)> { - match crate_type { - CrateType::Executable | CrateType::Cdylib | CrateType::Dylib => (), - CrateType::Staticlib | CrateType::ProcMacro | CrateType::Rlib => { - return Vec::new(); - } - } - - let mut symbols = Vec::new(); - - let export_threshold = symbol_export::crates_export_threshold(&[crate_type]); - for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| { - if info.level.is_below_threshold(export_threshold) || info.used { - symbols.push(( - symbol_export::linking_symbol_name_for_instance_in_crate(tcx, symbol, cnum), - info.kind, - )); - } - }); - - symbols -} - -/// Much simplified and explicit CLI for the NVPTX linker. The linker operates -/// with bitcode and uses LLVM backend to generate a PTX assembly. -pub struct PtxLinker<'a> { - cmd: Command, - sess: &'a Session, -} - -impl<'a> Linker for PtxLinker<'a> { - fn cmd(&mut self) -> &mut Command { - &mut self.cmd - } - - fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {} - - fn link_rlib(&mut self, path: &Path) { - self.cmd.arg("--rlib").arg(path); - } - - fn link_whole_rlib(&mut self, path: &Path) { - self.cmd.arg("--rlib").arg(path); - } - - fn include_path(&mut self, path: &Path) { - self.cmd.arg("-L").arg(path); - } - - fn debuginfo(&mut self, _strip: Strip, _: &[PathBuf]) { - self.cmd.arg("--debug"); - } - - fn add_object(&mut self, path: &Path) { - self.cmd.arg("--bitcode").arg(path); - } - - fn optimize(&mut self) { - match self.sess.lto() { - Lto::Thin | Lto::Fat | Lto::ThinLocal => { - self.cmd.arg("-Olto"); - } - - Lto::No => {} - }; - } - - fn output_filename(&mut self, path: &Path) { - self.cmd.arg("-o").arg(path); - } - - fn link_dylib(&mut self, _lib: &str, _verbatim: bool, _as_needed: bool) { - panic!("external dylibs not supported") - } - - fn link_rust_dylib(&mut self, _lib: &str, _path: &Path) { - panic!("external dylibs not supported") - } - - fn link_staticlib(&mut self, _lib: &str, _verbatim: bool) { - panic!("staticlibs not supported") - } - - fn link_whole_staticlib(&mut self, _lib: &str, _verbatim: bool, _search_path: &[PathBuf]) { - panic!("staticlibs not supported") - } - - fn framework_path(&mut self, _path: &Path) { - panic!("frameworks not supported") - } - - fn link_framework(&mut self, _framework: &str, _as_needed: bool) { - panic!("frameworks not supported") - } - - fn full_relro(&mut self) {} - - fn partial_relro(&mut self) {} - - fn no_relro(&mut self) {} - - fn gc_sections(&mut self, _keep_metadata: bool) {} - - fn no_gc_sections(&mut self) {} - - fn pgo_gen(&mut self) {} - - fn no_crt_objects(&mut self) {} - - fn no_default_libraries(&mut self) {} - - fn control_flow_guard(&mut self) {} - - fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, _symbols: &[String]) {} - - fn subsystem(&mut self, _subsystem: &str) {} - - fn group_start(&mut self) {} - - fn group_end(&mut self) {} - - fn linker_plugin_lto(&mut self) {} -} - -pub struct BpfLinker<'a> { - cmd: Command, - sess: &'a Session, -} - -impl<'a> Linker for BpfLinker<'a> { - fn cmd(&mut self) -> &mut Command { - &mut self.cmd - } - - fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {} - - fn link_rlib(&mut self, path: &Path) { - self.cmd.arg(path); - } - - fn link_whole_rlib(&mut self, path: &Path) { - self.cmd.arg(path); - } - - fn include_path(&mut self, path: &Path) { - self.cmd.arg("-L").arg(path); - } - - fn debuginfo(&mut self, _strip: Strip, _: &[PathBuf]) { - self.cmd.arg("--debug"); - } - - fn add_object(&mut self, path: &Path) { - self.cmd.arg(path); - } - - fn optimize(&mut self) { - self.cmd.arg(match self.sess.opts.optimize { - OptLevel::No => "-O0", - OptLevel::Less => "-O1", - OptLevel::Default => "-O2", - OptLevel::Aggressive => "-O3", - OptLevel::Size => "-Os", - OptLevel::SizeMin => "-Oz", - }); - } - - fn output_filename(&mut self, path: &Path) { - self.cmd.arg("-o").arg(path); - } - - fn link_dylib(&mut self, _lib: &str, _verbatim: bool, _as_needed: bool) { - panic!("external dylibs not supported") - } - - fn link_rust_dylib(&mut self, _lib: &str, _path: &Path) { - panic!("external dylibs not supported") - } - - fn link_staticlib(&mut self, _lib: &str, _verbatim: bool) { - panic!("staticlibs not supported") - } - - fn link_whole_staticlib(&mut self, _lib: &str, _verbatim: bool, _search_path: &[PathBuf]) { - panic!("staticlibs not supported") - } - - fn framework_path(&mut self, _path: &Path) { - panic!("frameworks not supported") - } - - fn link_framework(&mut self, _framework: &str, _as_needed: bool) { - panic!("frameworks not supported") - } - - fn full_relro(&mut self) {} - - fn partial_relro(&mut self) {} - - fn no_relro(&mut self) {} - - fn gc_sections(&mut self, _keep_metadata: bool) {} - - fn no_gc_sections(&mut self) {} - - fn pgo_gen(&mut self) {} - - fn no_crt_objects(&mut self) {} - - fn no_default_libraries(&mut self) {} - - fn control_flow_guard(&mut self) {} - - fn export_symbols(&mut self, tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) { - let path = tmpdir.join("symbols"); - let res: io::Result<()> = try { - let mut f = BufWriter::new(File::create(&path)?); - for sym in symbols { - writeln!(f, "{}", sym)?; - } - }; - if let Err(e) = res { - self.sess.fatal(&format!("failed to write symbols file: {}", e)); - } else { - self.cmd.arg("--export-symbols").arg(&path); - } - } - - fn subsystem(&mut self, _subsystem: &str) {} - - fn group_start(&mut self) {} - - fn group_end(&mut self) {} - - fn linker_plugin_lto(&mut self) {} -} diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs deleted file mode 100644 index cb6244050df..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/lto.rs +++ /dev/null @@ -1,104 +0,0 @@ -use super::write::CodegenContext; -use crate::traits::*; -use crate::ModuleCodegen; - -use rustc_data_structures::memmap::Mmap; -use rustc_errors::FatalError; - -use std::ffi::CString; -use std::sync::Arc; - -pub struct ThinModule { - pub shared: Arc>, - pub idx: usize, -} - -impl ThinModule { - pub fn name(&self) -> &str { - self.shared.module_names[self.idx].to_str().unwrap() - } - - pub fn cost(&self) -> u64 { - // Yes, that's correct, we're using the size of the bytecode as an - // indicator for how costly this codegen unit is. - self.data().len() as u64 - } - - pub fn data(&self) -> &[u8] { - let a = self.shared.thin_buffers.get(self.idx).map(|b| b.data()); - a.unwrap_or_else(|| { - let len = self.shared.thin_buffers.len(); - self.shared.serialized_modules[self.idx - len].data() - }) - } -} - -pub struct ThinShared { - pub data: B::ThinData, - pub thin_buffers: Vec, - pub serialized_modules: Vec>, - pub module_names: Vec, -} - -pub enum LtoModuleCodegen { - Fat { - module: ModuleCodegen, - _serialized_bitcode: Vec>, - }, - - Thin(ThinModule), -} - -impl LtoModuleCodegen { - pub fn name(&self) -> &str { - match *self { - LtoModuleCodegen::Fat { .. } => "everything", - LtoModuleCodegen::Thin(ref m) => m.name(), - } - } - - /// Optimize this module within the given codegen context. - /// - /// This function is unsafe as it'll return a `ModuleCodegen` still - /// points to LLVM data structures owned by this `LtoModuleCodegen`. - /// It's intended that the module returned is immediately code generated and - /// dropped, and then this LTO module is dropped. - pub unsafe fn optimize( - self, - cgcx: &CodegenContext, - ) -> Result, FatalError> { - match self { - LtoModuleCodegen::Fat { mut module, .. } => { - B::optimize_fat(cgcx, &mut module)?; - Ok(module) - } - LtoModuleCodegen::Thin(thin) => B::optimize_thin(cgcx, thin), - } - } - - /// A "gauge" of how costly it is to optimize this module, used to sort - /// biggest modules first. - pub fn cost(&self) -> u64 { - match *self { - // Only one module with fat LTO, so the cost doesn't matter. - LtoModuleCodegen::Fat { .. } => 0, - LtoModuleCodegen::Thin(ref m) => m.cost(), - } - } -} - -pub enum SerializedModule { - Local(M), - FromRlib(Vec), - FromUncompressedFile(Mmap), -} - -impl SerializedModule { - pub fn data(&self) -> &[u8] { - match *self { - SerializedModule::Local(ref m) => m.data(), - SerializedModule::FromRlib(ref m) => m, - SerializedModule::FromUncompressedFile(ref m) => m, - } - } -} diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs deleted file mode 100644 index 0302c28815a..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ /dev/null @@ -1,314 +0,0 @@ -//! Reading of the rustc metadata for rlibs and dylibs - -use std::fs::File; -use std::io::Write; -use std::path::Path; - -use object::write::{self, StandardSegment, Symbol, SymbolSection}; -use object::{ - elf, pe, Architecture, BinaryFormat, Endianness, FileFlags, Object, ObjectSection, - SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope, -}; - -use snap::write::FrameEncoder; - -use rustc_data_structures::memmap::Mmap; -use rustc_data_structures::owning_ref::OwningRef; -use rustc_data_structures::rustc_erase_owner; -use rustc_data_structures::sync::MetadataRef; -use rustc_metadata::fs::METADATA_FILENAME; -use rustc_metadata::EncodedMetadata; -use rustc_session::cstore::MetadataLoader; -use rustc_session::Session; -use rustc_target::abi::Endian; -use rustc_target::spec::{RelocModel, Target}; - -/// The default metadata loader. This is used by cg_llvm and cg_clif. -/// -/// # Metadata location -/// -///
-///
rlib
-///
The metadata can be found in the `lib.rmeta` file inside of the ar archive.
-///
dylib
-///
The metadata can be found in the `.rustc` section of the shared library.
-///
-pub struct DefaultMetadataLoader; - -fn load_metadata_with( - path: &Path, - f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>, -) -> Result { - let file = - File::open(path).map_err(|e| format!("failed to open file '{}': {}", path.display(), e))?; - let data = unsafe { Mmap::map(file) } - .map_err(|e| format!("failed to mmap file '{}': {}", path.display(), e))?; - let metadata = OwningRef::new(data).try_map(f)?; - return Ok(rustc_erase_owner!(metadata.map_owner_box())); -} - -impl MetadataLoader for DefaultMetadataLoader { - fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result { - load_metadata_with(path, |data| { - let archive = object::read::archive::ArchiveFile::parse(&*data) - .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; - - for entry_result in archive.members() { - let entry = entry_result - .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; - if entry.name() == METADATA_FILENAME.as_bytes() { - let data = entry - .data(data) - .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?; - return search_for_metadata(path, data, ".rmeta"); - } - } - - Err(format!("metadata not found in rlib '{}'", path.display())) - }) - } - - fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result { - load_metadata_with(path, |data| search_for_metadata(path, data, ".rustc")) - } -} - -fn search_for_metadata<'a>( - path: &Path, - bytes: &'a [u8], - section: &str, -) -> Result<&'a [u8], String> { - let Ok(file) = object::File::parse(bytes) else { - // The parse above could fail for odd reasons like corruption, but for - // now we just interpret it as this target doesn't support metadata - // emission in object files so the entire byte slice itself is probably - // a metadata file. Ideally though if necessary we could at least check - // the prefix of bytes to see if it's an actual metadata object and if - // not forward the error along here. - return Ok(bytes); - }; - file.section_by_name(section) - .ok_or_else(|| format!("no `{}` section in '{}'", section, path.display()))? - .data() - .map_err(|e| format!("failed to read {} section in '{}': {}", section, path.display(), e)) -} - -pub(crate) fn create_object_file(sess: &Session) -> Option> { - let endianness = match sess.target.options.endian { - Endian::Little => Endianness::Little, - Endian::Big => Endianness::Big, - }; - let architecture = match &sess.target.arch[..] { - "arm" => Architecture::Arm, - "aarch64" => Architecture::Aarch64, - "x86" => Architecture::I386, - "s390x" => Architecture::S390x, - "mips" => Architecture::Mips, - "mips64" => Architecture::Mips64, - "x86_64" => { - if sess.target.pointer_width == 32 { - Architecture::X86_64_X32 - } else { - Architecture::X86_64 - } - } - "powerpc" => Architecture::PowerPc, - "powerpc64" => Architecture::PowerPc64, - "riscv32" => Architecture::Riscv32, - "riscv64" => Architecture::Riscv64, - "sparc64" => Architecture::Sparc64, - // Unsupported architecture. - _ => return None, - }; - let binary_format = if sess.target.is_like_osx { - BinaryFormat::MachO - } else if sess.target.is_like_windows { - BinaryFormat::Coff - } else { - BinaryFormat::Elf - }; - - let mut file = write::Object::new(binary_format, architecture, endianness); - let e_flags = match architecture { - Architecture::Mips => { - let arch = match sess.target.options.cpu.as_ref() { - "mips1" => elf::EF_MIPS_ARCH_1, - "mips2" => elf::EF_MIPS_ARCH_2, - "mips3" => elf::EF_MIPS_ARCH_3, - "mips4" => elf::EF_MIPS_ARCH_4, - "mips5" => elf::EF_MIPS_ARCH_5, - s if s.contains("r6") => elf::EF_MIPS_ARCH_32R6, - _ => elf::EF_MIPS_ARCH_32R2, - }; - // The only ABI LLVM supports for 32-bit MIPS CPUs is o32. - let mut e_flags = elf::EF_MIPS_CPIC | elf::EF_MIPS_ABI_O32 | arch; - if sess.target.options.relocation_model != RelocModel::Static { - e_flags |= elf::EF_MIPS_PIC; - } - if sess.target.options.cpu.contains("r6") { - e_flags |= elf::EF_MIPS_NAN2008; - } - e_flags - } - Architecture::Mips64 => { - // copied from `mips64el-linux-gnuabi64-gcc foo.c -c` - let e_flags = elf::EF_MIPS_CPIC - | elf::EF_MIPS_PIC - | if sess.target.options.cpu.contains("r6") { - elf::EF_MIPS_ARCH_64R6 | elf::EF_MIPS_NAN2008 - } else { - elf::EF_MIPS_ARCH_64R2 - }; - e_flags - } - Architecture::Riscv64 if sess.target.options.features.contains("+d") => { - // copied from `riscv64-linux-gnu-gcc foo.c -c`, note though - // that the `+d` target feature represents whether the double - // float abi is enabled. - let e_flags = elf::EF_RISCV_RVC | elf::EF_RISCV_FLOAT_ABI_DOUBLE; - e_flags - } - _ => 0, - }; - // adapted from LLVM's `MCELFObjectTargetWriter::getOSABI` - let os_abi = match sess.target.options.os.as_ref() { - "hermit" => elf::ELFOSABI_STANDALONE, - "freebsd" => elf::ELFOSABI_FREEBSD, - "solaris" => elf::ELFOSABI_SOLARIS, - _ => elf::ELFOSABI_NONE, - }; - let abi_version = 0; - file.flags = FileFlags::Elf { os_abi, abi_version, e_flags }; - Some(file) -} - -pub enum MetadataPosition { - First, - Last, -} - -// For rlibs we "pack" rustc metadata into a dummy object file. When rustc -// creates a dylib crate type it will pass `--whole-archive` (or the -// platform equivalent) to include all object files from an rlib into the -// final dylib itself. This causes linkers to iterate and try to include all -// files located in an archive, so if metadata is stored in an archive then -// it needs to be of a form that the linker will be able to process. -// -// Note, though, that we don't actually want this metadata to show up in any -// final output of the compiler. Instead this is purely for rustc's own -// metadata tracking purposes. -// -// With the above in mind, each "flavor" of object format gets special -// handling here depending on the target: -// -// * MachO - macos-like targets will insert the metadata into a section that -// is sort of fake dwarf debug info. Inspecting the source of the macos -// linker this causes these sections to be skipped automatically because -// it's not in an allowlist of otherwise well known dwarf section names to -// go into the final artifact. -// -// * WebAssembly - we actually don't have any container format for this -// target. WebAssembly doesn't support the `dylib` crate type anyway so -// there's no need for us to support this at this time. Consequently the -// metadata bytes are simply stored as-is into an rlib. -// -// * COFF - Windows-like targets create an object with a section that has -// the `IMAGE_SCN_LNK_REMOVE` flag set which ensures that if the linker -// ever sees the section it doesn't process it and it's removed. -// -// * ELF - All other targets are similar to Windows in that there's a -// `SHF_EXCLUDE` flag we can set on sections in an object file to get -// automatically removed from the final output. -pub fn create_rmeta_file(sess: &Session, metadata: &[u8]) -> (Vec, MetadataPosition) { - let Some(mut file) = create_object_file(sess) else { - // This is used to handle all "other" targets. This includes targets - // in two categories: - // - // * Some targets don't have support in the `object` crate just yet - // to write an object file. These targets are likely to get filled - // out over time. - // - // * Targets like WebAssembly don't support dylibs, so the purpose - // of putting metadata in object files, to support linking rlibs - // into dylibs, is moot. - // - // In both of these cases it means that linking into dylibs will - // not be supported by rustc. This doesn't matter for targets like - // WebAssembly and for targets not supported by the `object` crate - // yet it means that work will need to be done in the `object` crate - // to add a case above. - return (metadata.to_vec(), MetadataPosition::Last); - }; - let section = file.add_section( - file.segment_name(StandardSegment::Debug).to_vec(), - b".rmeta".to_vec(), - SectionKind::Debug, - ); - match file.format() { - BinaryFormat::Coff => { - file.section_mut(section).flags = - SectionFlags::Coff { characteristics: pe::IMAGE_SCN_LNK_REMOVE }; - } - BinaryFormat::Elf => { - file.section_mut(section).flags = - SectionFlags::Elf { sh_flags: elf::SHF_EXCLUDE as u64 }; - } - _ => {} - }; - file.append_section_data(section, metadata, 1); - (file.write().unwrap(), MetadataPosition::First) -} - -// Historical note: -// -// When using link.exe it was seen that the section name `.note.rustc` -// was getting shortened to `.note.ru`, and according to the PE and COFF -// specification: -// -// > Executable images do not use a string table and do not support -// > section names longer than 8 characters -// -// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format -// -// As a result, we choose a slightly shorter name! As to why -// `.note.rustc` works on MinGW, see -// https://github.com/llvm/llvm-project/blob/llvmorg-12.0.0/lld/COFF/Writer.cpp#L1190-L1197 -pub fn create_compressed_metadata_file( - sess: &Session, - metadata: &EncodedMetadata, - symbol_name: &str, -) -> Vec { - let mut compressed = rustc_metadata::METADATA_HEADER.to_vec(); - FrameEncoder::new(&mut compressed).write_all(metadata.raw_data()).unwrap(); - let Some(mut file) = create_object_file(sess) else { - return compressed.to_vec(); - }; - let section = file.add_section( - file.segment_name(StandardSegment::Data).to_vec(), - b".rustc".to_vec(), - SectionKind::ReadOnlyData, - ); - match file.format() { - BinaryFormat::Elf => { - // Explicitly set no flags to avoid SHF_ALLOC default for data section. - file.section_mut(section).flags = SectionFlags::Elf { sh_flags: 0 }; - } - _ => {} - }; - let offset = file.append_section_data(section, &compressed, 1); - - // For MachO and probably PE this is necessary to prevent the linker from throwing away the - // .rustc section. For ELF this isn't necessary, but it also doesn't harm. - file.add_symbol(Symbol { - name: symbol_name.as_bytes().to_vec(), - value: offset, - size: compressed.len() as u64, - kind: SymbolKind::Data, - scope: SymbolScope::Dynamic, - weak: false, - section: SymbolSection::Section(section), - flags: SymbolFlags::None, - }); - - file.write().unwrap() -} diff --git a/compiler/rustc_codegen_ssa/src/back/mod.rs b/compiler/rustc_codegen_ssa/src/back/mod.rs deleted file mode 100644 index d11ed54eb20..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -pub mod archive; -pub mod command; -pub mod link; -pub mod linker; -pub mod lto; -pub mod metadata; -pub mod rpath; -pub mod symbol_export; -pub mod write; diff --git a/compiler/rustc_codegen_ssa/src/back/rpath.rs b/compiler/rustc_codegen_ssa/src/back/rpath.rs deleted file mode 100644 index 0b5656c9ad1..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/rpath.rs +++ /dev/null @@ -1,114 +0,0 @@ -use pathdiff::diff_paths; -use rustc_data_structures::fx::FxHashSet; -use std::env; -use std::fs; -use std::path::{Path, PathBuf}; - -pub struct RPathConfig<'a> { - pub libs: &'a [&'a Path], - pub out_filename: PathBuf, - pub is_like_osx: bool, - pub has_rpath: bool, - pub linker_is_gnu: bool, -} - -pub fn get_rpath_flags(config: &mut RPathConfig<'_>) -> Vec { - // No rpath on windows - if !config.has_rpath { - return Vec::new(); - } - - debug!("preparing the RPATH!"); - - let rpaths = get_rpaths(config); - let mut flags = rpaths_to_flags(&rpaths); - - if config.linker_is_gnu { - // Use DT_RUNPATH instead of DT_RPATH if available - flags.push("-Wl,--enable-new-dtags".to_owned()); - - // Set DF_ORIGIN for substitute $ORIGIN - flags.push("-Wl,-z,origin".to_owned()); - } - - flags -} - -fn rpaths_to_flags(rpaths: &[String]) -> Vec { - let mut ret = Vec::with_capacity(rpaths.len()); // the minimum needed capacity - - for rpath in rpaths { - if rpath.contains(',') { - ret.push("-Wl,-rpath".into()); - ret.push("-Xlinker".into()); - ret.push(rpath.clone()); - } else { - ret.push(format!("-Wl,-rpath,{}", &(*rpath))); - } - } - - ret -} - -fn get_rpaths(config: &mut RPathConfig<'_>) -> Vec { - debug!("output: {:?}", config.out_filename.display()); - debug!("libs:"); - for libpath in config.libs { - debug!(" {:?}", libpath.display()); - } - - // Use relative paths to the libraries. Binaries can be moved - // as long as they maintain the relative relationship to the - // crates they depend on. - let rpaths = get_rpaths_relative_to_output(config); - - debug!("rpaths:"); - for rpath in &rpaths { - debug!(" {}", rpath); - } - - // Remove duplicates - minimize_rpaths(&rpaths) -} - -fn get_rpaths_relative_to_output(config: &mut RPathConfig<'_>) -> Vec { - config.libs.iter().map(|a| get_rpath_relative_to_output(config, a)).collect() -} - -fn get_rpath_relative_to_output(config: &mut RPathConfig<'_>, lib: &Path) -> String { - // Mac doesn't appear to support $ORIGIN - let prefix = if config.is_like_osx { "@loader_path" } else { "$ORIGIN" }; - - let cwd = env::current_dir().unwrap(); - let mut lib = fs::canonicalize(&cwd.join(lib)).unwrap_or_else(|_| cwd.join(lib)); - lib.pop(); // strip filename - let mut output = cwd.join(&config.out_filename); - output.pop(); // strip filename - let output = fs::canonicalize(&output).unwrap_or(output); - let relative = path_relative_from(&lib, &output) - .unwrap_or_else(|| panic!("couldn't create relative path from {:?} to {:?}", output, lib)); - // FIXME (#9639): This needs to handle non-utf8 paths - format!("{}/{}", prefix, relative.to_str().expect("non-utf8 component in path")) -} - -// This routine is adapted from the *old* Path's `path_relative_from` -// function, which works differently from the new `relative_from` function. -// In particular, this handles the case on unix where both paths are -// absolute but with only the root as the common directory. -fn path_relative_from(path: &Path, base: &Path) -> Option { - diff_paths(path, base) -} - -fn minimize_rpaths(rpaths: &[String]) -> Vec { - let mut set = FxHashSet::default(); - let mut minimized = Vec::new(); - for rpath in rpaths { - if set.insert(rpath) { - minimized.push(rpath.clone()); - } - } - minimized -} - -#[cfg(all(unix, test))] -mod tests; diff --git a/compiler/rustc_codegen_ssa/src/back/rpath/tests.rs b/compiler/rustc_codegen_ssa/src/back/rpath/tests.rs deleted file mode 100644 index 604f19144a6..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/rpath/tests.rs +++ /dev/null @@ -1,72 +0,0 @@ -use super::RPathConfig; -use super::{get_rpath_relative_to_output, minimize_rpaths, rpaths_to_flags}; -use std::path::{Path, PathBuf}; - -#[test] -fn test_rpaths_to_flags() { - let flags = rpaths_to_flags(&["path1".to_string(), "path2".to_string()]); - assert_eq!(flags, ["-Wl,-rpath,path1", "-Wl,-rpath,path2"]); -} - -#[test] -fn test_minimize1() { - let res = minimize_rpaths(&["rpath1".to_string(), "rpath2".to_string(), "rpath1".to_string()]); - assert!(res == ["rpath1", "rpath2",]); -} - -#[test] -fn test_minimize2() { - let res = minimize_rpaths(&[ - "1a".to_string(), - "2".to_string(), - "2".to_string(), - "1a".to_string(), - "4a".to_string(), - "1a".to_string(), - "2".to_string(), - "3".to_string(), - "4a".to_string(), - "3".to_string(), - ]); - assert!(res == ["1a", "2", "4a", "3",]); -} - -#[test] -fn test_rpath_relative() { - if cfg!(target_os = "macos") { - let config = &mut RPathConfig { - libs: &[], - has_rpath: true, - is_like_osx: true, - linker_is_gnu: false, - out_filename: PathBuf::from("bin/rustc"), - }; - let res = get_rpath_relative_to_output(config, Path::new("lib/libstd.so")); - assert_eq!(res, "@loader_path/../lib"); - } else { - let config = &mut RPathConfig { - libs: &[], - out_filename: PathBuf::from("bin/rustc"), - has_rpath: true, - is_like_osx: false, - linker_is_gnu: true, - }; - let res = get_rpath_relative_to_output(config, Path::new("lib/libstd.so")); - assert_eq!(res, "$ORIGIN/../lib"); - } -} - -#[test] -fn test_xlinker() { - let args = rpaths_to_flags(&["a/normal/path".to_string(), "a,comma,path".to_string()]); - - assert_eq!( - args, - vec![ - "-Wl,-rpath,a/normal/path".to_string(), - "-Wl,-rpath".to_string(), - "-Xlinker".to_string(), - "a,comma,path".to_string() - ] - ); -} diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs deleted file mode 100644 index e6b6055759d..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ /dev/null @@ -1,590 +0,0 @@ -use std::collections::hash_map::Entry::*; - -use rustc_ast::expand::allocator::ALLOCATOR_METHODS; -use rustc_data_structures::fx::FxHashMap; -use rustc_hir as hir; -use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LOCAL_CRATE}; -use rustc_hir::Node; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::middle::exported_symbols::{ - metadata_symbol_name, ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel, -}; -use rustc_middle::ty::query::{ExternProviders, Providers}; -use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; -use rustc_middle::ty::Instance; -use rustc_middle::ty::{self, SymbolName, TyCtxt}; -use rustc_session::config::CrateType; -use rustc_target::spec::SanitizerSet; - -pub fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel { - crates_export_threshold(&tcx.sess.crate_types()) -} - -fn crate_export_threshold(crate_type: CrateType) -> SymbolExportLevel { - match crate_type { - CrateType::Executable | CrateType::Staticlib | CrateType::ProcMacro | CrateType::Cdylib => { - SymbolExportLevel::C - } - CrateType::Rlib | CrateType::Dylib => SymbolExportLevel::Rust, - } -} - -pub fn crates_export_threshold(crate_types: &[CrateType]) -> SymbolExportLevel { - if crate_types - .iter() - .any(|&crate_type| crate_export_threshold(crate_type) == SymbolExportLevel::Rust) - { - SymbolExportLevel::Rust - } else { - SymbolExportLevel::C - } -} - -fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap { - assert_eq!(cnum, LOCAL_CRATE); - - if !tcx.sess.opts.output_types.should_codegen() { - return Default::default(); - } - - // Check to see if this crate is a "special runtime crate". These - // crates, implementation details of the standard library, typically - // have a bunch of `pub extern` and `#[no_mangle]` functions as the - // ABI between them. We don't want their symbols to have a `C` - // export level, however, as they're just implementation details. - // Down below we'll hardwire all of the symbols to the `Rust` export - // level instead. - let special_runtime_crate = - tcx.is_panic_runtime(LOCAL_CRATE) || tcx.is_compiler_builtins(LOCAL_CRATE); - - let mut reachable_non_generics: DefIdMap<_> = tcx - .reachable_set(()) - .iter() - .filter_map(|&def_id| { - // We want to ignore some FFI functions that are not exposed from - // this crate. Reachable FFI functions can be lumped into two - // categories: - // - // 1. Those that are included statically via a static library - // 2. Those included otherwise (e.g., dynamically or via a framework) - // - // Although our LLVM module is not literally emitting code for the - // statically included symbols, it's an export of our library which - // needs to be passed on to the linker and encoded in the metadata. - // - // As a result, if this id is an FFI item (foreign item) then we only - // let it through if it's included statically. - match tcx.hir().get_by_def_id(def_id) { - Node::ForeignItem(..) => { - tcx.is_statically_included_foreign_item(def_id).then_some(def_id) - } - - // Only consider nodes that actually have exported symbols. - Node::Item(&hir::Item { - kind: hir::ItemKind::Static(..) | hir::ItemKind::Fn(..), - .. - }) - | Node::ImplItem(&hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. }) => { - let generics = tcx.generics_of(def_id); - if !generics.requires_monomorphization(tcx) - // Functions marked with #[inline] are codegened with "internal" - // linkage and are not exported unless marked with an extern - // indicator - && (!Instance::mono(tcx, def_id.to_def_id()).def.generates_cgu_internal_copy(tcx) - || tcx.codegen_fn_attrs(def_id.to_def_id()).contains_extern_indicator()) - { - Some(def_id) - } else { - None - } - } - - _ => None, - } - }) - .map(|def_id| { - let (export_level, used) = if special_runtime_crate { - let name = tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())).name; - // We won't link right if these symbols are stripped during LTO. - let used = match name { - "rust_eh_personality" - | "rust_eh_register_frames" - | "rust_eh_unregister_frames" => true, - _ => false, - }; - (SymbolExportLevel::Rust, used) - } else { - (symbol_export_level(tcx, def_id.to_def_id()), false) - }; - let codegen_attrs = tcx.codegen_fn_attrs(def_id.to_def_id()); - debug!( - "EXPORTED SYMBOL (local): {} ({:?})", - tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())), - export_level - ); - (def_id.to_def_id(), SymbolExportInfo { - level: export_level, - kind: if tcx.is_static(def_id.to_def_id()) { - if codegen_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { - SymbolExportKind::Tls - } else { - SymbolExportKind::Data - } - } else { - SymbolExportKind::Text - }, - used: codegen_attrs.flags.contains(CodegenFnAttrFlags::USED) - || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) || used, - }) - }) - .collect(); - - if let Some(id) = tcx.proc_macro_decls_static(()) { - reachable_non_generics.insert( - id.to_def_id(), - SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: false, - }, - ); - } - - reachable_non_generics -} - -fn is_reachable_non_generic_provider_local(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - let export_threshold = threshold(tcx); - - if let Some(&info) = tcx.reachable_non_generics(def_id.krate).get(&def_id) { - info.level.is_below_threshold(export_threshold) - } else { - false - } -} - -fn is_reachable_non_generic_provider_extern(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - tcx.reachable_non_generics(def_id.krate).contains_key(&def_id) -} - -fn exported_symbols_provider_local<'tcx>( - tcx: TyCtxt<'tcx>, - cnum: CrateNum, -) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] { - assert_eq!(cnum, LOCAL_CRATE); - - if !tcx.sess.opts.output_types.should_codegen() { - return &[]; - } - - let mut symbols: Vec<_> = tcx - .reachable_non_generics(LOCAL_CRATE) - .iter() - .map(|(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)) - .collect(); - - if tcx.entry_fn(()).is_some() { - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, "main")); - - symbols.push(( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Text, - used: false, - }, - )); - } - - if tcx.allocator_kind(()).is_some() { - for method in ALLOCATOR_METHODS { - let symbol_name = format!("__rust_{}", method.name); - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); - - symbols.push(( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Text, - used: false, - }, - )); - } - } - - if tcx.sess.instrument_coverage() || tcx.sess.opts.cg.profile_generate.enabled() { - // These are weak symbols that point to the profile version and the - // profile name, which need to be treated as exported so LTO doesn't nix - // them. - const PROFILER_WEAK_SYMBOLS: [&str; 2] = - ["__llvm_profile_raw_version", "__llvm_profile_filename"]; - - symbols.extend(PROFILER_WEAK_SYMBOLS.iter().map(|sym| { - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym)); - ( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: false, - }, - ) - })); - } - - if tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) { - let mut msan_weak_symbols = Vec::new(); - - // Similar to profiling, preserve weak msan symbol during LTO. - if tcx.sess.opts.unstable_opts.sanitizer_recover.contains(SanitizerSet::MEMORY) { - msan_weak_symbols.push("__msan_keep_going"); - } - - if tcx.sess.opts.unstable_opts.sanitizer_memory_track_origins != 0 { - msan_weak_symbols.push("__msan_track_origins"); - } - - symbols.extend(msan_weak_symbols.into_iter().map(|sym| { - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym)); - ( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: false, - }, - ) - })); - } - - if tcx.sess.crate_types().contains(&CrateType::Dylib) - || tcx.sess.crate_types().contains(&CrateType::ProcMacro) - { - let symbol_name = metadata_symbol_name(tcx); - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); - - symbols.push(( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: true, - }, - )); - } - - if tcx.sess.opts.share_generics() && tcx.local_crate_exports_generics() { - use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility}; - use rustc_middle::ty::InstanceDef; - - // Normally, we require that shared monomorphizations are not hidden, - // because if we want to re-use a monomorphization from a Rust dylib, it - // needs to be exported. - // However, on platforms that don't allow for Rust dylibs, having - // external linkage is enough for monomorphization to be linked to. - let need_visibility = tcx.sess.target.dynamic_linking && !tcx.sess.target.only_cdylib; - - let (_, cgus) = tcx.collect_and_partition_mono_items(()); - - for (mono_item, &(linkage, visibility)) in cgus.iter().flat_map(|cgu| cgu.items().iter()) { - if linkage != Linkage::External { - // We can only re-use things with external linkage, otherwise - // we'll get a linker error - continue; - } - - if need_visibility && visibility == Visibility::Hidden { - // If we potentially share things from Rust dylibs, they must - // not be hidden - continue; - } - - match *mono_item { - MonoItem::Fn(Instance { def: InstanceDef::Item(def), substs }) => { - if substs.non_erasable_generics().next().is_some() { - let symbol = ExportedSymbol::Generic(def.did, substs); - symbols.push(( - symbol, - SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Text, - used: false, - }, - )); - } - } - MonoItem::Fn(Instance { def: InstanceDef::DropGlue(_, Some(ty)), substs }) => { - // A little sanity-check - debug_assert_eq!( - substs.non_erasable_generics().next(), - Some(GenericArgKind::Type(ty)) - ); - symbols.push(( - ExportedSymbol::DropGlue(ty), - SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Text, - used: false, - }, - )); - } - _ => { - // Any other symbols don't qualify for sharing - } - } - } - } - - // Sort so we get a stable incr. comp. hash. - symbols.sort_by_cached_key(|s| s.0.symbol_name_for_local_instance(tcx)); - - tcx.arena.alloc_from_iter(symbols) -} - -fn upstream_monomorphizations_provider( - tcx: TyCtxt<'_>, - (): (), -) -> DefIdMap, CrateNum>> { - let cnums = tcx.crates(()); - - let mut instances: DefIdMap> = Default::default(); - - let drop_in_place_fn_def_id = tcx.lang_items().drop_in_place_fn(); - - for &cnum in cnums.iter() { - for (exported_symbol, _) in tcx.exported_symbols(cnum).iter() { - let (def_id, substs) = match *exported_symbol { - ExportedSymbol::Generic(def_id, substs) => (def_id, substs), - ExportedSymbol::DropGlue(ty) => { - if let Some(drop_in_place_fn_def_id) = drop_in_place_fn_def_id { - (drop_in_place_fn_def_id, tcx.intern_substs(&[ty.into()])) - } else { - // `drop_in_place` in place does not exist, don't try - // to use it. - continue; - } - } - ExportedSymbol::NonGeneric(..) | ExportedSymbol::NoDefId(..) => { - // These are no monomorphizations - continue; - } - }; - - let substs_map = instances.entry(def_id).or_default(); - - match substs_map.entry(substs) { - Occupied(mut e) => { - // If there are multiple monomorphizations available, - // we select one deterministically. - let other_cnum = *e.get(); - if tcx.stable_crate_id(other_cnum) > tcx.stable_crate_id(cnum) { - e.insert(cnum); - } - } - Vacant(e) => { - e.insert(cnum); - } - } - } - } - - instances -} - -fn upstream_monomorphizations_for_provider( - tcx: TyCtxt<'_>, - def_id: DefId, -) -> Option<&FxHashMap, CrateNum>> { - debug_assert!(!def_id.is_local()); - tcx.upstream_monomorphizations(()).get(&def_id) -} - -fn upstream_drop_glue_for_provider<'tcx>( - tcx: TyCtxt<'tcx>, - substs: SubstsRef<'tcx>, -) -> Option { - if let Some(def_id) = tcx.lang_items().drop_in_place_fn() { - tcx.upstream_monomorphizations_for(def_id).and_then(|monos| monos.get(&substs).cloned()) - } else { - None - } -} - -fn is_unreachable_local_definition_provider(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { - !tcx.reachable_set(()).contains(&def_id) -} - -pub fn provide(providers: &mut Providers) { - providers.reachable_non_generics = reachable_non_generics_provider; - providers.is_reachable_non_generic = is_reachable_non_generic_provider_local; - providers.exported_symbols = exported_symbols_provider_local; - providers.upstream_monomorphizations = upstream_monomorphizations_provider; - providers.is_unreachable_local_definition = is_unreachable_local_definition_provider; - providers.upstream_drop_glue_for = upstream_drop_glue_for_provider; - providers.wasm_import_module_map = wasm_import_module_map; -} - -pub fn provide_extern(providers: &mut ExternProviders) { - providers.is_reachable_non_generic = is_reachable_non_generic_provider_extern; - providers.upstream_monomorphizations_for = upstream_monomorphizations_for_provider; -} - -fn symbol_export_level(tcx: TyCtxt<'_>, sym_def_id: DefId) -> SymbolExportLevel { - // We export anything that's not mangled at the "C" layer as it probably has - // to do with ABI concerns. We do not, however, apply such treatment to - // special symbols in the standard library for various plumbing between - // core/std/allocators/etc. For example symbols used to hook up allocation - // are not considered for export - let codegen_fn_attrs = tcx.codegen_fn_attrs(sym_def_id); - let is_extern = codegen_fn_attrs.contains_extern_indicator(); - let std_internal = - codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL); - - if is_extern && !std_internal { - let target = &tcx.sess.target.llvm_target; - // WebAssembly cannot export data symbols, so reduce their export level - if target.contains("emscripten") { - if let Some(Node::Item(&hir::Item { kind: hir::ItemKind::Static(..), .. })) = - tcx.hir().get_if_local(sym_def_id) - { - return SymbolExportLevel::Rust; - } - } - - SymbolExportLevel::C - } else { - SymbolExportLevel::Rust - } -} - -/// This is the symbol name of the given instance instantiated in a specific crate. -pub fn symbol_name_for_instance_in_crate<'tcx>( - tcx: TyCtxt<'tcx>, - symbol: ExportedSymbol<'tcx>, - instantiating_crate: CrateNum, -) -> String { - // If this is something instantiated in the local crate then we might - // already have cached the name as a query result. - if instantiating_crate == LOCAL_CRATE { - return symbol.symbol_name_for_local_instance(tcx).to_string(); - } - - // This is something instantiated in an upstream crate, so we have to use - // the slower (because uncached) version of computing the symbol name. - match symbol { - ExportedSymbol::NonGeneric(def_id) => { - rustc_symbol_mangling::symbol_name_for_instance_in_crate( - tcx, - Instance::mono(tcx, def_id), - instantiating_crate, - ) - } - ExportedSymbol::Generic(def_id, substs) => { - rustc_symbol_mangling::symbol_name_for_instance_in_crate( - tcx, - Instance::new(def_id, substs), - instantiating_crate, - ) - } - ExportedSymbol::DropGlue(ty) => rustc_symbol_mangling::symbol_name_for_instance_in_crate( - tcx, - Instance::resolve_drop_in_place(tcx, ty), - instantiating_crate, - ), - ExportedSymbol::NoDefId(symbol_name) => symbol_name.to_string(), - } -} - -/// This is the symbol name of the given instance as seen by the linker. -/// -/// On 32-bit Windows symbols are decorated according to their calling conventions. -pub fn linking_symbol_name_for_instance_in_crate<'tcx>( - tcx: TyCtxt<'tcx>, - symbol: ExportedSymbol<'tcx>, - instantiating_crate: CrateNum, -) -> String { - use rustc_target::abi::call::Conv; - - let mut undecorated = symbol_name_for_instance_in_crate(tcx, symbol, instantiating_crate); - - let target = &tcx.sess.target; - if !target.is_like_windows { - // Mach-O has a global "_" suffix and `object` crate will handle it. - // ELF does not have any symbol decorations. - return undecorated; - } - - let x86 = match &target.arch[..] { - "x86" => true, - "x86_64" => false, - // Only x86/64 use symbol decorations. - _ => return undecorated, - }; - - let instance = match symbol { - ExportedSymbol::NonGeneric(def_id) | ExportedSymbol::Generic(def_id, _) - if tcx.is_static(def_id) => - { - None - } - ExportedSymbol::NonGeneric(def_id) => Some(Instance::mono(tcx, def_id)), - ExportedSymbol::Generic(def_id, substs) => Some(Instance::new(def_id, substs)), - // DropGlue always use the Rust calling convention and thus follow the target's default - // symbol decoration scheme. - ExportedSymbol::DropGlue(..) => None, - // NoDefId always follow the target's default symbol decoration scheme. - ExportedSymbol::NoDefId(..) => None, - }; - - let (conv, args) = instance - .map(|i| { - tcx.fn_abi_of_instance(ty::ParamEnv::reveal_all().and((i, ty::List::empty()))) - .unwrap_or_else(|_| bug!("fn_abi_of_instance({i:?}) failed")) - }) - .map(|fnabi| (fnabi.conv, &fnabi.args[..])) - .unwrap_or((Conv::Rust, &[])); - - // Decorate symbols with prefices, suffices and total number of bytes of arguments. - // Reference: https://docs.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170 - let (prefix, suffix) = match conv { - Conv::X86Fastcall => ("@", "@"), - Conv::X86Stdcall => ("_", "@"), - Conv::X86VectorCall => ("", "@@"), - _ => { - if x86 { - undecorated.insert(0, '_'); - } - return undecorated; - } - }; - - let args_in_bytes: u64 = args - .iter() - .map(|abi| abi.layout.size.bytes().next_multiple_of(target.pointer_width as u64 / 8)) - .sum(); - format!("{prefix}{undecorated}{suffix}{args_in_bytes}") -} - -fn wasm_import_module_map(tcx: TyCtxt<'_>, cnum: CrateNum) -> FxHashMap { - // Build up a map from DefId to a `NativeLib` structure, where - // `NativeLib` internally contains information about - // `#[link(wasm_import_module = "...")]` for example. - let native_libs = tcx.native_libraries(cnum); - - let def_id_to_native_lib = native_libs - .iter() - .filter_map(|lib| lib.foreign_module.map(|id| (id, lib))) - .collect::>(); - - let mut ret = FxHashMap::default(); - for (def_id, lib) in tcx.foreign_modules(cnum).iter() { - let module = def_id_to_native_lib.get(&def_id).and_then(|s| s.wasm_import_module); - let Some(module) = module else { continue }; - ret.extend(lib.foreign_items.iter().map(|id| { - assert_eq!(id.krate, cnum); - (*id, module.to_string()) - })); - } - - ret -} diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs deleted file mode 100644 index 1b5ad87107a..00000000000 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ /dev/null @@ -1,2015 +0,0 @@ -use super::link::{self, ensure_removed}; -use super::lto::{self, SerializedModule}; -use super::symbol_export::symbol_name_for_instance_in_crate; - -use crate::{ - CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind, -}; - -use crate::traits::*; -use jobserver::{Acquired, Client}; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::memmap::Mmap; -use rustc_data_structures::profiling::SelfProfilerRef; -use rustc_data_structures::profiling::TimingGuard; -use rustc_data_structures::profiling::VerboseTimingGuard; -use rustc_data_structures::sync::Lrc; -use rustc_errors::emitter::Emitter; -use rustc_errors::{DiagnosticId, FatalError, Handler, Level}; -use rustc_fs_util::link_or_copy; -use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; -use rustc_incremental::{ - copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess, -}; -use rustc_metadata::EncodedMetadata; -use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::middle::exported_symbols::SymbolExportInfo; -use rustc_middle::ty::TyCtxt; -use rustc_session::cgu_reuse_tracker::CguReuseTracker; -use rustc_session::config::{self, CrateType, Lto, OutputFilenames, OutputType}; -use rustc_session::config::{Passes, SwitchWithOptPath}; -use rustc_session::Session; -use rustc_span::source_map::SourceMap; -use rustc_span::symbol::sym; -use rustc_span::{BytePos, FileName, InnerSpan, Pos, Span}; -use rustc_target::spec::{MergeFunctions, SanitizerSet}; - -use std::any::Any; -use std::fs; -use std::io; -use std::marker::PhantomData; -use std::mem; -use std::path::{Path, PathBuf}; -use std::str; -use std::sync::mpsc::{channel, Receiver, Sender}; -use std::sync::Arc; -use std::thread; - -const PRE_LTO_BC_EXT: &str = "pre-lto.bc"; - -/// What kind of object file to emit. -#[derive(Clone, Copy, PartialEq)] -pub enum EmitObj { - // No object file. - None, - - // Just uncompressed llvm bitcode. Provides easy compatibility with - // emscripten's ecc compiler, when used as the linker. - Bitcode, - - // Object code, possibly augmented with a bitcode section. - ObjectCode(BitcodeSection), -} - -/// What kind of llvm bitcode section to embed in an object file. -#[derive(Clone, Copy, PartialEq)] -pub enum BitcodeSection { - // No bitcode section. - None, - - // A full, uncompressed bitcode section. - Full, -} - -/// Module-specific configuration for `optimize_and_codegen`. -pub struct ModuleConfig { - /// Names of additional optimization passes to run. - pub passes: Vec, - /// Some(level) to optimize at a certain level, or None to run - /// absolutely no optimizations (used for the metadata module). - pub opt_level: Option, - - /// Some(level) to optimize binary size, or None to not affect program size. - pub opt_size: Option, - - pub pgo_gen: SwitchWithOptPath, - pub pgo_use: Option, - pub pgo_sample_use: Option, - pub debug_info_for_profiling: bool, - pub instrument_coverage: bool, - pub instrument_gcov: bool, - - pub sanitizer: SanitizerSet, - pub sanitizer_recover: SanitizerSet, - pub sanitizer_memory_track_origins: usize, - - // Flags indicating which outputs to produce. - pub emit_pre_lto_bc: bool, - pub emit_no_opt_bc: bool, - pub emit_bc: bool, - pub emit_ir: bool, - pub emit_asm: bool, - pub emit_obj: EmitObj, - pub emit_thin_lto: bool, - pub bc_cmdline: String, - - // Miscellaneous flags. These are mostly copied from command-line - // options. - pub verify_llvm_ir: bool, - pub no_prepopulate_passes: bool, - pub no_builtins: bool, - pub time_module: bool, - pub vectorize_loop: bool, - pub vectorize_slp: bool, - pub merge_functions: bool, - pub inline_threshold: Option, - pub new_llvm_pass_manager: Option, - pub emit_lifetime_markers: bool, - pub llvm_plugins: Vec, -} - -impl ModuleConfig { - fn new( - kind: ModuleKind, - sess: &Session, - no_builtins: bool, - is_compiler_builtins: bool, - ) -> ModuleConfig { - // If it's a regular module, use `$regular`, otherwise use `$other`. - // `$regular` and `$other` are evaluated lazily. - macro_rules! if_regular { - ($regular: expr, $other: expr) => { - if let ModuleKind::Regular = kind { $regular } else { $other } - }; - } - - let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None); - - let save_temps = sess.opts.cg.save_temps; - - let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe) - || match kind { - ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object), - ModuleKind::Allocator => false, - ModuleKind::Metadata => sess.opts.output_types.contains_key(&OutputType::Metadata), - }; - - let emit_obj = if !should_emit_obj { - EmitObj::None - } else if sess.target.obj_is_bitcode - || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins) - { - // This case is selected if the target uses objects as bitcode, or - // if linker plugin LTO is enabled. In the linker plugin LTO case - // the assumption is that the final link-step will read the bitcode - // and convert it to object code. This may be done by either the - // native linker or rustc itself. - // - // Note, however, that the linker-plugin-lto requested here is - // explicitly ignored for `#![no_builtins]` crates. These crates are - // specifically ignored by rustc's LTO passes and wouldn't work if - // loaded into the linker. These crates define symbols that LLVM - // lowers intrinsics to, and these symbol dependencies aren't known - // until after codegen. As a result any crate marked - // `#![no_builtins]` is assumed to not participate in LTO and - // instead goes on to generate object code. - EmitObj::Bitcode - } else if need_bitcode_in_object(sess) { - EmitObj::ObjectCode(BitcodeSection::Full) - } else { - EmitObj::ObjectCode(BitcodeSection::None) - }; - - ModuleConfig { - passes: if_regular!(sess.opts.cg.passes.clone(), vec![]), - - opt_level: opt_level_and_size, - opt_size: opt_level_and_size, - - pgo_gen: if_regular!( - sess.opts.cg.profile_generate.clone(), - SwitchWithOptPath::Disabled - ), - pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None), - pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None), - debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling, - instrument_coverage: if_regular!(sess.instrument_coverage(), false), - instrument_gcov: if_regular!( - // compiler_builtins overrides the codegen-units settings, - // which is incompatible with -Zprofile which requires that - // only a single codegen unit is used per crate. - sess.opts.unstable_opts.profile && !is_compiler_builtins, - false - ), - - sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()), - sanitizer_recover: if_regular!( - sess.opts.unstable_opts.sanitizer_recover, - SanitizerSet::empty() - ), - sanitizer_memory_track_origins: if_regular!( - sess.opts.unstable_opts.sanitizer_memory_track_origins, - 0 - ), - - emit_pre_lto_bc: if_regular!( - save_temps || need_pre_lto_bitcode_for_incr_comp(sess), - false - ), - emit_no_opt_bc: if_regular!(save_temps, false), - emit_bc: if_regular!( - save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode), - save_temps - ), - emit_ir: if_regular!( - sess.opts.output_types.contains_key(&OutputType::LlvmAssembly), - false - ), - emit_asm: if_regular!( - sess.opts.output_types.contains_key(&OutputType::Assembly), - false - ), - emit_obj, - emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto, - bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(), - - verify_llvm_ir: sess.verify_llvm_ir(), - no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes, - no_builtins: no_builtins || sess.target.no_builtins, - - // Exclude metadata and allocator modules from time_passes output, - // since they throw off the "LLVM passes" measurement. - time_module: if_regular!(true, false), - - // Copy what clang does by turning on loop vectorization at O2 and - // slp vectorization at O3. - vectorize_loop: !sess.opts.cg.no_vectorize_loops - && (sess.opts.optimize == config::OptLevel::Default - || sess.opts.optimize == config::OptLevel::Aggressive), - vectorize_slp: !sess.opts.cg.no_vectorize_slp - && sess.opts.optimize == config::OptLevel::Aggressive, - - // Some targets (namely, NVPTX) interact badly with the - // MergeFunctions pass. This is because MergeFunctions can generate - // new function calls which may interfere with the target calling - // convention; e.g. for the NVPTX target, PTX kernels should not - // call other PTX kernels. MergeFunctions can also be configured to - // generate aliases instead, but aliases are not supported by some - // backends (again, NVPTX). Therefore, allow targets to opt out of - // the MergeFunctions pass, but otherwise keep the pass enabled (at - // O2 and O3) since it can be useful for reducing code size. - merge_functions: match sess - .opts - .unstable_opts - .merge_functions - .unwrap_or(sess.target.merge_functions) - { - MergeFunctions::Disabled => false, - MergeFunctions::Trampolines | MergeFunctions::Aliases => { - sess.opts.optimize == config::OptLevel::Default - || sess.opts.optimize == config::OptLevel::Aggressive - } - }, - - inline_threshold: sess.opts.cg.inline_threshold, - new_llvm_pass_manager: sess.opts.unstable_opts.new_llvm_pass_manager, - emit_lifetime_markers: sess.emit_lifetime_markers(), - llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]), - } - } - - pub fn bitcode_needed(&self) -> bool { - self.emit_bc - || self.emit_obj == EmitObj::Bitcode - || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) - } -} - -/// Configuration passed to the function returned by the `target_machine_factory`. -pub struct TargetMachineFactoryConfig { - /// Split DWARF is enabled in LLVM by checking that `TM.MCOptions.SplitDwarfFile` isn't empty, - /// so the path to the dwarf object has to be provided when we create the target machine. - /// This can be ignored by backends which do not need it for their Split DWARF support. - pub split_dwarf_file: Option, -} - -impl TargetMachineFactoryConfig { - pub fn new( - cgcx: &CodegenContext, - module_name: &str, - ) -> TargetMachineFactoryConfig { - let split_dwarf_file = if cgcx.target_can_use_split_dwarf { - cgcx.output_filenames.split_dwarf_path( - cgcx.split_debuginfo, - cgcx.split_dwarf_kind, - Some(module_name), - ) - } else { - None - }; - TargetMachineFactoryConfig { split_dwarf_file } - } -} - -pub type TargetMachineFactoryFn = Arc< - dyn Fn(TargetMachineFactoryConfig) -> Result<::TargetMachine, String> - + Send - + Sync, ->; - -pub type ExportedSymbols = FxHashMap>>; - -/// Additional resources used by optimize_and_codegen (not module specific) -#[derive(Clone)] -pub struct CodegenContext { - // Resources needed when running LTO - pub backend: B, - pub prof: SelfProfilerRef, - pub lto: Lto, - pub save_temps: bool, - pub fewer_names: bool, - pub time_trace: bool, - pub exported_symbols: Option>, - pub opts: Arc, - pub crate_types: Vec, - pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>, - pub output_filenames: Arc, - pub regular_module_config: Arc, - pub metadata_module_config: Arc, - pub allocator_module_config: Arc, - pub tm_factory: TargetMachineFactoryFn, - pub msvc_imps_needed: bool, - pub is_pe_coff: bool, - pub target_can_use_split_dwarf: bool, - pub target_pointer_width: u32, - pub target_arch: String, - pub debuginfo: config::DebugInfo, - pub split_debuginfo: rustc_target::spec::SplitDebuginfo, - pub split_dwarf_kind: rustc_session::config::SplitDwarfKind, - - // Number of cgus excluding the allocator/metadata modules - pub total_cgus: usize, - // Handler to use for diagnostics produced during codegen. - pub diag_emitter: SharedEmitter, - // LLVM optimizations for which we want to print remarks. - pub remark: Passes, - // Worker thread number - pub worker: usize, - // The incremental compilation session directory, or None if we are not - // compiling incrementally - pub incr_comp_session_dir: Option, - // Used to update CGU re-use information during the thinlto phase. - pub cgu_reuse_tracker: CguReuseTracker, - // Channel back to the main control thread to send messages to - pub coordinator_send: Sender>, -} - -impl CodegenContext { - pub fn create_diag_handler(&self) -> Handler { - Handler::with_emitter(true, None, Box::new(self.diag_emitter.clone())) - } - - pub fn config(&self, kind: ModuleKind) -> &ModuleConfig { - match kind { - ModuleKind::Regular => &self.regular_module_config, - ModuleKind::Metadata => &self.metadata_module_config, - ModuleKind::Allocator => &self.allocator_module_config, - } - } -} - -fn generate_lto_work( - cgcx: &CodegenContext, - needs_fat_lto: Vec>, - needs_thin_lto: Vec<(String, B::ThinBuffer)>, - import_only_modules: Vec<(SerializedModule, WorkProduct)>, -) -> Vec<(WorkItem, u64)> { - let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work"); - - let (lto_modules, copy_jobs) = if !needs_fat_lto.is_empty() { - assert!(needs_thin_lto.is_empty()); - let lto_module = - B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise()); - (vec![lto_module], vec![]) - } else { - assert!(needs_fat_lto.is_empty()); - B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules).unwrap_or_else(|e| e.raise()) - }; - - lto_modules - .into_iter() - .map(|module| { - let cost = module.cost(); - (WorkItem::LTO(module), cost) - }) - .chain(copy_jobs.into_iter().map(|wp| { - ( - WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen { - name: wp.cgu_name.clone(), - source: wp, - }), - 0, - ) - })) - .collect() -} - -pub struct CompiledModules { - pub modules: Vec, - pub allocator_module: Option, -} - -fn need_bitcode_in_object(sess: &Session) -> bool { - let requested_for_rlib = sess.opts.cg.embed_bitcode - && sess.crate_types().contains(&CrateType::Rlib) - && sess.opts.output_types.contains_key(&OutputType::Exe); - let forced_by_target = sess.target.forces_embed_bitcode; - requested_for_rlib || forced_by_target -} - -fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool { - if sess.opts.incremental.is_none() { - return false; - } - - match sess.lto() { - Lto::No => false, - Lto::Fat | Lto::Thin | Lto::ThinLocal => true, - } -} - -pub fn start_async_codegen( - backend: B, - tcx: TyCtxt<'_>, - target_cpu: String, - metadata: EncodedMetadata, - metadata_module: Option, - total_cgus: usize, -) -> OngoingCodegen { - let (coordinator_send, coordinator_receive) = channel(); - let sess = tcx.sess; - - let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID); - let no_builtins = tcx.sess.contains_name(crate_attrs, sym::no_builtins); - let is_compiler_builtins = tcx.sess.contains_name(crate_attrs, sym::compiler_builtins); - - let crate_info = CrateInfo::new(tcx, target_cpu); - - let regular_config = - ModuleConfig::new(ModuleKind::Regular, sess, no_builtins, is_compiler_builtins); - let metadata_config = - ModuleConfig::new(ModuleKind::Metadata, sess, no_builtins, is_compiler_builtins); - let allocator_config = - ModuleConfig::new(ModuleKind::Allocator, sess, no_builtins, is_compiler_builtins); - - let (shared_emitter, shared_emitter_main) = SharedEmitter::new(); - let (codegen_worker_send, codegen_worker_receive) = channel(); - - let coordinator_thread = start_executing_work( - backend.clone(), - tcx, - &crate_info, - shared_emitter, - codegen_worker_send, - coordinator_receive, - total_cgus, - sess.jobserver.clone(), - Arc::new(regular_config), - Arc::new(metadata_config), - Arc::new(allocator_config), - coordinator_send.clone(), - ); - - OngoingCodegen { - backend, - metadata, - metadata_module, - crate_info, - - codegen_worker_receive, - shared_emitter_main, - coordinator: Coordinator { - sender: coordinator_send, - future: Some(coordinator_thread), - phantom: PhantomData, - }, - output_filenames: tcx.output_filenames(()).clone(), - } -} - -fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( - sess: &Session, - compiled_modules: &CompiledModules, -) -> FxHashMap { - let mut work_products = FxHashMap::default(); - - if sess.opts.incremental.is_none() { - return work_products; - } - - let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir"); - - for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) { - let mut files = Vec::new(); - if let Some(object_file_path) = &module.object { - files.push(("o", object_file_path.as_path())); - } - if let Some(dwarf_object_file_path) = &module.dwarf_object { - files.push(("dwo", dwarf_object_file_path.as_path())); - } - - if let Some((id, product)) = - copy_cgu_workproduct_to_incr_comp_cache_dir(sess, &module.name, files.as_slice()) - { - work_products.insert(id, product); - } - } - - work_products -} - -fn produce_final_output_artifacts( - sess: &Session, - compiled_modules: &CompiledModules, - crate_output: &OutputFilenames, -) { - let mut user_wants_bitcode = false; - let mut user_wants_objects = false; - - // Produce final compile outputs. - let copy_gracefully = |from: &Path, to: &Path| { - if let Err(e) = fs::copy(from, to) { - sess.err(&format!("could not copy {:?} to {:?}: {}", from, to, e)); - } - }; - - let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| { - if compiled_modules.modules.len() == 1 { - // 1) Only one codegen unit. In this case it's no difficulty - // to copy `foo.0.x` to `foo.x`. - let module_name = Some(&compiled_modules.modules[0].name[..]); - let path = crate_output.temp_path(output_type, module_name); - copy_gracefully(&path, &crate_output.path(output_type)); - if !sess.opts.cg.save_temps && !keep_numbered { - // The user just wants `foo.x`, not `foo.#module-name#.x`. - ensure_removed(sess.diagnostic(), &path); - } - } else { - let ext = crate_output - .temp_path(output_type, None) - .extension() - .unwrap() - .to_str() - .unwrap() - .to_owned(); - - if crate_output.outputs.contains_key(&output_type) { - // 2) Multiple codegen units, with `--emit foo=some_name`. We have - // no good solution for this case, so warn the user. - sess.warn(&format!( - "ignoring emit path because multiple .{} files \ - were produced", - ext - )); - } else if crate_output.single_output_file.is_some() { - // 3) Multiple codegen units, with `-o some_name`. We have - // no good solution for this case, so warn the user. - sess.warn(&format!( - "ignoring -o because multiple .{} files \ - were produced", - ext - )); - } else { - // 4) Multiple codegen units, but no explicit name. We - // just leave the `foo.0.x` files in place. - // (We don't have to do any work in this case.) - } - } - }; - - // Flag to indicate whether the user explicitly requested bitcode. - // Otherwise, we produced it only as a temporary output, and will need - // to get rid of it. - for output_type in crate_output.outputs.keys() { - match *output_type { - OutputType::Bitcode => { - user_wants_bitcode = true; - // Copy to .bc, but always keep the .0.bc. There is a later - // check to figure out if we should delete .0.bc files, or keep - // them for making an rlib. - copy_if_one_unit(OutputType::Bitcode, true); - } - OutputType::LlvmAssembly => { - copy_if_one_unit(OutputType::LlvmAssembly, false); - } - OutputType::Assembly => { - copy_if_one_unit(OutputType::Assembly, false); - } - OutputType::Object => { - user_wants_objects = true; - copy_if_one_unit(OutputType::Object, true); - } - OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {} - } - } - - // Clean up unwanted temporary files. - - // We create the following files by default: - // - #crate#.#module-name#.bc - // - #crate#.#module-name#.o - // - #crate#.crate.metadata.bc - // - #crate#.crate.metadata.o - // - #crate#.o (linked from crate.##.o) - // - #crate#.bc (copied from crate.##.bc) - // We may create additional files if requested by the user (through - // `-C save-temps` or `--emit=` flags). - - if !sess.opts.cg.save_temps { - // Remove the temporary .#module-name#.o objects. If the user didn't - // explicitly request bitcode (with --emit=bc), and the bitcode is not - // needed for building an rlib, then we must remove .#module-name#.bc as - // well. - - // Specific rules for keeping .#module-name#.bc: - // - If the user requested bitcode (`user_wants_bitcode`), and - // codegen_units > 1, then keep it. - // - If the user requested bitcode but codegen_units == 1, then we - // can toss .#module-name#.bc because we copied it to .bc earlier. - // - If we're not building an rlib and the user didn't request - // bitcode, then delete .#module-name#.bc. - // If you change how this works, also update back::link::link_rlib, - // where .#module-name#.bc files are (maybe) deleted after making an - // rlib. - let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe); - - let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units() > 1; - - let keep_numbered_objects = - needs_crate_object || (user_wants_objects && sess.codegen_units() > 1); - - for module in compiled_modules.modules.iter() { - if let Some(ref path) = module.object { - if !keep_numbered_objects { - ensure_removed(sess.diagnostic(), path); - } - } - - if let Some(ref path) = module.dwarf_object { - if !keep_numbered_objects { - ensure_removed(sess.diagnostic(), path); - } - } - - if let Some(ref path) = module.bytecode { - if !keep_numbered_bitcode { - ensure_removed(sess.diagnostic(), path); - } - } - } - - if !user_wants_bitcode { - if let Some(ref allocator_module) = compiled_modules.allocator_module { - if let Some(ref path) = allocator_module.bytecode { - ensure_removed(sess.diagnostic(), path); - } - } - } - } - - // We leave the following files around by default: - // - #crate#.o - // - #crate#.crate.metadata.o - // - #crate#.bc - // These are used in linking steps and will be cleaned up afterward. -} - -pub enum WorkItem { - /// Optimize a newly codegened, totally unoptimized module. - Optimize(ModuleCodegen), - /// Copy the post-LTO artifacts from the incremental cache to the output - /// directory. - CopyPostLtoArtifacts(CachedModuleCodegen), - /// Performs (Thin)LTO on the given module. - LTO(lto::LtoModuleCodegen), -} - -impl WorkItem { - pub fn module_kind(&self) -> ModuleKind { - match *self { - WorkItem::Optimize(ref m) => m.kind, - WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular, - } - } - - fn start_profiling<'a>(&self, cgcx: &'a CodegenContext) -> TimingGuard<'a> { - match *self { - WorkItem::Optimize(ref m) => { - cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name) - } - WorkItem::CopyPostLtoArtifacts(ref m) => cgcx - .prof - .generic_activity_with_arg("codegen_copy_artifacts_from_incr_cache", &*m.name), - WorkItem::LTO(ref m) => { - cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()) - } - } - } - - /// Generate a short description of this work item suitable for use as a thread name. - fn short_description(&self) -> String { - // `pthread_setname()` on *nix is limited to 15 characters and longer names are ignored. - // Use very short descriptions in this case to maximize the space available for the module name. - // Windows does not have that limitation so use slightly more descriptive names there. - match self { - WorkItem::Optimize(m) => { - #[cfg(windows)] - return format!("optimize module {}", m.name); - #[cfg(not(windows))] - return format!("opt {}", m.name); - } - WorkItem::CopyPostLtoArtifacts(m) => { - #[cfg(windows)] - return format!("copy LTO artifacts for {}", m.name); - #[cfg(not(windows))] - return format!("copy {}", m.name); - } - WorkItem::LTO(m) => { - #[cfg(windows)] - return format!("LTO module {}", m.name()); - #[cfg(not(windows))] - return format!("LTO {}", m.name()); - } - } - } -} - -enum WorkItemResult { - Compiled(CompiledModule), - NeedsLink(ModuleCodegen), - NeedsFatLTO(FatLTOInput), - NeedsThinLTO(String, B::ThinBuffer), -} - -pub enum FatLTOInput { - Serialized { name: String, buffer: B::ModuleBuffer }, - InMemory(ModuleCodegen), -} - -fn execute_work_item( - cgcx: &CodegenContext, - work_item: WorkItem, -) -> Result, FatalError> { - let module_config = cgcx.config(work_item.module_kind()); - - match work_item { - WorkItem::Optimize(module) => execute_optimize_work_item(cgcx, module, module_config), - WorkItem::CopyPostLtoArtifacts(module) => { - Ok(execute_copy_from_cache_work_item(cgcx, module, module_config)) - } - WorkItem::LTO(module) => execute_lto_work_item(cgcx, module, module_config), - } -} - -// Actual LTO type we end up choosing based on multiple factors. -pub enum ComputedLtoType { - No, - Thin, - Fat, -} - -pub fn compute_per_cgu_lto_type( - sess_lto: &Lto, - opts: &config::Options, - sess_crate_types: &[CrateType], - module_kind: ModuleKind, -) -> ComputedLtoType { - // Metadata modules never participate in LTO regardless of the lto - // settings. - if module_kind == ModuleKind::Metadata { - return ComputedLtoType::No; - } - - // If the linker does LTO, we don't have to do it. Note that we - // keep doing full LTO, if it is requested, as not to break the - // assumption that the output will be a single module. - let linker_does_lto = opts.cg.linker_plugin_lto.enabled(); - - // When we're automatically doing ThinLTO for multi-codegen-unit - // builds we don't actually want to LTO the allocator modules if - // it shows up. This is due to various linker shenanigans that - // we'll encounter later. - let is_allocator = module_kind == ModuleKind::Allocator; - - // We ignore a request for full crate graph LTO if the crate type - // is only an rlib, as there is no full crate graph to process, - // that'll happen later. - // - // This use case currently comes up primarily for targets that - // require LTO so the request for LTO is always unconditionally - // passed down to the backend, but we don't actually want to do - // anything about it yet until we've got a final product. - let is_rlib = sess_crate_types.len() == 1 && sess_crate_types[0] == CrateType::Rlib; - - match sess_lto { - Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin, - Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin, - Lto::Fat if !is_rlib => ComputedLtoType::Fat, - _ => ComputedLtoType::No, - } -} - -fn execute_optimize_work_item( - cgcx: &CodegenContext, - module: ModuleCodegen, - module_config: &ModuleConfig, -) -> Result, FatalError> { - let diag_handler = cgcx.create_diag_handler(); - - unsafe { - B::optimize(cgcx, &diag_handler, &module, module_config)?; - } - - // After we've done the initial round of optimizations we need to - // decide whether to synchronously codegen this module or ship it - // back to the coordinator thread for further LTO processing (which - // has to wait for all the initial modules to be optimized). - - let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind); - - // If we're doing some form of incremental LTO then we need to be sure to - // save our module to disk first. - let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc { - let filename = pre_lto_bitcode_filename(&module.name); - cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename)) - } else { - None - }; - - match lto_type { - ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config), - ComputedLtoType::Thin => { - let (name, thin_buffer) = B::prepare_thin(module); - if let Some(path) = bitcode { - fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| { - panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); - }); - } - Ok(WorkItemResult::NeedsThinLTO(name, thin_buffer)) - } - ComputedLtoType::Fat => match bitcode { - Some(path) => { - let (name, buffer) = B::serialize_module(module); - fs::write(&path, buffer.data()).unwrap_or_else(|e| { - panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); - }); - Ok(WorkItemResult::NeedsFatLTO(FatLTOInput::Serialized { name, buffer })) - } - None => Ok(WorkItemResult::NeedsFatLTO(FatLTOInput::InMemory(module))), - }, - } -} - -fn execute_copy_from_cache_work_item( - cgcx: &CodegenContext, - module: CachedModuleCodegen, - module_config: &ModuleConfig, -) -> WorkItemResult { - assert!(module_config.emit_obj != EmitObj::None); - - let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap(); - - let load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| { - let source_file = in_incr_comp_dir(&incr_comp_session_dir, saved_path); - debug!( - "copying pre-existing module `{}` from {:?} to {}", - module.name, - source_file, - output_path.display() - ); - match link_or_copy(&source_file, &output_path) { - Ok(_) => Some(output_path), - Err(err) => { - let diag_handler = cgcx.create_diag_handler(); - diag_handler.err(&format!( - "unable to copy {} to {}: {}", - source_file.display(), - output_path.display(), - err - )); - None - } - } - }; - - let object = load_from_incr_comp_dir( - cgcx.output_filenames.temp_path(OutputType::Object, Some(&module.name)), - &module.source.saved_files.get("o").expect("no saved object file in work product"), - ); - let dwarf_object = - module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| { - let dwarf_obj_out = cgcx - .output_filenames - .split_dwarf_path(cgcx.split_debuginfo, cgcx.split_dwarf_kind, Some(&module.name)) - .expect( - "saved dwarf object in work product but `split_dwarf_path` returned `None`", - ); - load_from_incr_comp_dir(dwarf_obj_out, &saved_dwarf_object_file) - }); - - WorkItemResult::Compiled(CompiledModule { - name: module.name, - kind: ModuleKind::Regular, - object, - dwarf_object, - bytecode: None, - }) -} - -fn execute_lto_work_item( - cgcx: &CodegenContext, - module: lto::LtoModuleCodegen, - module_config: &ModuleConfig, -) -> Result, FatalError> { - let module = unsafe { module.optimize(cgcx)? }; - finish_intra_module_work(cgcx, module, module_config) -} - -fn finish_intra_module_work( - cgcx: &CodegenContext, - module: ModuleCodegen, - module_config: &ModuleConfig, -) -> Result, FatalError> { - let diag_handler = cgcx.create_diag_handler(); - - if !cgcx.opts.unstable_opts.combine_cgu - || module.kind == ModuleKind::Metadata - || module.kind == ModuleKind::Allocator - { - let module = unsafe { B::codegen(cgcx, &diag_handler, module, module_config)? }; - Ok(WorkItemResult::Compiled(module)) - } else { - Ok(WorkItemResult::NeedsLink(module)) - } -} - -pub enum Message { - Token(io::Result), - NeedsFatLTO { - result: FatLTOInput, - worker_id: usize, - }, - NeedsThinLTO { - name: String, - thin_buffer: B::ThinBuffer, - worker_id: usize, - }, - NeedsLink { - module: ModuleCodegen, - worker_id: usize, - }, - Done { - result: Result>, - worker_id: usize, - }, - CodegenDone { - llvm_work_item: WorkItem, - cost: u64, - }, - AddImportOnlyModule { - module_data: SerializedModule, - work_product: WorkProduct, - }, - CodegenComplete, - CodegenItem, - CodegenAborted, -} - -struct Diagnostic { - msg: String, - code: Option, - lvl: Level, -} - -#[derive(PartialEq, Clone, Copy, Debug)] -enum MainThreadWorkerState { - Idle, - Codegenning, - LLVMing, -} - -fn start_executing_work( - backend: B, - tcx: TyCtxt<'_>, - crate_info: &CrateInfo, - shared_emitter: SharedEmitter, - codegen_worker_send: Sender>, - coordinator_receive: Receiver>, - total_cgus: usize, - jobserver: Client, - regular_config: Arc, - metadata_config: Arc, - allocator_config: Arc, - tx_to_llvm_workers: Sender>, -) -> thread::JoinHandle> { - let coordinator_send = tx_to_llvm_workers; - let sess = tcx.sess; - - // Compute the set of symbols we need to retain when doing LTO (if we need to) - let exported_symbols = { - let mut exported_symbols = FxHashMap::default(); - - let copy_symbols = |cnum| { - let symbols = tcx - .exported_symbols(cnum) - .iter() - .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl)) - .collect(); - Arc::new(symbols) - }; - - match sess.lto() { - Lto::No => None, - Lto::ThinLocal => { - exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE)); - Some(Arc::new(exported_symbols)) - } - Lto::Fat | Lto::Thin => { - exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE)); - for &cnum in tcx.crates(()).iter() { - exported_symbols.insert(cnum, copy_symbols(cnum)); - } - Some(Arc::new(exported_symbols)) - } - } - }; - - // First up, convert our jobserver into a helper thread so we can use normal - // mpsc channels to manage our messages and such. - // After we've requested tokens then we'll, when we can, - // get tokens on `coordinator_receive` which will - // get managed in the main loop below. - let coordinator_send2 = coordinator_send.clone(); - let helper = jobserver - .into_helper_thread(move |token| { - drop(coordinator_send2.send(Box::new(Message::Token::(token)))); - }) - .expect("failed to spawn helper thread"); - - let mut each_linked_rlib_for_lto = Vec::new(); - drop(link::each_linked_rlib(crate_info, &mut |cnum, path| { - if link::ignored_for_lto(sess, crate_info, cnum) { - return; - } - each_linked_rlib_for_lto.push((cnum, path.to_path_buf())); - })); - - let ol = - if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - // If we know that we won’t be doing codegen, create target machines without optimisation. - config::OptLevel::No - } else { - tcx.backend_optimization_level(()) - }; - let backend_features = tcx.global_backend_features(()); - let cgcx = CodegenContext:: { - backend: backend.clone(), - crate_types: sess.crate_types().to_vec(), - each_linked_rlib_for_lto, - lto: sess.lto(), - fewer_names: sess.fewer_names(), - save_temps: sess.opts.cg.save_temps, - time_trace: sess.opts.unstable_opts.llvm_time_trace, - opts: Arc::new(sess.opts.clone()), - prof: sess.prof.clone(), - exported_symbols, - remark: sess.opts.cg.remark.clone(), - worker: 0, - incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()), - cgu_reuse_tracker: sess.cgu_reuse_tracker.clone(), - coordinator_send, - diag_emitter: shared_emitter.clone(), - output_filenames: tcx.output_filenames(()).clone(), - regular_module_config: regular_config, - metadata_module_config: metadata_config, - allocator_module_config: allocator_config, - tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features), - total_cgus, - msvc_imps_needed: msvc_imps_needed(tcx), - is_pe_coff: tcx.sess.target.is_like_windows, - target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(), - target_pointer_width: tcx.sess.target.pointer_width, - target_arch: tcx.sess.target.arch.to_string(), - debuginfo: tcx.sess.opts.debuginfo, - split_debuginfo: tcx.sess.split_debuginfo(), - split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind, - }; - - // This is the "main loop" of parallel work happening for parallel codegen. - // It's here that we manage parallelism, schedule work, and work with - // messages coming from clients. - // - // There are a few environmental pre-conditions that shape how the system - // is set up: - // - // - Error reporting only can happen on the main thread because that's the - // only place where we have access to the compiler `Session`. - // - LLVM work can be done on any thread. - // - Codegen can only happen on the main thread. - // - Each thread doing substantial work must be in possession of a `Token` - // from the `Jobserver`. - // - The compiler process always holds one `Token`. Any additional `Tokens` - // have to be requested from the `Jobserver`. - // - // Error Reporting - // =============== - // The error reporting restriction is handled separately from the rest: We - // set up a `SharedEmitter` the holds an open channel to the main thread. - // When an error occurs on any thread, the shared emitter will send the - // error message to the receiver main thread (`SharedEmitterMain`). The - // main thread will periodically query this error message queue and emit - // any error messages it has received. It might even abort compilation if - // has received a fatal error. In this case we rely on all other threads - // being torn down automatically with the main thread. - // Since the main thread will often be busy doing codegen work, error - // reporting will be somewhat delayed, since the message queue can only be - // checked in between to work packages. - // - // Work Processing Infrastructure - // ============================== - // The work processing infrastructure knows three major actors: - // - // - the coordinator thread, - // - the main thread, and - // - LLVM worker threads - // - // The coordinator thread is running a message loop. It instructs the main - // thread about what work to do when, and it will spawn off LLVM worker - // threads as open LLVM WorkItems become available. - // - // The job of the main thread is to codegen CGUs into LLVM work package - // (since the main thread is the only thread that can do this). The main - // thread will block until it receives a message from the coordinator, upon - // which it will codegen one CGU, send it to the coordinator and block - // again. This way the coordinator can control what the main thread is - // doing. - // - // The coordinator keeps a queue of LLVM WorkItems, and when a `Token` is - // available, it will spawn off a new LLVM worker thread and let it process - // that a WorkItem. When a LLVM worker thread is done with its WorkItem, - // it will just shut down, which also frees all resources associated with - // the given LLVM module, and sends a message to the coordinator that the - // has been completed. - // - // Work Scheduling - // =============== - // The scheduler's goal is to minimize the time it takes to complete all - // work there is, however, we also want to keep memory consumption low - // if possible. These two goals are at odds with each other: If memory - // consumption were not an issue, we could just let the main thread produce - // LLVM WorkItems at full speed, assuring maximal utilization of - // Tokens/LLVM worker threads. However, since codegen is usually faster - // than LLVM processing, the queue of LLVM WorkItems would fill up and each - // WorkItem potentially holds on to a substantial amount of memory. - // - // So the actual goal is to always produce just enough LLVM WorkItems as - // not to starve our LLVM worker threads. That means, once we have enough - // WorkItems in our queue, we can block the main thread, so it does not - // produce more until we need them. - // - // Doing LLVM Work on the Main Thread - // ---------------------------------- - // Since the main thread owns the compiler processes implicit `Token`, it is - // wasteful to keep it blocked without doing any work. Therefore, what we do - // in this case is: We spawn off an additional LLVM worker thread that helps - // reduce the queue. The work it is doing corresponds to the implicit - // `Token`. The coordinator will mark the main thread as being busy with - // LLVM work. (The actual work happens on another OS thread but we just care - // about `Tokens`, not actual threads). - // - // When any LLVM worker thread finishes while the main thread is marked as - // "busy with LLVM work", we can do a little switcheroo: We give the Token - // of the just finished thread to the LLVM worker thread that is working on - // behalf of the main thread's implicit Token, thus freeing up the main - // thread again. The coordinator can then again decide what the main thread - // should do. This allows the coordinator to make decisions at more points - // in time. - // - // Striking a Balance between Throughput and Memory Consumption - // ------------------------------------------------------------ - // Since our two goals, (1) use as many Tokens as possible and (2) keep - // memory consumption as low as possible, are in conflict with each other, - // we have to find a trade off between them. Right now, the goal is to keep - // all workers busy, which means that no worker should find the queue empty - // when it is ready to start. - // How do we do achieve this? Good question :) We actually never know how - // many `Tokens` are potentially available so it's hard to say how much to - // fill up the queue before switching the main thread to LLVM work. Also we - // currently don't have a means to estimate how long a running LLVM worker - // will still be busy with it's current WorkItem. However, we know the - // maximal count of available Tokens that makes sense (=the number of CPU - // cores), so we can take a conservative guess. The heuristic we use here - // is implemented in the `queue_full_enough()` function. - // - // Some Background on Jobservers - // ----------------------------- - // It's worth also touching on the management of parallelism here. We don't - // want to just spawn a thread per work item because while that's optimal - // parallelism it may overload a system with too many threads or violate our - // configuration for the maximum amount of cpu to use for this process. To - // manage this we use the `jobserver` crate. - // - // Job servers are an artifact of GNU make and are used to manage - // parallelism between processes. A jobserver is a glorified IPC semaphore - // basically. Whenever we want to run some work we acquire the semaphore, - // and whenever we're done with that work we release the semaphore. In this - // manner we can ensure that the maximum number of parallel workers is - // capped at any one point in time. - // - // LTO and the coordinator thread - // ------------------------------ - // - // The final job the coordinator thread is responsible for is managing LTO - // and how that works. When LTO is requested what we'll to is collect all - // optimized LLVM modules into a local vector on the coordinator. Once all - // modules have been codegened and optimized we hand this to the `lto` - // module for further optimization. The `lto` module will return back a list - // of more modules to work on, which the coordinator will continue to spawn - // work for. - // - // Each LLVM module is automatically sent back to the coordinator for LTO if - // necessary. There's already optimizations in place to avoid sending work - // back to the coordinator if LTO isn't requested. - return B::spawn_thread(cgcx.time_trace, move || { - let mut worker_id_counter = 0; - let mut free_worker_ids = Vec::new(); - let mut get_worker_id = |free_worker_ids: &mut Vec| { - if let Some(id) = free_worker_ids.pop() { - id - } else { - let id = worker_id_counter; - worker_id_counter += 1; - id - } - }; - - // This is where we collect codegen units that have gone all the way - // through codegen and LLVM. - let mut compiled_modules = vec![]; - let mut compiled_allocator_module = None; - let mut needs_link = Vec::new(); - let mut needs_fat_lto = Vec::new(); - let mut needs_thin_lto = Vec::new(); - let mut lto_import_only_modules = Vec::new(); - let mut started_lto = false; - let mut codegen_aborted = false; - - // This flag tracks whether all items have gone through codegens - let mut codegen_done = false; - - // This is the queue of LLVM work items that still need processing. - let mut work_items = Vec::<(WorkItem, u64)>::new(); - - // This are the Jobserver Tokens we currently hold. Does not include - // the implicit Token the compiler process owns no matter what. - let mut tokens = Vec::new(); - - let mut main_thread_worker_state = MainThreadWorkerState::Idle; - let mut running = 0; - - let prof = &cgcx.prof; - let mut llvm_start_time: Option> = None; - - // Run the message loop while there's still anything that needs message - // processing. Note that as soon as codegen is aborted we simply want to - // wait for all existing work to finish, so many of the conditions here - // only apply if codegen hasn't been aborted as they represent pending - // work to be done. - while !codegen_done - || running > 0 - || main_thread_worker_state == MainThreadWorkerState::LLVMing - || (!codegen_aborted - && !(work_items.is_empty() - && needs_fat_lto.is_empty() - && needs_thin_lto.is_empty() - && lto_import_only_modules.is_empty() - && main_thread_worker_state == MainThreadWorkerState::Idle)) - { - // While there are still CGUs to be codegened, the coordinator has - // to decide how to utilize the compiler processes implicit Token: - // For codegenning more CGU or for running them through LLVM. - if !codegen_done { - if main_thread_worker_state == MainThreadWorkerState::Idle { - // Compute the number of workers that will be running once we've taken as many - // items from the work queue as we can, plus one for the main thread. It's not - // critically important that we use this instead of just `running`, but it - // prevents the `queue_full_enough` heuristic from fluctuating just because a - // worker finished up and we decreased the `running` count, even though we're - // just going to increase it right after this when we put a new worker to work. - let extra_tokens = tokens.len().checked_sub(running).unwrap(); - let additional_running = std::cmp::min(extra_tokens, work_items.len()); - let anticipated_running = running + additional_running + 1; - - if !queue_full_enough(work_items.len(), anticipated_running) { - // The queue is not full enough, codegen more items: - if codegen_worker_send.send(Message::CodegenItem).is_err() { - panic!("Could not send Message::CodegenItem to main thread") - } - main_thread_worker_state = MainThreadWorkerState::Codegenning; - } else { - // The queue is full enough to not let the worker - // threads starve. Use the implicit Token to do some - // LLVM work too. - let (item, _) = - work_items.pop().expect("queue empty - queue_full_enough() broken?"); - let cgcx = CodegenContext { - worker: get_worker_id(&mut free_worker_ids), - ..cgcx.clone() - }; - maybe_start_llvm_timer( - prof, - cgcx.config(item.module_kind()), - &mut llvm_start_time, - ); - main_thread_worker_state = MainThreadWorkerState::LLVMing; - spawn_work(cgcx, item); - } - } - } else if codegen_aborted { - // don't queue up any more work if codegen was aborted, we're - // just waiting for our existing children to finish - } else { - // If we've finished everything related to normal codegen - // then it must be the case that we've got some LTO work to do. - // Perform the serial work here of figuring out what we're - // going to LTO and then push a bunch of work items onto our - // queue to do LTO - if work_items.is_empty() - && running == 0 - && main_thread_worker_state == MainThreadWorkerState::Idle - { - assert!(!started_lto); - started_lto = true; - - let needs_fat_lto = mem::take(&mut needs_fat_lto); - let needs_thin_lto = mem::take(&mut needs_thin_lto); - let import_only_modules = mem::take(&mut lto_import_only_modules); - - for (work, cost) in - generate_lto_work(&cgcx, needs_fat_lto, needs_thin_lto, import_only_modules) - { - let insertion_index = work_items - .binary_search_by_key(&cost, |&(_, cost)| cost) - .unwrap_or_else(|e| e); - work_items.insert(insertion_index, (work, cost)); - if !cgcx.opts.unstable_opts.no_parallel_llvm { - helper.request_token(); - } - } - } - - // In this branch, we know that everything has been codegened, - // so it's just a matter of determining whether the implicit - // Token is free to use for LLVM work. - match main_thread_worker_state { - MainThreadWorkerState::Idle => { - if let Some((item, _)) = work_items.pop() { - let cgcx = CodegenContext { - worker: get_worker_id(&mut free_worker_ids), - ..cgcx.clone() - }; - maybe_start_llvm_timer( - prof, - cgcx.config(item.module_kind()), - &mut llvm_start_time, - ); - main_thread_worker_state = MainThreadWorkerState::LLVMing; - spawn_work(cgcx, item); - } else { - // There is no unstarted work, so let the main thread - // take over for a running worker. Otherwise the - // implicit token would just go to waste. - // We reduce the `running` counter by one. The - // `tokens.truncate()` below will take care of - // giving the Token back. - debug_assert!(running > 0); - running -= 1; - main_thread_worker_state = MainThreadWorkerState::LLVMing; - } - } - MainThreadWorkerState::Codegenning => bug!( - "codegen worker should not be codegenning after \ - codegen was already completed" - ), - MainThreadWorkerState::LLVMing => { - // Already making good use of that token - } - } - } - - // Spin up what work we can, only doing this while we've got available - // parallelism slots and work left to spawn. - while !codegen_aborted && !work_items.is_empty() && running < tokens.len() { - let (item, _) = work_items.pop().unwrap(); - - maybe_start_llvm_timer(prof, cgcx.config(item.module_kind()), &mut llvm_start_time); - - let cgcx = - CodegenContext { worker: get_worker_id(&mut free_worker_ids), ..cgcx.clone() }; - - spawn_work(cgcx, item); - running += 1; - } - - // Relinquish accidentally acquired extra tokens - tokens.truncate(running); - - // If a thread exits successfully then we drop a token associated - // with that worker and update our `running` count. We may later - // re-acquire a token to continue running more work. We may also not - // actually drop a token here if the worker was running with an - // "ephemeral token" - let mut free_worker = |worker_id| { - if main_thread_worker_state == MainThreadWorkerState::LLVMing { - main_thread_worker_state = MainThreadWorkerState::Idle; - } else { - running -= 1; - } - - free_worker_ids.push(worker_id); - }; - - let msg = coordinator_receive.recv().unwrap(); - match *msg.downcast::>().ok().unwrap() { - // Save the token locally and the next turn of the loop will use - // this to spawn a new unit of work, or it may get dropped - // immediately if we have no more work to spawn. - Message::Token(token) => { - match token { - Ok(token) => { - tokens.push(token); - - if main_thread_worker_state == MainThreadWorkerState::LLVMing { - // If the main thread token is used for LLVM work - // at the moment, we turn that thread into a regular - // LLVM worker thread, so the main thread is free - // to react to codegen demand. - main_thread_worker_state = MainThreadWorkerState::Idle; - running += 1; - } - } - Err(e) => { - let msg = &format!("failed to acquire jobserver token: {}", e); - shared_emitter.fatal(msg); - // Exit the coordinator thread - panic!("{}", msg) - } - } - } - - Message::CodegenDone { llvm_work_item, cost } => { - // We keep the queue sorted by estimated processing cost, - // so that more expensive items are processed earlier. This - // is good for throughput as it gives the main thread more - // time to fill up the queue and it avoids scheduling - // expensive items to the end. - // Note, however, that this is not ideal for memory - // consumption, as LLVM module sizes are not evenly - // distributed. - let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost); - let insertion_index = match insertion_index { - Ok(idx) | Err(idx) => idx, - }; - work_items.insert(insertion_index, (llvm_work_item, cost)); - - if !cgcx.opts.unstable_opts.no_parallel_llvm { - helper.request_token(); - } - assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning); - main_thread_worker_state = MainThreadWorkerState::Idle; - } - - Message::CodegenComplete => { - codegen_done = true; - assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning); - main_thread_worker_state = MainThreadWorkerState::Idle; - } - - // If codegen is aborted that means translation was aborted due - // to some normal-ish compiler error. In this situation we want - // to exit as soon as possible, but we want to make sure all - // existing work has finished. Flag codegen as being done, and - // then conditions above will ensure no more work is spawned but - // we'll keep executing this loop until `running` hits 0. - Message::CodegenAborted => { - codegen_done = true; - codegen_aborted = true; - } - Message::Done { result: Ok(compiled_module), worker_id } => { - free_worker(worker_id); - match compiled_module.kind { - ModuleKind::Regular => { - compiled_modules.push(compiled_module); - } - ModuleKind::Allocator => { - assert!(compiled_allocator_module.is_none()); - compiled_allocator_module = Some(compiled_module); - } - ModuleKind::Metadata => bug!("Should be handled separately"), - } - } - Message::NeedsLink { module, worker_id } => { - free_worker(worker_id); - needs_link.push(module); - } - Message::NeedsFatLTO { result, worker_id } => { - assert!(!started_lto); - free_worker(worker_id); - needs_fat_lto.push(result); - } - Message::NeedsThinLTO { name, thin_buffer, worker_id } => { - assert!(!started_lto); - free_worker(worker_id); - needs_thin_lto.push((name, thin_buffer)); - } - Message::AddImportOnlyModule { module_data, work_product } => { - assert!(!started_lto); - assert!(!codegen_done); - assert_eq!(main_thread_worker_state, MainThreadWorkerState::Codegenning); - lto_import_only_modules.push((module_data, work_product)); - main_thread_worker_state = MainThreadWorkerState::Idle; - } - // If the thread failed that means it panicked, so we abort immediately. - Message::Done { result: Err(None), worker_id: _ } => { - bug!("worker thread panicked"); - } - Message::Done { result: Err(Some(WorkerFatalError)), worker_id } => { - // Similar to CodegenAborted, wait for remaining work to finish. - free_worker(worker_id); - codegen_done = true; - codegen_aborted = true; - } - Message::CodegenItem => bug!("the coordinator should not receive codegen requests"), - } - } - - if codegen_aborted { - return Err(()); - } - - let needs_link = mem::take(&mut needs_link); - if !needs_link.is_empty() { - assert!(compiled_modules.is_empty()); - let diag_handler = cgcx.create_diag_handler(); - let module = B::run_link(&cgcx, &diag_handler, needs_link).map_err(|_| ())?; - let module = unsafe { - B::codegen(&cgcx, &diag_handler, module, cgcx.config(ModuleKind::Regular)) - .map_err(|_| ())? - }; - compiled_modules.push(module); - } - - // Drop to print timings - drop(llvm_start_time); - - // Regardless of what order these modules completed in, report them to - // the backend in the same order every time to ensure that we're handing - // out deterministic results. - compiled_modules.sort_by(|a, b| a.name.cmp(&b.name)); - - Ok(CompiledModules { - modules: compiled_modules, - allocator_module: compiled_allocator_module, - }) - }); - - // A heuristic that determines if we have enough LLVM WorkItems in the - // queue so that the main thread can do LLVM work instead of codegen - fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool { - // This heuristic scales ahead-of-time codegen according to available - // concurrency, as measured by `workers_running`. The idea is that the - // more concurrency we have available, the more demand there will be for - // work items, and the fuller the queue should be kept to meet demand. - // An important property of this approach is that we codegen ahead of - // time only as much as necessary, so as to keep fewer LLVM modules in - // memory at once, thereby reducing memory consumption. - // - // When the number of workers running is less than the max concurrency - // available to us, this heuristic can cause us to instruct the main - // thread to work on an LLVM item (that is, tell it to "LLVM") instead - // of codegen, even though it seems like it *should* be codegenning so - // that we can create more work items and spawn more LLVM workers. - // - // But this is not a problem. When the main thread is told to LLVM, - // according to this heuristic and how work is scheduled, there is - // always at least one item in the queue, and therefore at least one - // pending jobserver token request. If there *is* more concurrency - // available, we will immediately receive a token, which will upgrade - // the main thread's LLVM worker to a real one (conceptually), and free - // up the main thread to codegen if necessary. On the other hand, if - // there isn't more concurrency, then the main thread working on an LLVM - // item is appropriate, as long as the queue is full enough for demand. - // - // Speaking of which, how full should we keep the queue? Probably less - // full than you'd think. A lot has to go wrong for the queue not to be - // full enough and for that to have a negative effect on compile times. - // - // Workers are unlikely to finish at exactly the same time, so when one - // finishes and takes another work item off the queue, we often have - // ample time to codegen at that point before the next worker finishes. - // But suppose that codegen takes so long that the workers exhaust the - // queue, and we have one or more workers that have nothing to work on. - // Well, it might not be so bad. Of all the LLVM modules we create and - // optimize, one has to finish last. It's not necessarily the case that - // by losing some concurrency for a moment, we delay the point at which - // that last LLVM module is finished and the rest of compilation can - // proceed. Also, when we can't take advantage of some concurrency, we - // give tokens back to the job server. That enables some other rustc to - // potentially make use of the available concurrency. That could even - // *decrease* overall compile time if we're lucky. But yes, if no other - // rustc can make use of the concurrency, then we've squandered it. - // - // However, keeping the queue full is also beneficial when we have a - // surge in available concurrency. Then items can be taken from the - // queue immediately, without having to wait for codegen. - // - // So, the heuristic below tries to keep one item in the queue for every - // four running workers. Based on limited benchmarking, this appears to - // be more than sufficient to avoid increasing compilation times. - let quarter_of_workers = workers_running - 3 * workers_running / 4; - items_in_queue > 0 && items_in_queue >= quarter_of_workers - } - - fn maybe_start_llvm_timer<'a>( - prof: &'a SelfProfilerRef, - config: &ModuleConfig, - llvm_start_time: &mut Option>, - ) { - if config.time_module && llvm_start_time.is_none() { - *llvm_start_time = Some(prof.extra_verbose_generic_activity("LLVM_passes", "crate")); - } - } -} - -/// `FatalError` is explicitly not `Send`. -#[must_use] -pub struct WorkerFatalError; - -fn spawn_work(cgcx: CodegenContext, work: WorkItem) { - B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || { - // Set up a destructor which will fire off a message that we're done as - // we exit. - struct Bomb { - coordinator_send: Sender>, - result: Option, FatalError>>, - worker_id: usize, - } - impl Drop for Bomb { - fn drop(&mut self) { - let worker_id = self.worker_id; - let msg = match self.result.take() { - Some(Ok(WorkItemResult::Compiled(m))) => { - Message::Done:: { result: Ok(m), worker_id } - } - Some(Ok(WorkItemResult::NeedsLink(m))) => { - Message::NeedsLink:: { module: m, worker_id } - } - Some(Ok(WorkItemResult::NeedsFatLTO(m))) => { - Message::NeedsFatLTO:: { result: m, worker_id } - } - Some(Ok(WorkItemResult::NeedsThinLTO(name, thin_buffer))) => { - Message::NeedsThinLTO:: { name, thin_buffer, worker_id } - } - Some(Err(FatalError)) => { - Message::Done:: { result: Err(Some(WorkerFatalError)), worker_id } - } - None => Message::Done:: { result: Err(None), worker_id }, - }; - drop(self.coordinator_send.send(Box::new(msg))); - } - } - - let mut bomb = Bomb:: { - coordinator_send: cgcx.coordinator_send.clone(), - result: None, - worker_id: cgcx.worker, - }; - - // Execute the work itself, and if it finishes successfully then flag - // ourselves as a success as well. - // - // Note that we ignore any `FatalError` coming out of `execute_work_item`, - // as a diagnostic was already sent off to the main thread - just - // surface that there was an error in this worker. - bomb.result = { - let _prof_timer = work.start_profiling(&cgcx); - Some(execute_work_item(&cgcx, work)) - }; - }) - .expect("failed to spawn thread"); -} - -enum SharedEmitterMessage { - Diagnostic(Diagnostic), - InlineAsmError(u32, String, Level, Option<(String, Vec)>), - AbortIfErrors, - Fatal(String), -} - -#[derive(Clone)] -pub struct SharedEmitter { - sender: Sender, -} - -pub struct SharedEmitterMain { - receiver: Receiver, -} - -impl SharedEmitter { - pub fn new() -> (SharedEmitter, SharedEmitterMain) { - let (sender, receiver) = channel(); - - (SharedEmitter { sender }, SharedEmitterMain { receiver }) - } - - pub fn inline_asm_error( - &self, - cookie: u32, - msg: String, - level: Level, - source: Option<(String, Vec)>, - ) { - drop(self.sender.send(SharedEmitterMessage::InlineAsmError(cookie, msg, level, source))); - } - - pub fn fatal(&self, msg: &str) { - drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string()))); - } -} - -impl Emitter for SharedEmitter { - fn emit_diagnostic(&mut self, diag: &rustc_errors::Diagnostic) { - let fluent_args = self.to_fluent_args(diag.args()); - drop(self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic { - msg: self.translate_messages(&diag.message, &fluent_args).to_string(), - code: diag.code.clone(), - lvl: diag.level(), - }))); - for child in &diag.children { - drop(self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic { - msg: self.translate_messages(&child.message, &fluent_args).to_string(), - code: None, - lvl: child.level, - }))); - } - drop(self.sender.send(SharedEmitterMessage::AbortIfErrors)); - } - - fn source_map(&self) -> Option<&Lrc> { - None - } - - fn fluent_bundle(&self) -> Option<&Lrc> { - None - } - - fn fallback_fluent_bundle(&self) -> &rustc_errors::FluentBundle { - panic!("shared emitter attempted to translate a diagnostic"); - } -} - -impl SharedEmitterMain { - pub fn check(&self, sess: &Session, blocking: bool) { - loop { - let message = if blocking { - match self.receiver.recv() { - Ok(message) => Ok(message), - Err(_) => Err(()), - } - } else { - match self.receiver.try_recv() { - Ok(message) => Ok(message), - Err(_) => Err(()), - } - }; - - match message { - Ok(SharedEmitterMessage::Diagnostic(diag)) => { - let handler = sess.diagnostic(); - let mut d = rustc_errors::Diagnostic::new(diag.lvl, &diag.msg); - if let Some(code) = diag.code { - d.code(code); - } - handler.emit_diagnostic(&mut d); - } - Ok(SharedEmitterMessage::InlineAsmError(cookie, msg, level, source)) => { - let msg = msg.strip_prefix("error: ").unwrap_or(&msg); - - let mut err = match level { - Level::Error { lint: false } => sess.struct_err(msg).forget_guarantee(), - Level::Warning(_) => sess.struct_warn(msg), - Level::Note => sess.struct_note_without_error(msg), - _ => bug!("Invalid inline asm diagnostic level"), - }; - - // If the cookie is 0 then we don't have span information. - if cookie != 0 { - let pos = BytePos::from_u32(cookie); - let span = Span::with_root_ctxt(pos, pos); - err.set_span(span); - }; - - // Point to the generated assembly if it is available. - if let Some((buffer, spans)) = source { - let source = sess - .source_map() - .new_source_file(FileName::inline_asm_source_code(&buffer), buffer); - let source_span = Span::with_root_ctxt(source.start_pos, source.end_pos); - let spans: Vec<_> = - spans.iter().map(|sp| source_span.from_inner(*sp)).collect(); - err.span_note(spans, "instantiated into assembly here"); - } - - err.emit(); - } - Ok(SharedEmitterMessage::AbortIfErrors) => { - sess.abort_if_errors(); - } - Ok(SharedEmitterMessage::Fatal(msg)) => { - sess.fatal(&msg); - } - Err(_) => { - break; - } - } - } - } -} - -pub struct Coordinator { - pub sender: Sender>, - future: Option>>, - // Only used for the Message type. - phantom: PhantomData, -} - -impl Coordinator { - fn join(mut self) -> std::thread::Result> { - self.future.take().unwrap().join() - } -} - -impl Drop for Coordinator { - fn drop(&mut self) { - if let Some(future) = self.future.take() { - // If we haven't joined yet, signal to the coordinator that it should spawn no more - // work, and wait for worker threads to finish. - drop(self.sender.send(Box::new(Message::CodegenAborted::))); - drop(future.join()); - } - } -} - -pub struct OngoingCodegen { - pub backend: B, - pub metadata: EncodedMetadata, - pub metadata_module: Option, - pub crate_info: CrateInfo, - pub codegen_worker_receive: Receiver>, - pub shared_emitter_main: SharedEmitterMain, - pub output_filenames: Arc, - pub coordinator: Coordinator, -} - -impl OngoingCodegen { - pub fn join(self, sess: &Session) -> (CodegenResults, FxHashMap) { - let _timer = sess.timer("finish_ongoing_codegen"); - - self.shared_emitter_main.check(sess, true); - let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() { - Ok(Ok(compiled_modules)) => compiled_modules, - Ok(Err(())) => { - sess.abort_if_errors(); - panic!("expected abort due to worker thread errors") - } - Err(_) => { - bug!("panic during codegen/LLVM phase"); - } - }); - - sess.cgu_reuse_tracker.check_expected_reuse(sess.diagnostic()); - - sess.abort_if_errors(); - - let work_products = - copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules); - produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames); - - // FIXME: time_llvm_passes support - does this use a global context or - // something? - if sess.codegen_units() == 1 && sess.time_llvm_passes() { - self.backend.print_pass_timings() - } - - ( - CodegenResults { - metadata: self.metadata, - crate_info: self.crate_info, - - modules: compiled_modules.modules, - allocator_module: compiled_modules.allocator_module, - metadata_module: self.metadata_module, - }, - work_products, - ) - } - - pub fn submit_pre_codegened_module_to_llvm( - &self, - tcx: TyCtxt<'_>, - module: ModuleCodegen, - ) { - self.wait_for_signal_to_codegen_item(); - self.check_for_errors(tcx.sess); - - // These are generally cheap and won't throw off scheduling. - let cost = 0; - submit_codegened_module_to_llvm(&self.backend, &self.coordinator.sender, module, cost); - } - - pub fn codegen_finished(&self, tcx: TyCtxt<'_>) { - self.wait_for_signal_to_codegen_item(); - self.check_for_errors(tcx.sess); - drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::))); - } - - pub fn check_for_errors(&self, sess: &Session) { - self.shared_emitter_main.check(sess, false); - } - - pub fn wait_for_signal_to_codegen_item(&self) { - match self.codegen_worker_receive.recv() { - Ok(Message::CodegenItem) => { - // Nothing to do - } - Ok(_) => panic!("unexpected message"), - Err(_) => { - // One of the LLVM threads must have panicked, fall through so - // error handling can be reached. - } - } - } -} - -pub fn submit_codegened_module_to_llvm( - _backend: &B, - tx_to_llvm_workers: &Sender>, - module: ModuleCodegen, - cost: u64, -) { - let llvm_work_item = WorkItem::Optimize(module); - drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone:: { llvm_work_item, cost }))); -} - -pub fn submit_post_lto_module_to_llvm( - _backend: &B, - tx_to_llvm_workers: &Sender>, - module: CachedModuleCodegen, -) { - let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module); - drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone:: { llvm_work_item, cost: 0 }))); -} - -pub fn submit_pre_lto_module_to_llvm( - _backend: &B, - tcx: TyCtxt<'_>, - tx_to_llvm_workers: &Sender>, - module: CachedModuleCodegen, -) { - let filename = pre_lto_bitcode_filename(&module.name); - let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename); - let file = fs::File::open(&bc_path) - .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e)); - - let mmap = unsafe { - Mmap::map(file).unwrap_or_else(|e| { - panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e) - }) - }; - // Schedule the module to be loaded - drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule:: { - module_data: SerializedModule::FromUncompressedFile(mmap), - work_product: module.source, - }))); -} - -pub fn pre_lto_bitcode_filename(module_name: &str) -> String { - format!("{}.{}", module_name, PRE_LTO_BC_EXT) -} - -fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool { - // This should never be true (because it's not supported). If it is true, - // something is wrong with commandline arg validation. - assert!( - !(tcx.sess.opts.cg.linker_plugin_lto.enabled() - && tcx.sess.target.is_like_windows - && tcx.sess.opts.cg.prefer_dynamic) - ); - - tcx.sess.target.is_like_windows && - tcx.sess.crate_types().iter().any(|ct| *ct == CrateType::Rlib) && - // ThinLTO can't handle this workaround in all cases, so we don't - // emit the `__imp_` symbols. Instead we make them unnecessary by disallowing - // dynamic linking when linker plugin LTO is enabled. - !tcx.sess.opts.cg.linker_plugin_lto.enabled() -} diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs deleted file mode 100644 index a840b270974..00000000000 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ /dev/null @@ -1,961 +0,0 @@ -use crate::back::metadata::create_compressed_metadata_file; -use crate::back::write::{ - compute_per_cgu_lto_type, start_async_codegen, submit_codegened_module_to_llvm, - submit_post_lto_module_to_llvm, submit_pre_lto_module_to_llvm, ComputedLtoType, OngoingCodegen, -}; -use crate::common::{IntPredicate, RealPredicate, TypeKind}; -use crate::meth; -use crate::mir; -use crate::mir::operand::OperandValue; -use crate::mir::place::PlaceRef; -use crate::traits::*; -use crate::{CachedModuleCodegen, CompiledModule, CrateInfo, MemFlags, ModuleCodegen, ModuleKind}; - -use rustc_attr as attr; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry}; - -use rustc_data_structures::sync::par_iter; -#[cfg(parallel_compiler)] -use rustc_data_structures::sync::ParallelIterator; -use rustc_hir as hir; -use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_hir::lang_items::LangItem; -use rustc_index::vec::Idx; -use rustc_metadata::EncodedMetadata; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; -use rustc_middle::middle::exported_symbols; -use rustc_middle::middle::lang_items; -use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem}; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; -use rustc_middle::ty::query::Providers; -use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; -use rustc_session::cgu_reuse_tracker::CguReuse; -use rustc_session::config::{self, CrateType, EntryFnType, OutputType}; -use rustc_session::Session; -use rustc_span::symbol::sym; -use rustc_span::{DebuggerVisualizerFile, DebuggerVisualizerType}; -use rustc_target::abi::{Align, VariantIdx}; - -use std::collections::BTreeSet; -use std::convert::TryFrom; -use std::time::{Duration, Instant}; - -use itertools::Itertools; - -pub fn bin_op_to_icmp_predicate(op: hir::BinOpKind, signed: bool) -> IntPredicate { - match op { - hir::BinOpKind::Eq => IntPredicate::IntEQ, - hir::BinOpKind::Ne => IntPredicate::IntNE, - hir::BinOpKind::Lt => { - if signed { - IntPredicate::IntSLT - } else { - IntPredicate::IntULT - } - } - hir::BinOpKind::Le => { - if signed { - IntPredicate::IntSLE - } else { - IntPredicate::IntULE - } - } - hir::BinOpKind::Gt => { - if signed { - IntPredicate::IntSGT - } else { - IntPredicate::IntUGT - } - } - hir::BinOpKind::Ge => { - if signed { - IntPredicate::IntSGE - } else { - IntPredicate::IntUGE - } - } - op => bug!( - "comparison_op_to_icmp_predicate: expected comparison operator, \ - found {:?}", - op - ), - } -} - -pub fn bin_op_to_fcmp_predicate(op: hir::BinOpKind) -> RealPredicate { - match op { - hir::BinOpKind::Eq => RealPredicate::RealOEQ, - hir::BinOpKind::Ne => RealPredicate::RealUNE, - hir::BinOpKind::Lt => RealPredicate::RealOLT, - hir::BinOpKind::Le => RealPredicate::RealOLE, - hir::BinOpKind::Gt => RealPredicate::RealOGT, - hir::BinOpKind::Ge => RealPredicate::RealOGE, - op => { - bug!( - "comparison_op_to_fcmp_predicate: expected comparison operator, \ - found {:?}", - op - ); - } - } -} - -pub fn compare_simd_types<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - lhs: Bx::Value, - rhs: Bx::Value, - t: Ty<'tcx>, - ret_ty: Bx::Type, - op: hir::BinOpKind, -) -> Bx::Value { - let signed = match t.kind() { - ty::Float(_) => { - let cmp = bin_op_to_fcmp_predicate(op); - let cmp = bx.fcmp(cmp, lhs, rhs); - return bx.sext(cmp, ret_ty); - } - ty::Uint(_) => false, - ty::Int(_) => true, - _ => bug!("compare_simd_types: invalid SIMD type"), - }; - - let cmp = bin_op_to_icmp_predicate(op, signed); - let cmp = bx.icmp(cmp, lhs, rhs); - // LLVM outputs an `< size x i1 >`, so we need to perform a sign extension - // to get the correctly sized type. This will compile to a single instruction - // once the IR is converted to assembly if the SIMD instruction is supported - // by the target architecture. - bx.sext(cmp, ret_ty) -} - -/// Retrieves the information we are losing (making dynamic) in an unsizing -/// adjustment. -/// -/// The `old_info` argument is a bit odd. It is intended for use in an upcast, -/// where the new vtable for an object will be derived from the old one. -pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - source: Ty<'tcx>, - target: Ty<'tcx>, - old_info: Option, -) -> Bx::Value { - let cx = bx.cx(); - let (source, target) = - cx.tcx().struct_lockstep_tails_erasing_lifetimes(source, target, bx.param_env()); - match (source.kind(), target.kind()) { - (&ty::Array(_, len), &ty::Slice(_)) => { - cx.const_usize(len.eval_usize(cx.tcx(), ty::ParamEnv::reveal_all())) - } - (&ty::Dynamic(ref data_a, ..), &ty::Dynamic(ref data_b, ..)) => { - let old_info = - old_info.expect("unsized_info: missing old info for trait upcasting coercion"); - if data_a.principal_def_id() == data_b.principal_def_id() { - return old_info; - } - - // trait upcasting coercion - - let vptr_entry_idx = - cx.tcx().vtable_trait_upcasting_coercion_new_vptr_slot((source, target)); - - if let Some(entry_idx) = vptr_entry_idx { - let ptr_ty = cx.type_i8p(); - let ptr_align = cx.tcx().data_layout.pointer_align.abi; - let llvtable = bx.pointercast(old_info, bx.type_ptr_to(ptr_ty)); - let gep = bx.inbounds_gep( - ptr_ty, - llvtable, - &[bx.const_usize(u64::try_from(entry_idx).unwrap())], - ); - let new_vptr = bx.load(ptr_ty, gep, ptr_align); - bx.nonnull_metadata(new_vptr); - // VTable loads are invariant. - bx.set_invariant_load(new_vptr); - new_vptr - } else { - old_info - } - } - (_, &ty::Dynamic(ref data, ..)) => { - let vtable_ptr_ty = cx.scalar_pair_element_backend_type( - cx.layout_of(cx.tcx().mk_mut_ptr(target)), - 1, - true, - ); - cx.const_ptrcast(meth::get_vtable(cx, source, data.principal()), vtable_ptr_ty) - } - _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target), - } -} - -/// Coerces `src` to `dst_ty`. `src_ty` must be a pointer. -pub fn unsize_ptr<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - src: Bx::Value, - src_ty: Ty<'tcx>, - dst_ty: Ty<'tcx>, - old_info: Option, -) -> (Bx::Value, Bx::Value) { - debug!("unsize_ptr: {:?} => {:?}", src_ty, dst_ty); - match (src_ty.kind(), dst_ty.kind()) { - (&ty::Ref(_, a, _), &ty::Ref(_, b, _) | &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) - | (&ty::RawPtr(ty::TypeAndMut { ty: a, .. }), &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) => { - assert_eq!(bx.cx().type_is_sized(a), old_info.is_none()); - let ptr_ty = bx.cx().type_ptr_to(bx.cx().backend_type(bx.cx().layout_of(b))); - (bx.pointercast(src, ptr_ty), unsized_info(bx, a, b, old_info)) - } - (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { - assert_eq!(def_a, def_b); - let src_layout = bx.cx().layout_of(src_ty); - let dst_layout = bx.cx().layout_of(dst_ty); - if src_ty == dst_ty { - return (src, old_info.unwrap()); - } - let mut result = None; - for i in 0..src_layout.fields.count() { - let src_f = src_layout.field(bx.cx(), i); - if src_f.is_zst() { - continue; - } - - assert_eq!(src_layout.fields.offset(i).bytes(), 0); - assert_eq!(dst_layout.fields.offset(i).bytes(), 0); - assert_eq!(src_layout.size, src_f.size); - - let dst_f = dst_layout.field(bx.cx(), i); - assert_ne!(src_f.ty, dst_f.ty); - assert_eq!(result, None); - result = Some(unsize_ptr(bx, src, src_f.ty, dst_f.ty, old_info)); - } - let (lldata, llextra) = result.unwrap(); - let lldata_ty = bx.cx().scalar_pair_element_backend_type(dst_layout, 0, true); - let llextra_ty = bx.cx().scalar_pair_element_backend_type(dst_layout, 1, true); - // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. - (bx.bitcast(lldata, lldata_ty), bx.bitcast(llextra, llextra_ty)) - } - _ => bug!("unsize_ptr: called on bad types"), - } -} - -/// Coerces `src`, which is a reference to a value of type `src_ty`, -/// to a value of type `dst_ty`, and stores the result in `dst`. -pub fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - src: PlaceRef<'tcx, Bx::Value>, - dst: PlaceRef<'tcx, Bx::Value>, -) { - let src_ty = src.layout.ty; - let dst_ty = dst.layout.ty; - match (src_ty.kind(), dst_ty.kind()) { - (&ty::Ref(..), &ty::Ref(..) | &ty::RawPtr(..)) | (&ty::RawPtr(..), &ty::RawPtr(..)) => { - let (base, info) = match bx.load_operand(src).val { - OperandValue::Pair(base, info) => unsize_ptr(bx, base, src_ty, dst_ty, Some(info)), - OperandValue::Immediate(base) => unsize_ptr(bx, base, src_ty, dst_ty, None), - OperandValue::Ref(..) => bug!(), - }; - OperandValue::Pair(base, info).store(bx, dst); - } - - (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { - assert_eq!(def_a, def_b); - - for i in 0..def_a.variant(VariantIdx::new(0)).fields.len() { - let src_f = src.project_field(bx, i); - let dst_f = dst.project_field(bx, i); - - if dst_f.layout.is_zst() { - continue; - } - - if src_f.layout.ty == dst_f.layout.ty { - memcpy_ty( - bx, - dst_f.llval, - dst_f.align, - src_f.llval, - src_f.align, - src_f.layout, - MemFlags::empty(), - ); - } else { - coerce_unsized_into(bx, src_f, dst_f); - } - } - } - _ => bug!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty, dst_ty,), - } -} - -pub fn cast_shift_expr_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - op: hir::BinOpKind, - lhs: Bx::Value, - rhs: Bx::Value, -) -> Bx::Value { - cast_shift_rhs(bx, op, lhs, rhs) -} - -fn cast_shift_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - op: hir::BinOpKind, - lhs: Bx::Value, - rhs: Bx::Value, -) -> Bx::Value { - // Shifts may have any size int on the rhs - if op.is_shift() { - let mut rhs_llty = bx.cx().val_ty(rhs); - let mut lhs_llty = bx.cx().val_ty(lhs); - if bx.cx().type_kind(rhs_llty) == TypeKind::Vector { - rhs_llty = bx.cx().element_type(rhs_llty) - } - if bx.cx().type_kind(lhs_llty) == TypeKind::Vector { - lhs_llty = bx.cx().element_type(lhs_llty) - } - let rhs_sz = bx.cx().int_width(rhs_llty); - let lhs_sz = bx.cx().int_width(lhs_llty); - if lhs_sz < rhs_sz { - bx.trunc(rhs, lhs_llty) - } else if lhs_sz > rhs_sz { - // FIXME (#1877: If in the future shifting by negative - // values is no longer undefined then this is wrong. - bx.zext(rhs, lhs_llty) - } else { - rhs - } - } else { - rhs - } -} - -/// Returns `true` if this session's target will use SEH-based unwinding. -/// -/// This is only true for MSVC targets, and even then the 64-bit MSVC target -/// currently uses SEH-ish unwinding with DWARF info tables to the side (same as -/// 64-bit MinGW) instead of "full SEH". -pub fn wants_msvc_seh(sess: &Session) -> bool { - sess.target.is_like_msvc -} - -pub fn memcpy_ty<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - dst: Bx::Value, - dst_align: Align, - src: Bx::Value, - src_align: Align, - layout: TyAndLayout<'tcx>, - flags: MemFlags, -) { - let size = layout.size.bytes(); - if size == 0 { - return; - } - - bx.memcpy(dst, dst_align, src, src_align, bx.cx().const_usize(size), flags); -} - -pub fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>( - cx: &'a Bx::CodegenCx, - instance: Instance<'tcx>, -) { - // this is an info! to allow collecting monomorphization statistics - // and to allow finding the last function before LLVM aborts from - // release builds. - info!("codegen_instance({})", instance); - - mir::codegen_mir::(cx, instance); -} - -/// Creates the `main` function which will initialize the rust runtime and call -/// users main function. -pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - cx: &'a Bx::CodegenCx, -) -> Option { - let (main_def_id, entry_type) = cx.tcx().entry_fn(())?; - let main_is_local = main_def_id.is_local(); - let instance = Instance::mono(cx.tcx(), main_def_id); - - if main_is_local { - // We want to create the wrapper in the same codegen unit as Rust's main - // function. - if !cx.codegen_unit().contains_item(&MonoItem::Fn(instance)) { - return None; - } - } else if !cx.codegen_unit().is_primary() { - // We want to create the wrapper only when the codegen unit is the primary one - return None; - } - - let main_llfn = cx.get_fn_addr(instance); - - let use_start_lang_item = EntryFnType::Start != entry_type; - let entry_fn = create_entry_fn::(cx, main_llfn, main_def_id, use_start_lang_item); - return Some(entry_fn); - - fn create_entry_fn<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - cx: &'a Bx::CodegenCx, - rust_main: Bx::Value, - rust_main_def_id: DefId, - use_start_lang_item: bool, - ) -> Bx::Function { - // The entry function is either `int main(void)` or `int main(int argc, char **argv)`, - // depending on whether the target needs `argc` and `argv` to be passed in. - let llfty = if cx.sess().target.main_needs_argc_argv { - cx.type_func(&[cx.type_int(), cx.type_ptr_to(cx.type_i8p())], cx.type_int()) - } else { - cx.type_func(&[], cx.type_int()) - }; - - let main_ret_ty = cx.tcx().fn_sig(rust_main_def_id).output(); - // Given that `main()` has no arguments, - // then its return type cannot have - // late-bound regions, since late-bound - // regions must appear in the argument - // listing. - let main_ret_ty = cx.tcx().normalize_erasing_regions( - ty::ParamEnv::reveal_all(), - main_ret_ty.no_bound_vars().unwrap(), - ); - - let Some(llfn) = cx.declare_c_main(llfty) else { - // FIXME: We should be smart and show a better diagnostic here. - let span = cx.tcx().def_span(rust_main_def_id); - cx.sess() - .struct_span_err(span, "entry symbol `main` declared multiple times") - .help("did you use `#[no_mangle]` on `fn main`? Use `#[start]` instead") - .emit(); - cx.sess().abort_if_errors(); - bug!(); - }; - - // `main` should respect same config for frame pointer elimination as rest of code - cx.set_frame_pointer_type(llfn); - cx.apply_target_cpu_attr(llfn); - - let llbb = Bx::append_block(&cx, llfn, "top"); - let mut bx = Bx::build(&cx, llbb); - - bx.insert_reference_to_gdb_debug_scripts_section_global(); - - let isize_ty = cx.type_isize(); - let i8pp_ty = cx.type_ptr_to(cx.type_i8p()); - let (arg_argc, arg_argv) = get_argc_argv(cx, &mut bx); - - let (start_fn, start_ty, args) = if use_start_lang_item { - let start_def_id = cx.tcx().require_lang_item(LangItem::Start, None); - let start_fn = cx.get_fn_addr( - ty::Instance::resolve( - cx.tcx(), - ty::ParamEnv::reveal_all(), - start_def_id, - cx.tcx().intern_substs(&[main_ret_ty.into()]), - ) - .unwrap() - .unwrap(), - ); - let start_ty = cx.type_func(&[cx.val_ty(rust_main), isize_ty, i8pp_ty], isize_ty); - (start_fn, start_ty, vec![rust_main, arg_argc, arg_argv]) - } else { - debug!("using user-defined start fn"); - let start_ty = cx.type_func(&[isize_ty, i8pp_ty], isize_ty); - (rust_main, start_ty, vec![arg_argc, arg_argv]) - }; - - let result = bx.call(start_ty, start_fn, &args, None); - let cast = bx.intcast(result, cx.type_int(), true); - bx.ret(cast); - - llfn - } -} - -/// Obtain the `argc` and `argv` values to pass to the rust start function. -fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - cx: &'a Bx::CodegenCx, - bx: &mut Bx, -) -> (Bx::Value, Bx::Value) { - if cx.sess().target.main_needs_argc_argv { - // Params from native `main()` used as args for rust start function - let param_argc = bx.get_param(0); - let param_argv = bx.get_param(1); - let arg_argc = bx.intcast(param_argc, cx.type_isize(), true); - let arg_argv = param_argv; - (arg_argc, arg_argv) - } else { - // The Rust start function doesn't need `argc` and `argv`, so just pass zeros. - let arg_argc = bx.const_int(cx.type_int(), 0); - let arg_argv = bx.const_null(cx.type_ptr_to(cx.type_i8p())); - (arg_argc, arg_argv) - } -} - -/// This function returns all of the debugger visualizers specified for the -/// current crate as well as all upstream crates transitively that match the -/// `visualizer_type` specified. -pub fn collect_debugger_visualizers_transitive( - tcx: TyCtxt<'_>, - visualizer_type: DebuggerVisualizerType, -) -> BTreeSet { - tcx.debugger_visualizers(LOCAL_CRATE) - .iter() - .chain( - tcx.crates(()) - .iter() - .filter(|&cnum| { - let used_crate_source = tcx.used_crate_source(*cnum); - used_crate_source.rlib.is_some() || used_crate_source.rmeta.is_some() - }) - .flat_map(|&cnum| tcx.debugger_visualizers(cnum)), - ) - .filter(|visualizer| visualizer.visualizer_type == visualizer_type) - .cloned() - .collect::>() -} - -pub fn codegen_crate( - backend: B, - tcx: TyCtxt<'_>, - target_cpu: String, - metadata: EncodedMetadata, - need_metadata_module: bool, -) -> OngoingCodegen { - // Skip crate items and just output metadata in -Z no-codegen mode. - if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, metadata, None, 1); - - ongoing_codegen.codegen_finished(tcx); - - ongoing_codegen.check_for_errors(tcx.sess); - - return ongoing_codegen; - } - - let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx); - - // Run the monomorphization collector and partition the collected items into - // codegen units. - let codegen_units = tcx.collect_and_partition_mono_items(()).1; - - // Force all codegen_unit queries so they are already either red or green - // when compile_codegen_unit accesses them. We are not able to re-execute - // the codegen_unit query from just the DepNode, so an unknown color would - // lead to having to re-execute compile_codegen_unit, possibly - // unnecessarily. - if tcx.dep_graph.is_fully_enabled() { - for cgu in codegen_units { - tcx.ensure().codegen_unit(cgu.name()); - } - } - - let metadata_module = if need_metadata_module { - // Emit compressed metadata object. - let metadata_cgu_name = - cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("metadata")).to_string(); - tcx.sess.time("write_compressed_metadata", || { - let file_name = - tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); - let data = create_compressed_metadata_file( - tcx.sess, - &metadata, - &exported_symbols::metadata_symbol_name(tcx), - ); - if let Err(err) = std::fs::write(&file_name, data) { - tcx.sess.fatal(&format!("error writing metadata object file: {}", err)); - } - Some(CompiledModule { - name: metadata_cgu_name, - kind: ModuleKind::Metadata, - object: Some(file_name), - dwarf_object: None, - bytecode: None, - }) - }) - } else { - None - }; - - let ongoing_codegen = start_async_codegen( - backend.clone(), - tcx, - target_cpu, - metadata, - metadata_module, - codegen_units.len(), - ); - - // Codegen an allocator shim, if necessary. - // - // If the crate doesn't have an `allocator_kind` set then there's definitely - // no shim to generate. Otherwise we also check our dependency graph for all - // our output crate types. If anything there looks like its a `Dynamic` - // linkage, then it's already got an allocator shim and we'll be using that - // one instead. If nothing exists then it's our job to generate the - // allocator! - let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| { - use rustc_middle::middle::dependency_format::Linkage; - list.iter().any(|&linkage| linkage == Linkage::Dynamic) - }); - let allocator_module = if any_dynamic_crate { - None - } else if let Some(kind) = tcx.allocator_kind(()) { - let llmod_id = - cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("allocator")).to_string(); - let module_llvm = tcx.sess.time("write_allocator_module", || { - backend.codegen_allocator(tcx, &llmod_id, kind, tcx.lang_items().oom().is_some()) - }); - - Some(ModuleCodegen { name: llmod_id, module_llvm, kind: ModuleKind::Allocator }) - } else { - None - }; - - if let Some(allocator_module) = allocator_module { - ongoing_codegen.submit_pre_codegened_module_to_llvm(tcx, allocator_module); - } - - // For better throughput during parallel processing by LLVM, we used to sort - // CGUs largest to smallest. This would lead to better thread utilization - // by, for example, preventing a large CGU from being processed last and - // having only one LLVM thread working while the rest remained idle. - // - // However, this strategy would lead to high memory usage, as it meant the - // LLVM-IR for all of the largest CGUs would be resident in memory at once. - // - // Instead, we can compromise by ordering CGUs such that the largest and - // smallest are first, second largest and smallest are next, etc. If there - // are large size variations, this can reduce memory usage significantly. - let codegen_units: Vec<_> = { - let mut sorted_cgus = codegen_units.iter().collect::>(); - sorted_cgus.sort_by_cached_key(|cgu| cgu.size_estimate()); - - let (first_half, second_half) = sorted_cgus.split_at(sorted_cgus.len() / 2); - second_half.iter().rev().interleave(first_half).copied().collect() - }; - - // Calculate the CGU reuse - let cgu_reuse = tcx.sess.time("find_cgu_reuse", || { - codegen_units.iter().map(|cgu| determine_cgu_reuse(tcx, &cgu)).collect::>() - }); - - let mut total_codegen_time = Duration::new(0, 0); - let start_rss = tcx.sess.time_passes().then(|| get_resident_set_size()); - - // The non-parallel compiler can only translate codegen units to LLVM IR - // on a single thread, leading to a staircase effect where the N LLVM - // threads have to wait on the single codegen threads to generate work - // for them. The parallel compiler does not have this restriction, so - // we can pre-load the LLVM queue in parallel before handing off - // coordination to the OnGoingCodegen scheduler. - // - // This likely is a temporary measure. Once we don't have to support the - // non-parallel compiler anymore, we can compile CGUs end-to-end in - // parallel and get rid of the complicated scheduling logic. - let mut pre_compiled_cgus = if cfg!(parallel_compiler) { - tcx.sess.time("compile_first_CGU_batch", || { - // Try to find one CGU to compile per thread. - let cgus: Vec<_> = cgu_reuse - .iter() - .enumerate() - .filter(|&(_, reuse)| reuse == &CguReuse::No) - .take(tcx.sess.threads()) - .collect(); - - // Compile the found CGUs in parallel. - let start_time = Instant::now(); - - let pre_compiled_cgus = par_iter(cgus) - .map(|(i, _)| { - let module = backend.compile_codegen_unit(tcx, codegen_units[i].name()); - (i, module) - }) - .collect(); - - total_codegen_time += start_time.elapsed(); - - pre_compiled_cgus - }) - } else { - FxHashMap::default() - }; - - for (i, cgu) in codegen_units.iter().enumerate() { - ongoing_codegen.wait_for_signal_to_codegen_item(); - ongoing_codegen.check_for_errors(tcx.sess); - - let cgu_reuse = cgu_reuse[i]; - tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse); - - match cgu_reuse { - CguReuse::No => { - let (module, cost) = if let Some(cgu) = pre_compiled_cgus.remove(&i) { - cgu - } else { - let start_time = Instant::now(); - let module = backend.compile_codegen_unit(tcx, cgu.name()); - total_codegen_time += start_time.elapsed(); - module - }; - // This will unwind if there are errors, which triggers our `AbortCodegenOnDrop` - // guard. Unfortunately, just skipping the `submit_codegened_module_to_llvm` makes - // compilation hang on post-monomorphization errors. - tcx.sess.abort_if_errors(); - - submit_codegened_module_to_llvm( - &backend, - &ongoing_codegen.coordinator.sender, - module, - cost, - ); - false - } - CguReuse::PreLto => { - submit_pre_lto_module_to_llvm( - &backend, - tcx, - &ongoing_codegen.coordinator.sender, - CachedModuleCodegen { - name: cgu.name().to_string(), - source: cgu.previous_work_product(tcx), - }, - ); - true - } - CguReuse::PostLto => { - submit_post_lto_module_to_llvm( - &backend, - &ongoing_codegen.coordinator.sender, - CachedModuleCodegen { - name: cgu.name().to_string(), - source: cgu.previous_work_product(tcx), - }, - ); - true - } - }; - } - - ongoing_codegen.codegen_finished(tcx); - - // Since the main thread is sometimes blocked during codegen, we keep track - // -Ztime-passes output manually. - if tcx.sess.time_passes() { - let end_rss = get_resident_set_size(); - - print_time_passes_entry( - "codegen_to_LLVM_IR", - total_codegen_time, - start_rss.unwrap(), - end_rss, - ); - } - - ongoing_codegen.check_for_errors(tcx.sess); - ongoing_codegen -} - -impl CrateInfo { - pub fn new(tcx: TyCtxt<'_>, target_cpu: String) -> CrateInfo { - let exported_symbols = tcx - .sess - .crate_types() - .iter() - .map(|&c| (c, crate::back::linker::exported_symbols(tcx, c))) - .collect(); - let linked_symbols = tcx - .sess - .crate_types() - .iter() - .map(|&c| (c, crate::back::linker::linked_symbols(tcx, c))) - .collect(); - let local_crate_name = tcx.crate_name(LOCAL_CRATE); - let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID); - let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem); - let windows_subsystem = subsystem.map(|subsystem| { - if subsystem != sym::windows && subsystem != sym::console { - tcx.sess.fatal(&format!( - "invalid windows subsystem `{}`, only \ - `windows` and `console` are allowed", - subsystem - )); - } - subsystem.to_string() - }); - - // This list is used when generating the command line to pass through to - // system linker. The linker expects undefined symbols on the left of the - // command line to be defined in libraries on the right, not the other way - // around. For more info, see some comments in the add_used_library function - // below. - // - // In order to get this left-to-right dependency ordering, we use the reverse - // postorder of all crates putting the leaves at the right-most positions. - let used_crates = tcx - .postorder_cnums(()) - .iter() - .rev() - .copied() - .filter(|&cnum| !tcx.dep_kind(cnum).macros_only()) - .collect(); - - let mut info = CrateInfo { - target_cpu, - exported_symbols, - linked_symbols, - local_crate_name, - compiler_builtins: None, - profiler_runtime: None, - is_no_builtins: Default::default(), - native_libraries: Default::default(), - used_libraries: tcx.native_libraries(LOCAL_CRATE).iter().map(Into::into).collect(), - crate_name: Default::default(), - used_crates, - used_crate_source: Default::default(), - lang_item_to_crate: Default::default(), - missing_lang_items: Default::default(), - dependency_formats: tcx.dependency_formats(()).clone(), - windows_subsystem, - natvis_debugger_visualizers: Default::default(), - }; - let lang_items = tcx.lang_items(); - - let crates = tcx.crates(()); - - let n_crates = crates.len(); - info.native_libraries.reserve(n_crates); - info.crate_name.reserve(n_crates); - info.used_crate_source.reserve(n_crates); - info.missing_lang_items.reserve(n_crates); - - for &cnum in crates.iter() { - info.native_libraries - .insert(cnum, tcx.native_libraries(cnum).iter().map(Into::into).collect()); - info.crate_name.insert(cnum, tcx.crate_name(cnum)); - - let used_crate_source = tcx.used_crate_source(cnum); - info.used_crate_source.insert(cnum, used_crate_source.clone()); - if tcx.is_compiler_builtins(cnum) { - info.compiler_builtins = Some(cnum); - } - if tcx.is_profiler_runtime(cnum) { - info.profiler_runtime = Some(cnum); - } - if tcx.is_no_builtins(cnum) { - info.is_no_builtins.insert(cnum); - } - let missing = tcx.missing_lang_items(cnum); - for &item in missing.iter() { - if let Ok(id) = lang_items.require(item) { - info.lang_item_to_crate.insert(item, id.krate); - } - } - - // No need to look for lang items that don't actually need to exist. - let missing = - missing.iter().cloned().filter(|&l| lang_items::required(tcx, l)).collect(); - info.missing_lang_items.insert(cnum, missing); - } - - let embed_visualizers = tcx.sess.crate_types().iter().any(|&crate_type| match crate_type { - CrateType::Executable | CrateType::Dylib | CrateType::Cdylib => { - // These are crate types for which we invoke the linker and can embed - // NatVis visualizers. - true - } - CrateType::ProcMacro => { - // We could embed NatVis for proc macro crates too (to improve the debugging - // experience for them) but it does not seem like a good default, since - // this is a rare use case and we don't want to slow down the common case. - false - } - CrateType::Staticlib | CrateType::Rlib => { - // We don't invoke the linker for these, so we don't need to collect the NatVis for them. - false - } - }); - - if tcx.sess.target.is_like_msvc && embed_visualizers { - info.natvis_debugger_visualizers = - collect_debugger_visualizers_transitive(tcx, DebuggerVisualizerType::Natvis); - } - - info - } -} - -pub fn provide(providers: &mut Providers) { - providers.backend_optimization_level = |tcx, cratenum| { - let for_speed = match tcx.sess.opts.optimize { - // If globally no optimisation is done, #[optimize] has no effect. - // - // This is done because if we ended up "upgrading" to `-O2` here, we’d populate the - // pass manager and it is likely that some module-wide passes (such as inliner or - // cross-function constant propagation) would ignore the `optnone` annotation we put - // on the functions, thus necessarily involving these functions into optimisations. - config::OptLevel::No => return config::OptLevel::No, - // If globally optimise-speed is already specified, just use that level. - config::OptLevel::Less => return config::OptLevel::Less, - config::OptLevel::Default => return config::OptLevel::Default, - config::OptLevel::Aggressive => return config::OptLevel::Aggressive, - // If globally optimize-for-size has been requested, use -O2 instead (if optimize(size) - // are present). - config::OptLevel::Size => config::OptLevel::Default, - config::OptLevel::SizeMin => config::OptLevel::Default, - }; - - let (defids, _) = tcx.collect_and_partition_mono_items(cratenum); - for id in &*defids { - let CodegenFnAttrs { optimize, .. } = tcx.codegen_fn_attrs(*id); - match optimize { - attr::OptimizeAttr::None => continue, - attr::OptimizeAttr::Size => continue, - attr::OptimizeAttr::Speed => { - return for_speed; - } - } - } - tcx.sess.opts.optimize - }; -} - -fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguReuse { - if !tcx.dep_graph.is_fully_enabled() { - return CguReuse::No; - } - - let work_product_id = &cgu.work_product_id(); - if tcx.dep_graph.previous_work_product(work_product_id).is_none() { - // We don't have anything cached for this CGU. This can happen - // if the CGU did not exist in the previous session. - return CguReuse::No; - } - - // Try to mark the CGU as green. If it we can do so, it means that nothing - // affecting the LLVM module has changed and we can re-use a cached version. - // If we compile with any kind of LTO, this means we can re-use the bitcode - // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only - // know that later). If we are not doing LTO, there is only one optimized - // version of each module, so we re-use that. - let dep_node = cgu.codegen_dep_node(tcx); - assert!( - !tcx.dep_graph.dep_node_exists(&dep_node), - "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", - cgu.name() - ); - - if tcx.try_mark_green(&dep_node) { - // We can re-use either the pre- or the post-thinlto state. If no LTO is - // being performed then we can use post-LTO artifacts, otherwise we must - // reuse pre-LTO artifacts - match compute_per_cgu_lto_type( - &tcx.sess.lto(), - &tcx.sess.opts, - &tcx.sess.crate_types(), - ModuleKind::Regular, - ) { - ComputedLtoType::No => CguReuse::PostLto, - _ => CguReuse::PreLto, - } - } else { - CguReuse::No - } -} diff --git a/compiler/rustc_codegen_ssa/src/common.rs b/compiler/rustc_codegen_ssa/src/common.rs deleted file mode 100644 index 8ca1a6084cf..00000000000 --- a/compiler/rustc_codegen_ssa/src/common.rs +++ /dev/null @@ -1,223 +0,0 @@ -#![allow(non_camel_case_types)] - -use rustc_errors::struct_span_err; -use rustc_hir as hir; -use rustc_hir::LangItem; -use rustc_middle::mir::interpret::ConstValue; -use rustc_middle::ty::{self, layout::TyAndLayout, Ty, TyCtxt}; -use rustc_session::Session; -use rustc_span::Span; - -use crate::base; -use crate::traits::*; - -#[derive(Copy, Clone)] -pub enum IntPredicate { - IntEQ, - IntNE, - IntUGT, - IntUGE, - IntULT, - IntULE, - IntSGT, - IntSGE, - IntSLT, - IntSLE, -} - -#[derive(Copy, Clone)] -pub enum RealPredicate { - RealPredicateFalse, - RealOEQ, - RealOGT, - RealOGE, - RealOLT, - RealOLE, - RealONE, - RealORD, - RealUNO, - RealUEQ, - RealUGT, - RealUGE, - RealULT, - RealULE, - RealUNE, - RealPredicateTrue, -} - -#[derive(Copy, Clone)] -pub enum AtomicRmwBinOp { - AtomicXchg, - AtomicAdd, - AtomicSub, - AtomicAnd, - AtomicNand, - AtomicOr, - AtomicXor, - AtomicMax, - AtomicMin, - AtomicUMax, - AtomicUMin, -} - -#[derive(Copy, Clone)] -pub enum AtomicOrdering { - Unordered, - Relaxed, - Acquire, - Release, - AcquireRelease, - SequentiallyConsistent, -} - -#[derive(Copy, Clone)] -pub enum SynchronizationScope { - SingleThread, - CrossThread, -} - -#[derive(Copy, Clone, PartialEq, Debug)] -pub enum TypeKind { - Void, - Half, - Float, - Double, - X86_FP80, - FP128, - PPC_FP128, - Label, - Integer, - Function, - Struct, - Array, - Pointer, - Vector, - Metadata, - X86_MMX, - Token, - ScalableVector, - BFloat, - X86_AMX, -} - -// FIXME(mw): Anything that is produced via DepGraph::with_task() must implement -// the HashStable trait. Normally DepGraph::with_task() calls are -// hidden behind queries, but CGU creation is a special case in two -// ways: (1) it's not a query and (2) CGU are output nodes, so their -// Fingerprints are not actually needed. It remains to be clarified -// how exactly this case will be handled in the red/green system but -// for now we content ourselves with providing a no-op HashStable -// implementation for CGUs. -mod temp_stable_hash_impls { - use crate::ModuleCodegen; - use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; - - impl HashStable for ModuleCodegen { - fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) { - // do nothing - } - } -} - -pub fn build_langcall<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &Bx, - span: Option, - li: LangItem, -) -> (Bx::FnAbiOfResult, Bx::Value) { - let tcx = bx.tcx(); - let def_id = tcx.require_lang_item(li, span); - let instance = ty::Instance::mono(tcx, def_id); - (bx.fn_abi_of_instance(instance, ty::List::empty()), bx.get_fn_addr(instance)) -} - -// To avoid UB from LLVM, these two functions mask RHS with an -// appropriate mask unconditionally (i.e., the fallback behavior for -// all shifts). For 32- and 64-bit types, this matches the semantics -// of Java. (See related discussion on #1877 and #10183.) - -pub fn build_unchecked_lshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - lhs: Bx::Value, - rhs: Bx::Value, -) -> Bx::Value { - let rhs = base::cast_shift_expr_rhs(bx, hir::BinOpKind::Shl, lhs, rhs); - // #1877, #10183: Ensure that input is always valid - let rhs = shift_mask_rhs(bx, rhs); - bx.shl(lhs, rhs) -} - -pub fn build_unchecked_rshift<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - lhs_t: Ty<'tcx>, - lhs: Bx::Value, - rhs: Bx::Value, -) -> Bx::Value { - let rhs = base::cast_shift_expr_rhs(bx, hir::BinOpKind::Shr, lhs, rhs); - // #1877, #10183: Ensure that input is always valid - let rhs = shift_mask_rhs(bx, rhs); - let is_signed = lhs_t.is_signed(); - if is_signed { bx.ashr(lhs, rhs) } else { bx.lshr(lhs, rhs) } -} - -fn shift_mask_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - rhs: Bx::Value, -) -> Bx::Value { - let rhs_llty = bx.val_ty(rhs); - let shift_val = shift_mask_val(bx, rhs_llty, rhs_llty, false); - bx.and(rhs, shift_val) -} - -pub fn shift_mask_val<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - llty: Bx::Type, - mask_llty: Bx::Type, - invert: bool, -) -> Bx::Value { - let kind = bx.type_kind(llty); - match kind { - TypeKind::Integer => { - // i8/u8 can shift by at most 7, i16/u16 by at most 15, etc. - let val = bx.int_width(llty) - 1; - if invert { - bx.const_int(mask_llty, !val as i64) - } else { - bx.const_uint(mask_llty, val) - } - } - TypeKind::Vector => { - let mask = - shift_mask_val(bx, bx.element_type(llty), bx.element_type(mask_llty), invert); - bx.vector_splat(bx.vector_length(mask_llty), mask) - } - _ => bug!("shift_mask_val: expected Integer or Vector, found {:?}", kind), - } -} - -pub fn span_invalid_monomorphization_error(a: &Session, b: Span, c: &str) { - struct_span_err!(a, b, E0511, "{}", c).emit(); -} - -pub fn asm_const_to_str<'tcx>( - tcx: TyCtxt<'tcx>, - sp: Span, - const_value: ConstValue<'tcx>, - ty_and_layout: TyAndLayout<'tcx>, -) -> String { - let ConstValue::Scalar(scalar) = const_value else { - span_bug!(sp, "expected Scalar for promoted asm const, but got {:#?}", const_value) - }; - let value = scalar.assert_bits(ty_and_layout.size); - match ty_and_layout.ty.kind() { - ty::Uint(_) => value.to_string(), - ty::Int(int_ty) => match int_ty.normalize(tcx.sess.target.pointer_width) { - ty::IntTy::I8 => (value as i8).to_string(), - ty::IntTy::I16 => (value as i16).to_string(), - ty::IntTy::I32 => (value as i32).to_string(), - ty::IntTy::I64 => (value as i64).to_string(), - ty::IntTy::I128 => (value as i128).to_string(), - ty::IntTy::Isize => unreachable!(), - }, - _ => span_bug!(sp, "asm const has bad type {}", ty_and_layout.ty), - } -} diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs deleted file mode 100644 index e288760a02b..00000000000 --- a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs +++ /dev/null @@ -1,85 +0,0 @@ -use rustc_middle::mir::coverage::{CounterValueReference, MappedExpressionIndex}; - -/// Aligns with [llvm::coverage::Counter::CounterKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L95) -#[derive(Copy, Clone, Debug)] -#[repr(C)] -pub enum CounterKind { - Zero = 0, - CounterValueReference = 1, - Expression = 2, -} - -/// A reference to an instance of an abstract "counter" that will yield a value in a coverage -/// report. Note that `id` has different interpretations, depending on the `kind`: -/// * For `CounterKind::Zero`, `id` is assumed to be `0` -/// * For `CounterKind::CounterValueReference`, `id` matches the `counter_id` of the injected -/// instrumentation counter (the `index` argument to the LLVM intrinsic -/// `instrprof.increment()`) -/// * For `CounterKind::Expression`, `id` is the index into the coverage map's array of -/// counter expressions. -/// Aligns with [llvm::coverage::Counter](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L102-L103) -/// Important: The Rust struct layout (order and types of fields) must match its C++ counterpart. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -pub struct Counter { - // Important: The layout (order and types of fields) must match its C++ counterpart. - pub kind: CounterKind, - id: u32, -} - -impl Counter { - /// Constructs a new `Counter` of kind `Zero`. For this `CounterKind`, the - /// `id` is not used. - pub fn zero() -> Self { - Self { kind: CounterKind::Zero, id: 0 } - } - - /// Constructs a new `Counter` of kind `CounterValueReference`, and converts - /// the given 1-based counter_id to the required 0-based equivalent for - /// the `Counter` encoding. - pub fn counter_value_reference(counter_id: CounterValueReference) -> Self { - Self { kind: CounterKind::CounterValueReference, id: counter_id.zero_based_index() } - } - - /// Constructs a new `Counter` of kind `Expression`. - pub fn expression(mapped_expression_index: MappedExpressionIndex) -> Self { - Self { kind: CounterKind::Expression, id: mapped_expression_index.into() } - } - - /// Returns true if the `Counter` kind is `Zero`. - pub fn is_zero(&self) -> bool { - matches!(self.kind, CounterKind::Zero) - } - - /// An explicitly-named function to get the ID value, making it more obvious - /// that the stored value is now 0-based. - pub fn zero_based_id(&self) -> u32 { - debug_assert!(!self.is_zero(), "`id` is undefined for CounterKind::Zero"); - self.id - } -} - -/// Aligns with [llvm::coverage::CounterExpression::ExprKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L150) -#[derive(Copy, Clone, Debug)] -#[repr(C)] -pub enum ExprKind { - Subtract = 0, - Add = 1, -} - -/// Aligns with [llvm::coverage::CounterExpression](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L152) -/// Important: The Rust struct layout (order and types of fields) must match its C++ -/// counterpart. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -pub struct CounterExpression { - pub kind: ExprKind, - pub lhs: Counter, - pub rhs: Counter, -} - -impl CounterExpression { - pub fn new(lhs: Counter, kind: ExprKind, rhs: Counter) -> Self { - Self { kind, lhs, rhs } - } -} diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs deleted file mode 100644 index 1a6495cb15c..00000000000 --- a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs +++ /dev/null @@ -1,347 +0,0 @@ -pub use super::ffi::*; - -use rustc_index::vec::IndexVec; -use rustc_middle::mir::coverage::{ - CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, - InjectedExpressionIndex, MappedExpressionIndex, Op, -}; -use rustc_middle::ty::Instance; -use rustc_middle::ty::TyCtxt; - -#[derive(Clone, Debug, PartialEq)] -pub struct Expression { - lhs: ExpressionOperandId, - op: Op, - rhs: ExpressionOperandId, - region: Option, -} - -/// Collects all of the coverage regions associated with (a) injected counters, (b) counter -/// expressions (additions or subtraction), and (c) unreachable regions (always counted as zero), -/// for a given Function. Counters and counter expressions have non-overlapping `id`s because they -/// can both be operands in an expression. This struct also stores the `function_source_hash`, -/// computed during instrumentation, and forwarded with counters. -/// -/// Note, it may be important to understand LLVM's definitions of `unreachable` regions versus "gap -/// regions" (or "gap areas"). A gap region is a code region within a counted region (either counter -/// or expression), but the line or lines in the gap region are not executable (such as lines with -/// only whitespace or comments). According to LLVM Code Coverage Mapping documentation, "A count -/// for a gap area is only used as the line execution count if there are no other regions on a -/// line." -#[derive(Debug)] -pub struct FunctionCoverage<'tcx> { - instance: Instance<'tcx>, - source_hash: u64, - is_used: bool, - counters: IndexVec>, - expressions: IndexVec>, - unreachable_regions: Vec, -} - -impl<'tcx> FunctionCoverage<'tcx> { - /// Creates a new set of coverage data for a used (called) function. - pub fn new(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { - Self::create(tcx, instance, true) - } - - /// Creates a new set of coverage data for an unused (never called) function. - pub fn unused(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { - Self::create(tcx, instance, false) - } - - fn create(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, is_used: bool) -> Self { - let coverageinfo = tcx.coverageinfo(instance.def); - debug!( - "FunctionCoverage::create(instance={:?}) has coverageinfo={:?}. is_used={}", - instance, coverageinfo, is_used - ); - Self { - instance, - source_hash: 0, // will be set with the first `add_counter()` - is_used, - counters: IndexVec::from_elem_n(None, coverageinfo.num_counters as usize), - expressions: IndexVec::from_elem_n(None, coverageinfo.num_expressions as usize), - unreachable_regions: Vec::new(), - } - } - - /// Returns true for a used (called) function, and false for an unused function. - pub fn is_used(&self) -> bool { - self.is_used - } - - /// Sets the function source hash value. If called multiple times for the same function, all - /// calls should have the same hash value. - pub fn set_function_source_hash(&mut self, source_hash: u64) { - if self.source_hash == 0 { - self.source_hash = source_hash; - } else { - debug_assert_eq!(source_hash, self.source_hash); - } - } - - /// Adds a code region to be counted by an injected counter intrinsic. - pub fn add_counter(&mut self, id: CounterValueReference, region: CodeRegion) { - if let Some(previous_region) = self.counters[id].replace(region.clone()) { - assert_eq!(previous_region, region, "add_counter: code region for id changed"); - } - } - - /// Both counters and "counter expressions" (or simply, "expressions") can be operands in other - /// expressions. Expression IDs start from `u32::MAX` and go down, so the range of expression - /// IDs will not overlap with the range of counter IDs. Counters and expressions can be added in - /// any order, and expressions can still be assigned contiguous (though descending) IDs, without - /// knowing what the last counter ID will be. - /// - /// When storing the expression data in the `expressions` vector in the `FunctionCoverage` - /// struct, its vector index is computed, from the given expression ID, by subtracting from - /// `u32::MAX`. - /// - /// Since the expression operands (`lhs` and `rhs`) can reference either counters or - /// expressions, an operand that references an expression also uses its original ID, descending - /// from `u32::MAX`. Theses operands are translated only during code generation, after all - /// counters and expressions have been added. - pub fn add_counter_expression( - &mut self, - expression_id: InjectedExpressionId, - lhs: ExpressionOperandId, - op: Op, - rhs: ExpressionOperandId, - region: Option, - ) { - debug!( - "add_counter_expression({:?}, lhs={:?}, op={:?}, rhs={:?} at {:?}", - expression_id, lhs, op, rhs, region - ); - let expression_index = self.expression_index(u32::from(expression_id)); - debug_assert!( - expression_index.as_usize() < self.expressions.len(), - "expression_index {} is out of range for expressions.len() = {} - for {:?}", - expression_index.as_usize(), - self.expressions.len(), - self, - ); - if let Some(previous_expression) = self.expressions[expression_index].replace(Expression { - lhs, - op, - rhs, - region: region.clone(), - }) { - assert_eq!( - previous_expression, - Expression { lhs, op, rhs, region }, - "add_counter_expression: expression for id changed" - ); - } - } - - /// Add a region that will be marked as "unreachable", with a constant "zero counter". - pub fn add_unreachable_region(&mut self, region: CodeRegion) { - self.unreachable_regions.push(region) - } - - /// Return the source hash, generated from the HIR node structure, and used to indicate whether - /// or not the source code structure changed between different compilations. - pub fn source_hash(&self) -> u64 { - self.source_hash - } - - /// Generate an array of CounterExpressions, and an iterator over all `Counter`s and their - /// associated `Regions` (from which the LLVM-specific `CoverageMapGenerator` will create - /// `CounterMappingRegion`s. - pub fn get_expressions_and_counter_regions( - &self, - ) -> (Vec, impl Iterator) { - assert!( - self.source_hash != 0 || !self.is_used, - "No counters provided the source_hash for used function: {:?}", - self.instance - ); - - let counter_regions = self.counter_regions(); - let (counter_expressions, expression_regions) = self.expressions_with_regions(); - let unreachable_regions = self.unreachable_regions(); - - let counter_regions = - counter_regions.chain(expression_regions.into_iter().chain(unreachable_regions)); - (counter_expressions, counter_regions) - } - - fn counter_regions(&self) -> impl Iterator { - self.counters.iter_enumerated().filter_map(|(index, entry)| { - // Option::map() will return None to filter out missing counters. This may happen - // if, for example, a MIR-instrumented counter is removed during an optimization. - entry.as_ref().map(|region| (Counter::counter_value_reference(index), region)) - }) - } - - fn expressions_with_regions( - &self, - ) -> (Vec, impl Iterator) { - let mut counter_expressions = Vec::with_capacity(self.expressions.len()); - let mut expression_regions = Vec::with_capacity(self.expressions.len()); - let mut new_indexes = IndexVec::from_elem_n(None, self.expressions.len()); - - // This closure converts any `Expression` operand (`lhs` or `rhs` of the `Op::Add` or - // `Op::Subtract` operation) into its native `llvm::coverage::Counter::CounterKind` type - // and value. Operand ID value `0` maps to `CounterKind::Zero`; values in the known range - // of injected LLVM counters map to `CounterKind::CounterValueReference` (and the value - // matches the injected counter index); and any other value is converted into a - // `CounterKind::Expression` with the expression's `new_index`. - // - // Expressions will be returned from this function in a sequential vector (array) of - // `CounterExpression`, so the expression IDs must be mapped from their original, - // potentially sparse set of indexes, originally in reverse order from `u32::MAX`. - // - // An `Expression` as an operand will have already been encountered as an `Expression` with - // operands, so its new_index will already have been generated (as a 1-up index value). - // (If an `Expression` as an operand does not have a corresponding new_index, it was - // probably optimized out, after the expression was injected into the MIR, so it will - // get a `CounterKind::Zero` instead.) - // - // In other words, an `Expression`s at any given index can include other expressions as - // operands, but expression operands can only come from the subset of expressions having - // `expression_index`s lower than the referencing `Expression`. Therefore, it is - // reasonable to look up the new index of an expression operand while the `new_indexes` - // vector is only complete up to the current `ExpressionIndex`. - let id_to_counter = |new_indexes: &IndexVec< - InjectedExpressionIndex, - Option, - >, - id: ExpressionOperandId| { - if id == ExpressionOperandId::ZERO { - Some(Counter::zero()) - } else if id.index() < self.counters.len() { - debug_assert!( - id.index() > 0, - "ExpressionOperandId indexes for counters are 1-based, but this id={}", - id.index() - ); - // Note: Some codegen-injected Counters may be only referenced by `Expression`s, - // and may not have their own `CodeRegion`s, - let index = CounterValueReference::from(id.index()); - // Note, the conversion to LLVM `Counter` adjusts the index to be zero-based. - Some(Counter::counter_value_reference(index)) - } else { - let index = self.expression_index(u32::from(id)); - self.expressions - .get(index) - .expect("expression id is out of range") - .as_ref() - // If an expression was optimized out, assume it would have produced a count - // of zero. This ensures that expressions dependent on optimized-out - // expressions are still valid. - .map_or(Some(Counter::zero()), |_| new_indexes[index].map(Counter::expression)) - } - }; - - for (original_index, expression) in - self.expressions.iter_enumerated().filter_map(|(original_index, entry)| { - // Option::map() will return None to filter out missing expressions. This may happen - // if, for example, a MIR-instrumented expression is removed during an optimization. - entry.as_ref().map(|expression| (original_index, expression)) - }) - { - let optional_region = &expression.region; - let Expression { lhs, op, rhs, .. } = *expression; - - if let Some(Some((lhs_counter, mut rhs_counter))) = id_to_counter(&new_indexes, lhs) - .map(|lhs_counter| { - id_to_counter(&new_indexes, rhs).map(|rhs_counter| (lhs_counter, rhs_counter)) - }) - { - if lhs_counter.is_zero() && op.is_subtract() { - // The left side of a subtraction was probably optimized out. As an example, - // a branch condition might be evaluated as a constant expression, and the - // branch could be removed, dropping unused counters in the process. - // - // Since counters are unsigned, we must assume the result of the expression - // can be no more and no less than zero. An expression known to evaluate to zero - // does not need to be added to the coverage map. - // - // Coverage test `loops_branches.rs` includes multiple variations of branches - // based on constant conditional (literal `true` or `false`), and demonstrates - // that the expected counts are still correct. - debug!( - "Expression subtracts from zero (assume unreachable): \ - original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}", - original_index, lhs, op, rhs, optional_region, - ); - rhs_counter = Counter::zero(); - } - debug_assert!( - lhs_counter.is_zero() - // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16` - || ((lhs_counter.zero_based_id() as usize) - <= usize::max(self.counters.len(), self.expressions.len())), - "lhs id={} > both counters.len()={} and expressions.len()={} - ({:?} {:?} {:?})", - lhs_counter.zero_based_id(), - self.counters.len(), - self.expressions.len(), - lhs_counter, - op, - rhs_counter, - ); - - debug_assert!( - rhs_counter.is_zero() - // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16` - || ((rhs_counter.zero_based_id() as usize) - <= usize::max(self.counters.len(), self.expressions.len())), - "rhs id={} > both counters.len()={} and expressions.len()={} - ({:?} {:?} {:?})", - rhs_counter.zero_based_id(), - self.counters.len(), - self.expressions.len(), - lhs_counter, - op, - rhs_counter, - ); - - // Both operands exist. `Expression` operands exist in `self.expressions` and have - // been assigned a `new_index`. - let mapped_expression_index = - MappedExpressionIndex::from(counter_expressions.len()); - let expression = CounterExpression::new( - lhs_counter, - match op { - Op::Add => ExprKind::Add, - Op::Subtract => ExprKind::Subtract, - }, - rhs_counter, - ); - debug!( - "Adding expression {:?} = {:?}, region: {:?}", - mapped_expression_index, expression, optional_region - ); - counter_expressions.push(expression); - new_indexes[original_index] = Some(mapped_expression_index); - if let Some(region) = optional_region { - expression_regions.push((Counter::expression(mapped_expression_index), region)); - } - } else { - bug!( - "expression has one or more missing operands \ - original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}", - original_index, - lhs, - op, - rhs, - optional_region, - ); - } - } - (counter_expressions, expression_regions.into_iter()) - } - - fn unreachable_regions(&self) -> impl Iterator { - self.unreachable_regions.iter().map(|region| (Counter::zero(), region)) - } - - fn expression_index(&self, id_descending_from_max: u32) -> InjectedExpressionIndex { - debug_assert!(id_descending_from_max >= self.counters.len() as u32); - InjectedExpressionIndex::from(u32::MAX - id_descending_from_max) - } -} diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs deleted file mode 100644 index 569fd3f1a51..00000000000 --- a/compiler/rustc_codegen_ssa/src/coverageinfo/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod ffi; -pub mod map; diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/mod.rs b/compiler/rustc_codegen_ssa/src/debuginfo/mod.rs deleted file mode 100644 index 6e3f4f0b8ef..00000000000 --- a/compiler/rustc_codegen_ssa/src/debuginfo/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -use rustc_middle::ty::{self, layout::TyAndLayout}; -use rustc_target::abi::Size; - -// FIXME(eddyb) find a place for this (or a way to replace it). -pub mod type_names; - -/// Returns true if we want to generate a DW_TAG_enumeration_type description for -/// this instead of a DW_TAG_struct_type with DW_TAG_variant_part. -/// -/// NOTE: This is somewhat inconsistent right now: For empty enums and enums with a single -/// fieldless variant, we generate DW_TAG_struct_type, although a -/// DW_TAG_enumeration_type would be a better fit. -pub fn wants_c_like_enum_debuginfo<'tcx>(enum_type_and_layout: TyAndLayout<'tcx>) -> bool { - match enum_type_and_layout.ty.kind() { - ty::Adt(adt_def, _) => { - if !adt_def.is_enum() { - return false; - } - - match adt_def.variants().len() { - 0 => false, - 1 => { - // Univariant enums unless they are zero-sized - enum_type_and_layout.size != Size::ZERO && adt_def.all_fields().count() == 0 - } - _ => { - // Enums with more than one variant if they have no fields - adt_def.all_fields().count() == 0 - } - } - } - _ => false, - } -} diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs deleted file mode 100644 index 8cd5a0fc247..00000000000 --- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs +++ /dev/null @@ -1,821 +0,0 @@ -// Type Names for Debug Info. - -// Notes on targeting MSVC: -// In general, MSVC's debugger attempts to parse all arguments as C++ expressions, -// even if the argument is explicitly a symbol name. -// As such, there are many things that cause parsing issues: -// * `#` is treated as a special character for macros. -// * `{` or `<` at the beginning of a name is treated as an operator. -// * `>>` is always treated as a right-shift. -// * `[` in a name is treated like a regex bracket expression (match any char -// within the brackets). -// * `"` is treated as the start of a string. - -use rustc_data_structures::fx::FxHashSet; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; -use rustc_hir::def_id::DefId; -use rustc_hir::definitions::{DefPathData, DefPathDataName, DisambiguatedDefPathData}; -use rustc_hir::{AsyncGeneratorKind, GeneratorKind, Mutability}; -use rustc_middle::ty::layout::{IntegerExt, TyAndLayout}; -use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; -use rustc_middle::ty::{self, ExistentialProjection, GeneratorSubsts, ParamEnv, Ty, TyCtxt}; -use rustc_target::abi::{Integer, TagEncoding, Variants}; -use smallvec::SmallVec; - -use std::borrow::Cow; -use std::fmt::Write; - -use crate::debuginfo::wants_c_like_enum_debuginfo; - -// Compute the name of the type as it should be stored in debuginfo. Does not do -// any caching, i.e., calling the function twice with the same type will also do -// the work twice. The `qualified` parameter only affects the first level of the -// type name, further levels (i.e., type parameters) are always fully qualified. -pub fn compute_debuginfo_type_name<'tcx>( - tcx: TyCtxt<'tcx>, - t: Ty<'tcx>, - qualified: bool, -) -> String { - let _prof = tcx.prof.generic_activity("compute_debuginfo_type_name"); - - let mut result = String::with_capacity(64); - let mut visited = FxHashSet::default(); - push_debuginfo_type_name(tcx, t, qualified, &mut result, &mut visited); - result -} - -// Pushes the name of the type as it should be stored in debuginfo on the -// `output` String. See also compute_debuginfo_type_name(). -fn push_debuginfo_type_name<'tcx>( - tcx: TyCtxt<'tcx>, - t: Ty<'tcx>, - qualified: bool, - output: &mut String, - visited: &mut FxHashSet>, -) { - // When targeting MSVC, emit C++ style type names for compatibility with - // .natvis visualizers (and perhaps other existing native debuggers?) - let cpp_like_debuginfo = cpp_like_debuginfo(tcx); - - match *t.kind() { - ty::Bool => output.push_str("bool"), - ty::Char => output.push_str("char"), - ty::Str => output.push_str("str"), - ty::Never => { - if cpp_like_debuginfo { - output.push_str("never$"); - } else { - output.push('!'); - } - } - ty::Int(int_ty) => output.push_str(int_ty.name_str()), - ty::Uint(uint_ty) => output.push_str(uint_ty.name_str()), - ty::Float(float_ty) => output.push_str(float_ty.name_str()), - ty::Foreign(def_id) => push_item_name(tcx, def_id, qualified, output), - ty::Adt(def, substs) => { - // `layout_for_cpp_like_fallback` will be `Some` if we want to use the fallback encoding. - let layout_for_cpp_like_fallback = if cpp_like_debuginfo && def.is_enum() { - match tcx.layout_of(ParamEnv::reveal_all().and(t)) { - Ok(layout) => { - if !wants_c_like_enum_debuginfo(layout) { - Some(layout) - } else { - // This is a C-like enum so we don't want to use the fallback encoding - // for the name. - None - } - } - Err(e) => { - // Computing the layout can still fail here, e.g. if the target architecture - // cannot represent the type. See https://github.com/rust-lang/rust/issues/94961. - tcx.sess.fatal(&format!("{}", e)); - } - } - } else { - // We are not emitting cpp-like debuginfo or this isn't even an enum. - None - }; - - if let Some(ty_and_layout) = layout_for_cpp_like_fallback { - msvc_enum_fallback( - tcx, - ty_and_layout, - &|output, visited| { - push_item_name(tcx, def.did(), true, output); - push_generic_params_internal(tcx, substs, output, visited); - }, - output, - visited, - ); - } else { - push_item_name(tcx, def.did(), qualified, output); - push_generic_params_internal(tcx, substs, output, visited); - } - } - ty::Tuple(component_types) => { - if cpp_like_debuginfo { - output.push_str("tuple$<"); - } else { - output.push('('); - } - - for component_type in component_types { - push_debuginfo_type_name(tcx, component_type, true, output, visited); - push_arg_separator(cpp_like_debuginfo, output); - } - if !component_types.is_empty() { - pop_arg_separator(output); - } - - if cpp_like_debuginfo { - push_close_angle_bracket(cpp_like_debuginfo, output); - } else { - output.push(')'); - } - } - ty::RawPtr(ty::TypeAndMut { ty: inner_type, mutbl }) => { - if cpp_like_debuginfo { - match mutbl { - Mutability::Not => output.push_str("ptr_const$<"), - Mutability::Mut => output.push_str("ptr_mut$<"), - } - } else { - output.push('*'); - match mutbl { - Mutability::Not => output.push_str("const "), - Mutability::Mut => output.push_str("mut "), - } - } - - push_debuginfo_type_name(tcx, inner_type, qualified, output, visited); - - if cpp_like_debuginfo { - push_close_angle_bracket(cpp_like_debuginfo, output); - } - } - ty::Ref(_, inner_type, mutbl) => { - // Slices and `&str` are treated like C++ pointers when computing debug - // info for MSVC debugger. However, wrapping these types' names in a synthetic type - // causes the .natvis engine for WinDbg to fail to display their data, so we opt these - // types out to aid debugging in MSVC. - let is_slice_or_str = matches!(*inner_type.kind(), ty::Slice(_) | ty::Str); - - if !cpp_like_debuginfo { - output.push('&'); - output.push_str(mutbl.prefix_str()); - } else if !is_slice_or_str { - match mutbl { - Mutability::Not => output.push_str("ref$<"), - Mutability::Mut => output.push_str("ref_mut$<"), - } - } - - push_debuginfo_type_name(tcx, inner_type, qualified, output, visited); - - if cpp_like_debuginfo && !is_slice_or_str { - push_close_angle_bracket(cpp_like_debuginfo, output); - } - } - ty::Array(inner_type, len) => { - if cpp_like_debuginfo { - output.push_str("array$<"); - push_debuginfo_type_name(tcx, inner_type, true, output, visited); - match len.kind() { - ty::ConstKind::Param(param) => write!(output, ",{}>", param.name).unwrap(), - _ => write!(output, ",{}>", len.eval_usize(tcx, ty::ParamEnv::reveal_all())) - .unwrap(), - } - } else { - output.push('['); - push_debuginfo_type_name(tcx, inner_type, true, output, visited); - match len.kind() { - ty::ConstKind::Param(param) => write!(output, "; {}]", param.name).unwrap(), - _ => write!(output, "; {}]", len.eval_usize(tcx, ty::ParamEnv::reveal_all())) - .unwrap(), - } - } - } - ty::Slice(inner_type) => { - if cpp_like_debuginfo { - output.push_str("slice$<"); - } else { - output.push('['); - } - - push_debuginfo_type_name(tcx, inner_type, true, output, visited); - - if cpp_like_debuginfo { - push_close_angle_bracket(cpp_like_debuginfo, output); - } else { - output.push(']'); - } - } - ty::Dynamic(ref trait_data, ..) => { - let auto_traits: SmallVec<[DefId; 4]> = trait_data.auto_traits().collect(); - - let has_enclosing_parens = if cpp_like_debuginfo { - output.push_str("dyn$<"); - false - } else { - if trait_data.len() > 1 && auto_traits.len() != 0 { - // We need enclosing parens because there is more than one trait - output.push_str("(dyn "); - true - } else { - output.push_str("dyn "); - false - } - }; - - if let Some(principal) = trait_data.principal() { - let principal = - tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), principal); - push_item_name(tcx, principal.def_id, qualified, output); - let principal_has_generic_params = - push_generic_params_internal(tcx, principal.substs, output, visited); - - let projection_bounds: SmallVec<[_; 4]> = trait_data - .projection_bounds() - .map(|bound| { - let ExistentialProjection { item_def_id, term, .. } = - tcx.erase_late_bound_regions(bound); - // FIXME(associated_const_equality): allow for consts here - (item_def_id, term.ty().unwrap()) - }) - .collect(); - - if projection_bounds.len() != 0 { - if principal_has_generic_params { - // push_generic_params_internal() above added a `>` but we actually - // want to add more items to that list, so remove that again... - pop_close_angle_bracket(output); - // .. and add a comma to separate the regular generic args from the - // associated types. - push_arg_separator(cpp_like_debuginfo, output); - } else { - // push_generic_params_internal() did not add `<...>`, so we open - // angle brackets here. - output.push('<'); - } - - for (item_def_id, ty) in projection_bounds { - if cpp_like_debuginfo { - output.push_str("assoc$<"); - push_item_name(tcx, item_def_id, false, output); - push_arg_separator(cpp_like_debuginfo, output); - push_debuginfo_type_name(tcx, ty, true, output, visited); - push_close_angle_bracket(cpp_like_debuginfo, output); - } else { - push_item_name(tcx, item_def_id, false, output); - output.push('='); - push_debuginfo_type_name(tcx, ty, true, output, visited); - } - push_arg_separator(cpp_like_debuginfo, output); - } - - pop_arg_separator(output); - push_close_angle_bracket(cpp_like_debuginfo, output); - } - - if auto_traits.len() != 0 { - push_auto_trait_separator(cpp_like_debuginfo, output); - } - } - - if auto_traits.len() != 0 { - let mut auto_traits: SmallVec<[String; 4]> = auto_traits - .into_iter() - .map(|def_id| { - let mut name = String::with_capacity(20); - push_item_name(tcx, def_id, true, &mut name); - name - }) - .collect(); - auto_traits.sort_unstable(); - - for auto_trait in auto_traits { - output.push_str(&auto_trait); - push_auto_trait_separator(cpp_like_debuginfo, output); - } - - pop_auto_trait_separator(output); - } - - if cpp_like_debuginfo { - push_close_angle_bracket(cpp_like_debuginfo, output); - } else if has_enclosing_parens { - output.push(')'); - } - } - ty::FnDef(..) | ty::FnPtr(_) => { - // We've encountered a weird 'recursive type' - // Currently, the only way to generate such a type - // is by using 'impl trait': - // - // fn foo() -> impl Copy { foo } - // - // There's not really a sensible name we can generate, - // since we don't include 'impl trait' types (e.g. ty::Opaque) - // in the output - // - // Since we need to generate *something*, we just - // use a dummy string that should make it clear - // that something unusual is going on - if !visited.insert(t) { - output.push_str(if cpp_like_debuginfo { - "recursive_type$" - } else { - "" - }); - return; - } - - let sig = - tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), t.fn_sig(tcx)); - - if cpp_like_debuginfo { - // Format as a C++ function pointer: return_type (*)(params...) - if sig.output().is_unit() { - output.push_str("void"); - } else { - push_debuginfo_type_name(tcx, sig.output(), true, output, visited); - } - output.push_str(" (*)("); - } else { - output.push_str(sig.unsafety.prefix_str()); - - if sig.abi != rustc_target::spec::abi::Abi::Rust { - output.push_str("extern \""); - output.push_str(sig.abi.name()); - output.push_str("\" "); - } - - output.push_str("fn("); - } - - if !sig.inputs().is_empty() { - for ¶meter_type in sig.inputs() { - push_debuginfo_type_name(tcx, parameter_type, true, output, visited); - push_arg_separator(cpp_like_debuginfo, output); - } - pop_arg_separator(output); - } - - if sig.c_variadic { - if !sig.inputs().is_empty() { - output.push_str(", ..."); - } else { - output.push_str("..."); - } - } - - output.push(')'); - - if !cpp_like_debuginfo && !sig.output().is_unit() { - output.push_str(" -> "); - push_debuginfo_type_name(tcx, sig.output(), true, output, visited); - } - - // We only keep the type in 'visited' - // for the duration of the body of this method. - // It's fine for a particular function type - // to show up multiple times in one overall type - // (e.g. MyType u8, fn() -> u8> - // - // We only care about avoiding recursing - // directly back to the type we're currently - // processing - visited.remove(&t); - } - ty::Closure(def_id, substs) | ty::Generator(def_id, substs, ..) => { - // Name will be "{closure_env#0}", "{generator_env#0}", or - // "{async_fn_env#0}", etc. - // In the case of cpp-like debuginfo, the name additionally gets wrapped inside of - // an artificial `enum$<>` type, as defined in msvc_enum_fallback(). - if cpp_like_debuginfo && t.is_generator() { - let ty_and_layout = tcx.layout_of(ParamEnv::reveal_all().and(t)).unwrap(); - msvc_enum_fallback( - tcx, - ty_and_layout, - &|output, visited| { - push_closure_or_generator_name(tcx, def_id, substs, true, output, visited); - }, - output, - visited, - ); - } else { - push_closure_or_generator_name(tcx, def_id, substs, qualified, output, visited); - } - } - // Type parameters from polymorphized functions. - ty::Param(_) => { - write!(output, "{:?}", t).unwrap(); - } - ty::Error(_) - | ty::Infer(_) - | ty::Placeholder(..) - | ty::Projection(..) - | ty::Bound(..) - | ty::Opaque(..) - | ty::GeneratorWitness(..) => { - bug!( - "debuginfo: Trying to create type name for \ - unexpected type: {:?}", - t - ); - } - } - - /// MSVC names enums differently than other platforms so that the debugging visualization - // format (natvis) is able to understand enums and render the active variant correctly in the - // debugger. For more information, look in `src/etc/natvis/intrinsic.natvis` and - // `EnumMemberDescriptionFactor::create_member_descriptions`. - fn msvc_enum_fallback<'tcx>( - tcx: TyCtxt<'tcx>, - ty_and_layout: TyAndLayout<'tcx>, - push_inner: &dyn Fn(/*output*/ &mut String, /*visited*/ &mut FxHashSet>), - output: &mut String, - visited: &mut FxHashSet>, - ) { - debug_assert!(!wants_c_like_enum_debuginfo(ty_and_layout)); - let ty = ty_and_layout.ty; - - output.push_str("enum$<"); - push_inner(output, visited); - - let variant_name = |variant_index| match ty.kind() { - ty::Adt(adt_def, _) => { - debug_assert!(adt_def.is_enum()); - Cow::from(adt_def.variant(variant_index).name.as_str()) - } - ty::Generator(..) => GeneratorSubsts::variant_name(variant_index), - _ => unreachable!(), - }; - - if let Variants::Multiple { - tag_encoding: TagEncoding::Niche { dataful_variant, .. }, - tag, - variants, - .. - } = &ty_and_layout.variants - { - let dataful_variant_layout = &variants[*dataful_variant]; - - // calculate the range of values for the dataful variant - let dataful_discriminant_range = - dataful_variant_layout.largest_niche().unwrap().valid_range; - - let min = dataful_discriminant_range.start; - let min = tag.size(&tcx).truncate(min); - - let max = dataful_discriminant_range.end; - let max = tag.size(&tcx).truncate(max); - - let dataful_variant_name = variant_name(*dataful_variant); - write!(output, ", {}, {}, {}", min, max, dataful_variant_name).unwrap(); - } else if let Variants::Single { index: variant_idx } = &ty_and_layout.variants { - // Uninhabited enums can't be constructed and should never need to be visualized so - // skip this step for them. - if !ty_and_layout.abi.is_uninhabited() { - write!(output, ", {}", variant_name(*variant_idx)).unwrap(); - } - } - push_close_angle_bracket(true, output); - } - - const NON_CPP_AUTO_TRAIT_SEPARATOR: &str = " + "; - - fn push_auto_trait_separator(cpp_like_debuginfo: bool, output: &mut String) { - if cpp_like_debuginfo { - push_arg_separator(cpp_like_debuginfo, output); - } else { - output.push_str(NON_CPP_AUTO_TRAIT_SEPARATOR); - } - } - - fn pop_auto_trait_separator(output: &mut String) { - if output.ends_with(NON_CPP_AUTO_TRAIT_SEPARATOR) { - output.truncate(output.len() - NON_CPP_AUTO_TRAIT_SEPARATOR.len()); - } else { - pop_arg_separator(output); - } - } -} - -pub enum VTableNameKind { - // Is the name for the const/static holding the vtable? - GlobalVariable, - // Is the name for the type of the vtable? - Type, -} - -/// Computes a name for the global variable storing a vtable (or the type of that global variable). -/// -/// The name is of the form: -/// -/// `::{vtable}` -/// -/// or, when generating C++-like names: -/// -/// `impl$::vtable$` -/// -/// If `kind` is `VTableNameKind::Type` then the last component is `{vtable_ty}` instead of just -/// `{vtable}`, so that the type and the corresponding global variable get assigned different -/// names. -pub fn compute_debuginfo_vtable_name<'tcx>( - tcx: TyCtxt<'tcx>, - t: Ty<'tcx>, - trait_ref: Option>, - kind: VTableNameKind, -) -> String { - let cpp_like_debuginfo = cpp_like_debuginfo(tcx); - - let mut vtable_name = String::with_capacity(64); - - if cpp_like_debuginfo { - vtable_name.push_str("impl$<"); - } else { - vtable_name.push('<'); - } - - let mut visited = FxHashSet::default(); - push_debuginfo_type_name(tcx, t, true, &mut vtable_name, &mut visited); - - if cpp_like_debuginfo { - vtable_name.push_str(", "); - } else { - vtable_name.push_str(" as "); - } - - if let Some(trait_ref) = trait_ref { - let trait_ref = - tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), trait_ref); - push_item_name(tcx, trait_ref.def_id, true, &mut vtable_name); - visited.clear(); - push_generic_params_internal(tcx, trait_ref.substs, &mut vtable_name, &mut visited); - } else { - vtable_name.push_str("_"); - } - - push_close_angle_bracket(cpp_like_debuginfo, &mut vtable_name); - - let suffix = match (cpp_like_debuginfo, kind) { - (true, VTableNameKind::GlobalVariable) => "::vtable$", - (false, VTableNameKind::GlobalVariable) => "::{vtable}", - (true, VTableNameKind::Type) => "::vtable_type$", - (false, VTableNameKind::Type) => "::{vtable_type}", - }; - - vtable_name.reserve_exact(suffix.len()); - vtable_name.push_str(suffix); - - vtable_name -} - -pub fn push_item_name(tcx: TyCtxt<'_>, def_id: DefId, qualified: bool, output: &mut String) { - let def_key = tcx.def_key(def_id); - if qualified { - if let Some(parent) = def_key.parent { - push_item_name(tcx, DefId { krate: def_id.krate, index: parent }, true, output); - output.push_str("::"); - } - } - - push_unqualified_item_name(tcx, def_id, def_key.disambiguated_data, output); -} - -fn generator_kind_label(generator_kind: Option) -> &'static str { - match generator_kind { - Some(GeneratorKind::Async(AsyncGeneratorKind::Block)) => "async_block", - Some(GeneratorKind::Async(AsyncGeneratorKind::Closure)) => "async_closure", - Some(GeneratorKind::Async(AsyncGeneratorKind::Fn)) => "async_fn", - Some(GeneratorKind::Gen) => "generator", - None => "closure", - } -} - -fn push_disambiguated_special_name( - label: &str, - disambiguator: u32, - cpp_like_debuginfo: bool, - output: &mut String, -) { - if cpp_like_debuginfo { - write!(output, "{}${}", label, disambiguator).unwrap(); - } else { - write!(output, "{{{}#{}}}", label, disambiguator).unwrap(); - } -} - -fn push_unqualified_item_name( - tcx: TyCtxt<'_>, - def_id: DefId, - disambiguated_data: DisambiguatedDefPathData, - output: &mut String, -) { - match disambiguated_data.data { - DefPathData::CrateRoot => { - output.push_str(tcx.crate_name(def_id.krate).as_str()); - } - DefPathData::ClosureExpr => { - let label = generator_kind_label(tcx.generator_kind(def_id)); - - push_disambiguated_special_name( - label, - disambiguated_data.disambiguator, - cpp_like_debuginfo(tcx), - output, - ); - } - _ => match disambiguated_data.data.name() { - DefPathDataName::Named(name) => { - output.push_str(name.as_str()); - } - DefPathDataName::Anon { namespace } => { - push_disambiguated_special_name( - namespace.as_str(), - disambiguated_data.disambiguator, - cpp_like_debuginfo(tcx), - output, - ); - } - }, - }; -} - -fn push_generic_params_internal<'tcx>( - tcx: TyCtxt<'tcx>, - substs: SubstsRef<'tcx>, - output: &mut String, - visited: &mut FxHashSet>, -) -> bool { - if substs.non_erasable_generics().next().is_none() { - return false; - } - - debug_assert_eq!(substs, tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs)); - - let cpp_like_debuginfo = cpp_like_debuginfo(tcx); - - output.push('<'); - - for type_parameter in substs.non_erasable_generics() { - match type_parameter { - GenericArgKind::Type(type_parameter) => { - push_debuginfo_type_name(tcx, type_parameter, true, output, visited); - } - GenericArgKind::Const(ct) => { - push_const_param(tcx, ct, output); - } - other => bug!("Unexpected non-erasable generic: {:?}", other), - } - - push_arg_separator(cpp_like_debuginfo, output); - } - pop_arg_separator(output); - push_close_angle_bracket(cpp_like_debuginfo, output); - - true -} - -fn push_const_param<'tcx>(tcx: TyCtxt<'tcx>, ct: ty::Const<'tcx>, output: &mut String) { - match ct.kind() { - ty::ConstKind::Param(param) => { - write!(output, "{}", param.name) - } - _ => match ct.ty().kind() { - ty::Int(ity) => { - let bits = ct.eval_bits(tcx, ty::ParamEnv::reveal_all(), ct.ty()); - let val = Integer::from_int_ty(&tcx, *ity).size().sign_extend(bits) as i128; - write!(output, "{}", val) - } - ty::Uint(_) => { - let val = ct.eval_bits(tcx, ty::ParamEnv::reveal_all(), ct.ty()); - write!(output, "{}", val) - } - ty::Bool => { - let val = ct.try_eval_bool(tcx, ty::ParamEnv::reveal_all()).unwrap(); - write!(output, "{}", val) - } - _ => { - // If we cannot evaluate the constant to a known type, we fall back - // to emitting a stable hash value of the constant. This isn't very pretty - // but we get a deterministic, virtually unique value for the constant. - // - // Let's only emit 64 bits of the hash value. That should be plenty for - // avoiding collisions and will make the emitted type names shorter. - let hash_short = tcx.with_stable_hashing_context(|mut hcx| { - let mut hasher = StableHasher::new(); - let ct = ct.eval(tcx, ty::ParamEnv::reveal_all()); - hcx.while_hashing_spans(false, |hcx| { - ct.to_valtree().hash_stable(hcx, &mut hasher) - }); - // Note: Don't use `StableHashResult` impl of `u64` here directly, since that - // would lead to endianness problems. - let hash: u128 = hasher.finish(); - (hash.to_le() as u64).to_le() - }); - - if cpp_like_debuginfo(tcx) { - write!(output, "CONST${:x}", hash_short) - } else { - write!(output, "{{CONST#{:x}}}", hash_short) - } - } - }, - } - .unwrap(); -} - -pub fn push_generic_params<'tcx>(tcx: TyCtxt<'tcx>, substs: SubstsRef<'tcx>, output: &mut String) { - let _prof = tcx.prof.generic_activity("compute_debuginfo_type_name"); - let mut visited = FxHashSet::default(); - push_generic_params_internal(tcx, substs, output, &mut visited); -} - -fn push_closure_or_generator_name<'tcx>( - tcx: TyCtxt<'tcx>, - def_id: DefId, - substs: SubstsRef<'tcx>, - qualified: bool, - output: &mut String, - visited: &mut FxHashSet>, -) { - // Name will be "{closure_env#0}", "{generator_env#0}", or - // "{async_fn_env#0}", etc. - let def_key = tcx.def_key(def_id); - let generator_kind = tcx.generator_kind(def_id); - - if qualified { - let parent_def_id = DefId { index: def_key.parent.unwrap(), ..def_id }; - push_item_name(tcx, parent_def_id, true, output); - output.push_str("::"); - } - - let mut label = String::with_capacity(20); - write!(&mut label, "{}_env", generator_kind_label(generator_kind)).unwrap(); - - push_disambiguated_special_name( - &label, - def_key.disambiguated_data.disambiguator, - cpp_like_debuginfo(tcx), - output, - ); - - // We also need to add the generic arguments of the async fn/generator or - // the enclosing function (for closures or async blocks), so that we end - // up with a unique name for every instantiation. - - // Find the generics of the enclosing function, as defined in the source code. - let enclosing_fn_def_id = tcx.typeck_root_def_id(def_id); - let generics = tcx.generics_of(enclosing_fn_def_id); - - // Truncate the substs to the length of the above generics. This will cut off - // anything closure- or generator-specific. - let substs = substs.truncate_to(tcx, generics); - push_generic_params_internal(tcx, substs, output, visited); -} - -fn push_close_angle_bracket(cpp_like_debuginfo: bool, output: &mut String) { - // MSVC debugger always treats `>>` as a shift, even when parsing templates, - // so add a space to avoid confusion. - if cpp_like_debuginfo && output.ends_with('>') { - output.push(' ') - }; - - output.push('>'); -} - -fn pop_close_angle_bracket(output: &mut String) { - assert!(output.ends_with('>'), "'output' does not end with '>': {}", output); - output.pop(); - if output.ends_with(' ') { - output.pop(); - } -} - -fn push_arg_separator(cpp_like_debuginfo: bool, output: &mut String) { - // Natvis does not always like having spaces between parts of the type name - // and this causes issues when we need to write a typename in natvis, for example - // as part of a cast like the `HashMap` visualizer does. - if cpp_like_debuginfo { - output.push(','); - } else { - output.push_str(", "); - }; -} - -fn pop_arg_separator(output: &mut String) { - if output.ends_with(' ') { - output.pop(); - } - - assert!(output.ends_with(',')); - - output.pop(); -} - -/// Check if we should generate C++ like names and debug information. -pub fn cpp_like_debuginfo(tcx: TyCtxt<'_>) -> bool { - tcx.sess.target.is_like_msvc -} diff --git a/compiler/rustc_codegen_ssa/src/glue.rs b/compiler/rustc_codegen_ssa/src/glue.rs deleted file mode 100644 index e6f402ef19d..00000000000 --- a/compiler/rustc_codegen_ssa/src/glue.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! -// -// Code relating to drop glue. - -use crate::common::IntPredicate; -use crate::meth; -use crate::traits::*; -use rustc_middle::ty::{self, Ty}; -use rustc_target::abi::WrappingRange; - -pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - t: Ty<'tcx>, - info: Option, -) -> (Bx::Value, Bx::Value) { - let layout = bx.layout_of(t); - debug!("size_and_align_of_dst(ty={}, info={:?}): layout: {:?}", t, info, layout); - if !layout.is_unsized() { - let size = bx.const_usize(layout.size.bytes()); - let align = bx.const_usize(layout.align.abi.bytes()); - return (size, align); - } - match t.kind() { - ty::Dynamic(..) => { - // Load size/align from vtable. - let vtable = info.unwrap(); - let size = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_SIZE) - .get_usize(bx, vtable); - let align = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_ALIGN) - .get_usize(bx, vtable); - - // Alignment is always nonzero. - bx.range_metadata(align, WrappingRange { start: 1, end: !0 }); - - (size, align) - } - ty::Slice(_) | ty::Str => { - let unit = layout.field(bx, 0); - // The info in this case is the length of the str, so the size is that - // times the unit size. - ( - // All slice sizes must fit into `isize`, so this multiplication cannot (signed) wrap. - // NOTE: ideally, we want the effects of both `unchecked_smul` and `unchecked_umul` - // (resulting in `mul nsw nuw` in LLVM IR), since we know that the multiplication - // cannot signed wrap, and that both operands are non-negative. But at the time of writing, - // `BuilderMethods` can't do this, and it doesn't seem to enable any further optimizations. - bx.unchecked_smul(info.unwrap(), bx.const_usize(unit.size.bytes())), - bx.const_usize(unit.align.abi.bytes()), - ) - } - _ => { - // First get the size of all statically known fields. - // Don't use size_of because it also rounds up to alignment, which we - // want to avoid, as the unsized field's alignment could be smaller. - assert!(!t.is_simd()); - debug!("DST {} layout: {:?}", t, layout); - - let i = layout.fields.count() - 1; - let sized_size = layout.fields.offset(i).bytes(); - let sized_align = layout.align.abi.bytes(); - debug!("DST {} statically sized prefix size: {} align: {}", t, sized_size, sized_align); - let sized_size = bx.const_usize(sized_size); - let sized_align = bx.const_usize(sized_align); - - // Recurse to get the size of the dynamically sized field (must be - // the last field). - let field_ty = layout.field(bx, i).ty; - let (unsized_size, mut unsized_align) = size_and_align_of_dst(bx, field_ty, info); - - // FIXME (#26403, #27023): We should be adding padding - // to `sized_size` (to accommodate the `unsized_align` - // required of the unsized field that follows) before - // summing it with `sized_size`. (Note that since #26403 - // is unfixed, we do not yet add the necessary padding - // here. But this is where the add would go.) - - // Return the sum of sizes and max of aligns. - let size = bx.add(sized_size, unsized_size); - - // Packed types ignore the alignment of their fields. - if let ty::Adt(def, _) = t.kind() { - if def.repr().packed() { - unsized_align = sized_align; - } - } - - // Choose max of two known alignments (combined value must - // be aligned according to more restrictive of the two). - let align = match ( - bx.const_to_opt_u128(sized_align, false), - bx.const_to_opt_u128(unsized_align, false), - ) { - (Some(sized_align), Some(unsized_align)) => { - // If both alignments are constant, (the sized_align should always be), then - // pick the correct alignment statically. - bx.const_usize(std::cmp::max(sized_align, unsized_align) as u64) - } - _ => { - let cmp = bx.icmp(IntPredicate::IntUGT, sized_align, unsized_align); - bx.select(cmp, sized_align, unsized_align) - } - }; - - // Issue #27023: must add any necessary padding to `size` - // (to make it a multiple of `align`) before returning it. - // - // Namely, the returned size should be, in C notation: - // - // `size + ((size & (align-1)) ? align : 0)` - // - // emulated via the semi-standard fast bit trick: - // - // `(size + (align-1)) & -align` - let one = bx.const_usize(1); - let addend = bx.sub(align, one); - let add = bx.add(size, addend); - let neg = bx.neg(align); - let size = bx.and(add, neg); - - (size, align) - } - } -} diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs deleted file mode 100644 index 1802eedf193..00000000000 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ /dev/null @@ -1,244 +0,0 @@ -#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![feature(box_patterns)] -#![feature(try_blocks)] -#![feature(let_else)] -#![feature(once_cell)] -#![feature(associated_type_bounds)] -#![feature(strict_provenance)] -#![feature(int_roundings)] -#![feature(if_let_guard)] -#![recursion_limit = "256"] -#![allow(rustc::potential_query_instability)] - -//! This crate contains codegen code that is used by all codegen backends (LLVM and others). -//! The backend-agnostic functions of this crate use functions defined in various traits that -//! have to be implemented by each backends. - -#[macro_use] -extern crate rustc_macros; -#[macro_use] -extern crate tracing; -#[macro_use] -extern crate rustc_middle; - -use rustc_ast as ast; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; -use rustc_data_structures::sync::Lrc; -use rustc_hir::def_id::CrateNum; -use rustc_hir::LangItem; -use rustc_middle::dep_graph::WorkProduct; -use rustc_middle::middle::dependency_format::Dependencies; -use rustc_middle::middle::exported_symbols::SymbolExportKind; -use rustc_middle::ty::query::{ExternProviders, Providers}; -use rustc_serialize::opaque::{MemDecoder, MemEncoder}; -use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; -use rustc_session::config::{CrateType, OutputFilenames, OutputType, RUST_CGU_EXT}; -use rustc_session::cstore::{self, CrateSource}; -use rustc_session::utils::NativeLibKind; -use rustc_span::symbol::Symbol; -use rustc_span::DebuggerVisualizerFile; -use std::collections::BTreeSet; -use std::path::{Path, PathBuf}; - -pub mod back; -pub mod base; -pub mod common; -pub mod coverageinfo; -pub mod debuginfo; -pub mod glue; -pub mod meth; -pub mod mir; -pub mod mono_item; -pub mod target_features; -pub mod traits; - -pub struct ModuleCodegen { - /// The name of the module. When the crate may be saved between - /// compilations, incremental compilation requires that name be - /// unique amongst **all** crates. Therefore, it should contain - /// something unique to this crate (e.g., a module path) as well - /// as the crate name and disambiguator. - /// We currently generate these names via CodegenUnit::build_cgu_name(). - pub name: String, - pub module_llvm: M, - pub kind: ModuleKind, -} - -impl ModuleCodegen { - pub fn into_compiled_module( - self, - emit_obj: bool, - emit_dwarf_obj: bool, - emit_bc: bool, - outputs: &OutputFilenames, - ) -> CompiledModule { - let object = emit_obj.then(|| outputs.temp_path(OutputType::Object, Some(&self.name))); - let dwarf_object = emit_dwarf_obj.then(|| outputs.temp_path_dwo(Some(&self.name))); - let bytecode = emit_bc.then(|| outputs.temp_path(OutputType::Bitcode, Some(&self.name))); - - CompiledModule { name: self.name.clone(), kind: self.kind, object, dwarf_object, bytecode } - } -} - -#[derive(Debug, Encodable, Decodable)] -pub struct CompiledModule { - pub name: String, - pub kind: ModuleKind, - pub object: Option, - pub dwarf_object: Option, - pub bytecode: Option, -} - -pub struct CachedModuleCodegen { - pub name: String, - pub source: WorkProduct, -} - -#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable)] -pub enum ModuleKind { - Regular, - Metadata, - Allocator, -} - -bitflags::bitflags! { - pub struct MemFlags: u8 { - const VOLATILE = 1 << 0; - const NONTEMPORAL = 1 << 1; - const UNALIGNED = 1 << 2; - } -} - -#[derive(Clone, Debug, Encodable, Decodable, HashStable)] -pub struct NativeLib { - pub kind: NativeLibKind, - pub name: Option, - pub cfg: Option, - pub verbatim: Option, - pub dll_imports: Vec, -} - -impl From<&cstore::NativeLib> for NativeLib { - fn from(lib: &cstore::NativeLib) -> Self { - NativeLib { - kind: lib.kind, - name: lib.name, - cfg: lib.cfg.clone(), - verbatim: lib.verbatim, - dll_imports: lib.dll_imports.clone(), - } - } -} - -/// Misc info we load from metadata to persist beyond the tcx. -/// -/// Note: though `CrateNum` is only meaningful within the same tcx, information within `CrateInfo` -/// is self-contained. `CrateNum` can be viewed as a unique identifier within a `CrateInfo`, where -/// `used_crate_source` contains all `CrateSource` of the dependents, and maintains a mapping from -/// identifiers (`CrateNum`) to `CrateSource`. The other fields map `CrateNum` to the crate's own -/// additional properties, so that effectively we can retrieve each dependent crate's `CrateSource` -/// and the corresponding properties without referencing information outside of a `CrateInfo`. -#[derive(Debug, Encodable, Decodable)] -pub struct CrateInfo { - pub target_cpu: String, - pub exported_symbols: FxHashMap>, - pub linked_symbols: FxHashMap>, - pub local_crate_name: Symbol, - pub compiler_builtins: Option, - pub profiler_runtime: Option, - pub is_no_builtins: FxHashSet, - pub native_libraries: FxHashMap>, - pub crate_name: FxHashMap, - pub used_libraries: Vec, - pub used_crate_source: FxHashMap>, - pub used_crates: Vec, - pub lang_item_to_crate: FxHashMap, - pub missing_lang_items: FxHashMap>, - pub dependency_formats: Lrc, - pub windows_subsystem: Option, - pub natvis_debugger_visualizers: BTreeSet, -} - -#[derive(Encodable, Decodable)] -pub struct CodegenResults { - pub modules: Vec, - pub allocator_module: Option, - pub metadata_module: Option, - pub metadata: rustc_metadata::EncodedMetadata, - pub crate_info: CrateInfo, -} - -pub fn provide(providers: &mut Providers) { - crate::back::symbol_export::provide(providers); - crate::base::provide(providers); - crate::target_features::provide(providers); -} - -pub fn provide_extern(providers: &mut ExternProviders) { - crate::back::symbol_export::provide_extern(providers); -} - -/// Checks if the given filename ends with the `.rcgu.o` extension that `rustc` -/// uses for the object files it generates. -pub fn looks_like_rust_object_file(filename: &str) -> bool { - let path = Path::new(filename); - let ext = path.extension().and_then(|s| s.to_str()); - if ext != Some(OutputType::Object.extension()) { - // The file name does not end with ".o", so it can't be an object file. - return false; - } - - // Strip the ".o" at the end - let ext2 = path.file_stem().and_then(|s| Path::new(s).extension()).and_then(|s| s.to_str()); - - // Check if the "inner" extension - ext2 == Some(RUST_CGU_EXT) -} - -const RLINK_VERSION: u32 = 1; -const RLINK_MAGIC: &[u8] = b"rustlink"; - -const RUSTC_VERSION: Option<&str> = option_env!("CFG_VERSION"); - -impl CodegenResults { - pub fn serialize_rlink(codegen_results: &CodegenResults) -> Vec { - let mut encoder = MemEncoder::new(); - encoder.emit_raw_bytes(RLINK_MAGIC); - // `emit_raw_bytes` is used to make sure that the version representation does not depend on - // Encoder's inner representation of `u32`. - encoder.emit_raw_bytes(&RLINK_VERSION.to_be_bytes()); - encoder.emit_str(RUSTC_VERSION.unwrap()); - Encodable::encode(codegen_results, &mut encoder); - encoder.finish() - } - - pub fn deserialize_rlink(data: Vec) -> Result { - // The Decodable machinery is not used here because it panics if the input data is invalid - // and because its internal representation may change. - if !data.starts_with(RLINK_MAGIC) { - return Err("The input does not look like a .rlink file".to_string()); - } - let data = &data[RLINK_MAGIC.len()..]; - if data.len() < 4 { - return Err("The input does not contain version number".to_string()); - } - - let mut version_array: [u8; 4] = Default::default(); - version_array.copy_from_slice(&data[..4]); - if u32::from_be_bytes(version_array) != RLINK_VERSION { - return Err(".rlink file was produced with encoding version {version_array}, but the current version is {RLINK_VERSION}".to_string()); - } - - let mut decoder = MemDecoder::new(&data[4..], 0); - let rustc_version = decoder.read_str(); - let current_version = RUSTC_VERSION.unwrap(); - if rustc_version != current_version { - return Err(format!( - ".rlink file was produced by rustc version {rustc_version}, but the current version is {current_version}." - )); - } - - let codegen_results = CodegenResults::decode(&mut decoder); - Ok(codegen_results) - } -} diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs deleted file mode 100644 index 27d791d90a5..00000000000 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ /dev/null @@ -1,116 +0,0 @@ -use crate::traits::*; - -use rustc_middle::ty::{self, subst::GenericArgKind, ExistentialPredicate, Ty, TyCtxt}; -use rustc_session::config::Lto; -use rustc_symbol_mangling::typeid_for_trait_ref; -use rustc_target::abi::call::FnAbi; - -#[derive(Copy, Clone, Debug)] -pub struct VirtualIndex(u64); - -impl<'a, 'tcx> VirtualIndex { - pub fn from_index(index: usize) -> Self { - VirtualIndex(index as u64) - } - - pub fn get_fn>( - self, - bx: &mut Bx, - llvtable: Bx::Value, - ty: Ty<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - ) -> Bx::Value { - // Load the data pointer from the object. - debug!("get_fn({llvtable:?}, {ty:?}, {self:?})"); - let llty = bx.fn_ptr_backend_type(fn_abi); - let llvtable = bx.pointercast(llvtable, bx.type_ptr_to(llty)); - - if bx.cx().sess().opts.unstable_opts.virtual_function_elimination - && bx.cx().sess().lto() == Lto::Fat - { - let typeid = - bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), get_trait_ref(bx.tcx(), ty))); - let vtable_byte_offset = self.0 * bx.data_layout().pointer_size.bytes(); - let type_checked_load = bx.type_checked_load(llvtable, vtable_byte_offset, typeid); - let func = bx.extract_value(type_checked_load, 0); - bx.pointercast(func, llty) - } else { - let ptr_align = bx.tcx().data_layout.pointer_align.abi; - let gep = bx.inbounds_gep(llty, llvtable, &[bx.const_usize(self.0)]); - let ptr = bx.load(llty, gep, ptr_align); - bx.nonnull_metadata(ptr); - // VTable loads are invariant. - bx.set_invariant_load(ptr); - ptr - } - } - - pub fn get_usize>( - self, - bx: &mut Bx, - llvtable: Bx::Value, - ) -> Bx::Value { - // Load the data pointer from the object. - debug!("get_int({:?}, {:?})", llvtable, self); - - let llty = bx.type_isize(); - let llvtable = bx.pointercast(llvtable, bx.type_ptr_to(llty)); - let usize_align = bx.tcx().data_layout.pointer_align.abi; - let gep = bx.inbounds_gep(llty, llvtable, &[bx.const_usize(self.0)]); - let ptr = bx.load(llty, gep, usize_align); - // VTable loads are invariant. - bx.set_invariant_load(ptr); - ptr - } -} - -fn get_trait_ref<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> { - for arg in ty.peel_refs().walk() { - if let GenericArgKind::Type(ty) = arg.unpack() { - if let ty::Dynamic(trait_refs, _) = ty.kind() { - return trait_refs[0].map_bound(|trait_ref| match trait_ref { - ExistentialPredicate::Trait(tr) => tr, - ExistentialPredicate::Projection(proj) => proj.trait_ref(tcx), - ExistentialPredicate::AutoTrait(_) => { - bug!("auto traits don't have functions") - } - }); - } - } - } - - bug!("expected a `dyn Trait` ty, found {ty:?}") -} - -/// Creates a dynamic vtable for the given type and vtable origin. -/// This is used only for objects. -/// -/// The vtables are cached instead of created on every call. -/// -/// The `trait_ref` encodes the erased self type. Hence if we are -/// making an object `Foo` from a value of type `Foo`, then -/// `trait_ref` would map `T: Trait`. -pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( - cx: &Cx, - ty: Ty<'tcx>, - trait_ref: Option>, -) -> Cx::Value { - let tcx = cx.tcx(); - - debug!("get_vtable(ty={:?}, trait_ref={:?})", ty, trait_ref); - - // Check the cache. - if let Some(&val) = cx.vtables().borrow().get(&(ty, trait_ref)) { - return val; - } - - let vtable_alloc_id = tcx.vtable_allocation((ty, trait_ref)); - let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory(); - let vtable_const = cx.const_data_from_alloc(vtable_allocation); - let align = cx.data_layout().pointer_align.abi; - let vtable = cx.static_addr_of(vtable_const, align, Some("vtable")); - - cx.create_vtable_debuginfo(ty, trait_ref, vtable); - cx.vtables().borrow_mut().insert((ty, trait_ref), vtable); - vtable -} diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs deleted file mode 100644 index 24da48ead63..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ /dev/null @@ -1,368 +0,0 @@ -//! An analysis to determine which locals require allocas and -//! which do not. - -use super::FunctionCx; -use crate::traits::*; -use rustc_data_structures::graph::dominators::Dominators; -use rustc_index::bit_set::BitSet; -use rustc_index::vec::IndexVec; -use rustc_middle::mir::traversal; -use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; -use rustc_middle::mir::{self, Location, TerminatorKind}; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; - -pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - fx: &FunctionCx<'a, 'tcx, Bx>, -) -> BitSet { - let mir = fx.mir; - let dominators = mir.basic_blocks.dominators(); - let locals = mir - .local_decls - .iter() - .map(|decl| { - let ty = fx.monomorphize(decl.ty); - let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span); - if layout.is_zst() { - LocalKind::ZST - } else if fx.cx.is_backend_immediate(layout) || fx.cx.is_backend_scalar_pair(layout) { - LocalKind::Unused - } else { - LocalKind::Memory - } - }) - .collect(); - - let mut analyzer = LocalAnalyzer { fx, dominators, locals }; - - // Arguments get assigned to by means of the function being called - for arg in mir.args_iter() { - analyzer.assign(arg, mir::START_BLOCK.start_location()); - } - - // If there exists a local definition that dominates all uses of that local, - // the definition should be visited first. Traverse blocks in an order that - // is a topological sort of dominance partial order. - for (bb, data) in traversal::reverse_postorder(&mir) { - analyzer.visit_basic_block_data(bb, data); - } - - let mut non_ssa_locals = BitSet::new_empty(analyzer.locals.len()); - for (local, kind) in analyzer.locals.iter_enumerated() { - if matches!(kind, LocalKind::Memory) { - non_ssa_locals.insert(local); - } - } - - non_ssa_locals -} - -#[derive(Copy, Clone, PartialEq, Eq)] -enum LocalKind { - ZST, - /// A local that requires an alloca. - Memory, - /// A scalar or a scalar pair local that is neither defined nor used. - Unused, - /// A scalar or a scalar pair local with a single definition that dominates all uses. - SSA(mir::Location), -} - -struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { - fx: &'mir FunctionCx<'a, 'tcx, Bx>, - dominators: Dominators, - locals: IndexVec, -} - -impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, Bx> { - fn assign(&mut self, local: mir::Local, location: Location) { - let kind = &mut self.locals[local]; - match *kind { - LocalKind::ZST => {} - LocalKind::Memory => {} - LocalKind::Unused => { - *kind = LocalKind::SSA(location); - } - LocalKind::SSA(_) => { - *kind = LocalKind::Memory; - } - } - } - - fn process_place( - &mut self, - place_ref: &mir::PlaceRef<'tcx>, - context: PlaceContext, - location: Location, - ) { - let cx = self.fx.cx; - - if let Some((place_base, elem)) = place_ref.last_projection() { - let mut base_context = if context.is_mutating_use() { - PlaceContext::MutatingUse(MutatingUseContext::Projection) - } else { - PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection) - }; - - // Allow uses of projections that are ZSTs or from scalar fields. - let is_consume = matches!( - context, - PlaceContext::NonMutatingUse( - NonMutatingUseContext::Copy | NonMutatingUseContext::Move, - ) - ); - if is_consume { - let base_ty = place_base.ty(self.fx.mir, cx.tcx()); - let base_ty = self.fx.monomorphize(base_ty); - - // ZSTs don't require any actual memory access. - let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(elem)).ty; - let span = self.fx.mir.local_decls[place_ref.local].source_info.span; - if cx.spanned_layout_of(elem_ty, span).is_zst() { - return; - } - - if let mir::ProjectionElem::Field(..) = elem { - let layout = cx.spanned_layout_of(base_ty.ty, span); - if cx.is_backend_immediate(layout) || cx.is_backend_scalar_pair(layout) { - // Recurse with the same context, instead of `Projection`, - // potentially stopping at non-operand projections, - // which would trigger `not_ssa` on locals. - base_context = context; - } - } - } - - if let mir::ProjectionElem::Deref = elem { - // Deref projections typically only read the pointer. - base_context = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy); - } - - self.process_place(&place_base, base_context, location); - // HACK(eddyb) this emulates the old `visit_projection_elem`, this - // entire `visit_place`-like `process_place` method should be rewritten, - // now that we have moved to the "slice of projections" representation. - if let mir::ProjectionElem::Index(local) = elem { - self.visit_local( - local, - PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy), - location, - ); - } - } else { - self.visit_local(place_ref.local, context, location); - } - } -} - -impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx> - for LocalAnalyzer<'mir, 'a, 'tcx, Bx> -{ - fn visit_assign( - &mut self, - place: &mir::Place<'tcx>, - rvalue: &mir::Rvalue<'tcx>, - location: Location, - ) { - debug!("visit_assign(place={:?}, rvalue={:?})", place, rvalue); - - if let Some(local) = place.as_local() { - self.assign(local, location); - if self.locals[local] != LocalKind::Memory { - let decl_span = self.fx.mir.local_decls[local].source_info.span; - if !self.fx.rvalue_creates_operand(rvalue, decl_span) { - self.locals[local] = LocalKind::Memory; - } - } - } else { - self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), location); - } - - self.visit_rvalue(rvalue, location); - } - - fn visit_place(&mut self, place: &mir::Place<'tcx>, context: PlaceContext, location: Location) { - debug!("visit_place(place={:?}, context={:?})", place, context); - self.process_place(&place.as_ref(), context, location); - } - - fn visit_local(&mut self, local: mir::Local, context: PlaceContext, location: Location) { - match context { - PlaceContext::MutatingUse(MutatingUseContext::Call) - | PlaceContext::MutatingUse(MutatingUseContext::Yield) => { - self.assign(local, location); - } - - PlaceContext::NonUse(_) | PlaceContext::MutatingUse(MutatingUseContext::Retag) => {} - - PlaceContext::NonMutatingUse( - NonMutatingUseContext::Copy | NonMutatingUseContext::Move, - ) => match &mut self.locals[local] { - LocalKind::ZST => {} - LocalKind::Memory => {} - LocalKind::SSA(def) if def.dominates(location, &self.dominators) => {} - // Reads from uninitialized variables (e.g., in dead code, after - // optimizations) require locals to be in (uninitialized) memory. - // N.B., there can be uninitialized reads of a local visited after - // an assignment to that local, if they happen on disjoint paths. - kind @ (LocalKind::Unused | LocalKind::SSA(_)) => { - *kind = LocalKind::Memory; - } - }, - - PlaceContext::MutatingUse( - MutatingUseContext::Store - | MutatingUseContext::Deinit - | MutatingUseContext::SetDiscriminant - | MutatingUseContext::AsmOutput - | MutatingUseContext::Borrow - | MutatingUseContext::AddressOf - | MutatingUseContext::Projection, - ) - | PlaceContext::NonMutatingUse( - NonMutatingUseContext::Inspect - | NonMutatingUseContext::SharedBorrow - | NonMutatingUseContext::UniqueBorrow - | NonMutatingUseContext::ShallowBorrow - | NonMutatingUseContext::AddressOf - | NonMutatingUseContext::Projection, - ) => { - self.locals[local] = LocalKind::Memory; - } - - PlaceContext::MutatingUse(MutatingUseContext::Drop) => { - let kind = &mut self.locals[local]; - if *kind != LocalKind::Memory { - let ty = self.fx.mir.local_decls[local].ty; - let ty = self.fx.monomorphize(ty); - if self.fx.cx.type_needs_drop(ty) { - // Only need the place if we're actually dropping it. - *kind = LocalKind::Memory; - } - } - } - } - } -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum CleanupKind { - NotCleanup, - Funclet, - Internal { funclet: mir::BasicBlock }, -} - -impl CleanupKind { - pub fn funclet_bb(self, for_bb: mir::BasicBlock) -> Option { - match self { - CleanupKind::NotCleanup => None, - CleanupKind::Funclet => Some(for_bb), - CleanupKind::Internal { funclet } => Some(funclet), - } - } -} - -pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec { - fn discover_masters<'tcx>( - result: &mut IndexVec, - mir: &mir::Body<'tcx>, - ) { - for (bb, data) in mir.basic_blocks().iter_enumerated() { - match data.terminator().kind { - TerminatorKind::Goto { .. } - | TerminatorKind::Resume - | TerminatorKind::Abort - | TerminatorKind::Return - | TerminatorKind::GeneratorDrop - | TerminatorKind::Unreachable - | TerminatorKind::SwitchInt { .. } - | TerminatorKind::Yield { .. } - | TerminatorKind::FalseEdge { .. } - | TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ } - TerminatorKind::Call { cleanup: unwind, .. } - | TerminatorKind::InlineAsm { cleanup: unwind, .. } - | TerminatorKind::Assert { cleanup: unwind, .. } - | TerminatorKind::DropAndReplace { unwind, .. } - | TerminatorKind::Drop { unwind, .. } => { - if let Some(unwind) = unwind { - debug!( - "cleanup_kinds: {:?}/{:?} registering {:?} as funclet", - bb, data, unwind - ); - result[unwind] = CleanupKind::Funclet; - } - } - } - } - } - - fn propagate<'tcx>(result: &mut IndexVec, mir: &mir::Body<'tcx>) { - let mut funclet_succs = IndexVec::from_elem(None, mir.basic_blocks()); - - let mut set_successor = |funclet: mir::BasicBlock, succ| match funclet_succs[funclet] { - ref mut s @ None => { - debug!("set_successor: updating successor of {:?} to {:?}", funclet, succ); - *s = Some(succ); - } - Some(s) => { - if s != succ { - span_bug!( - mir.span, - "funclet {:?} has 2 parents - {:?} and {:?}", - funclet, - s, - succ - ); - } - } - }; - - for (bb, data) in traversal::reverse_postorder(mir) { - let funclet = match result[bb] { - CleanupKind::NotCleanup => continue, - CleanupKind::Funclet => bb, - CleanupKind::Internal { funclet } => funclet, - }; - - debug!( - "cleanup_kinds: {:?}/{:?}/{:?} propagating funclet {:?}", - bb, data, result[bb], funclet - ); - - for succ in data.terminator().successors() { - let kind = result[succ]; - debug!("cleanup_kinds: propagating {:?} to {:?}/{:?}", funclet, succ, kind); - match kind { - CleanupKind::NotCleanup => { - result[succ] = CleanupKind::Internal { funclet }; - } - CleanupKind::Funclet => { - if funclet != succ { - set_successor(funclet, succ); - } - } - CleanupKind::Internal { funclet: succ_funclet } => { - if funclet != succ_funclet { - // `succ` has 2 different funclet going into it, so it must - // be a funclet by itself. - - debug!( - "promoting {:?} to a funclet and updating {:?}", - succ, succ_funclet - ); - result[succ] = CleanupKind::Funclet; - set_successor(succ_funclet, succ); - set_successor(funclet, succ); - } - } - } - } - } - } - - let mut result = IndexVec::from_elem(CleanupKind::NotCleanup, mir.basic_blocks()); - - discover_masters(&mut result, mir); - propagate(&mut result, mir); - debug!("cleanup_kinds: result={:?}", result); - result -} diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs deleted file mode 100644 index 3eee58d9d1c..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ /dev/null @@ -1,1654 +0,0 @@ -use super::operand::OperandRef; -use super::operand::OperandValue::{Immediate, Pair, Ref}; -use super::place::PlaceRef; -use super::{FunctionCx, LocalRef}; - -use crate::base; -use crate::common::{self, IntPredicate}; -use crate::meth; -use crate::traits::*; -use crate::MemFlags; - -use rustc_ast as ast; -use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; -use rustc_hir::lang_items::LangItem; -use rustc_index::vec::Idx; -use rustc_middle::mir::AssertKind; -use rustc_middle::mir::{self, SwitchTargets}; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; -use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; -use rustc_middle::ty::{self, Instance, Ty, TypeVisitable}; -use rustc_span::source_map::Span; -use rustc_span::{sym, Symbol}; -use rustc_symbol_mangling::typeid::typeid_for_fnabi; -use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode}; -use rustc_target::abi::{self, HasDataLayout, WrappingRange}; -use rustc_target::spec::abi::Abi; - -/// Used by `FunctionCx::codegen_terminator` for emitting common patterns -/// e.g., creating a basic block, calling a function, etc. -struct TerminatorCodegenHelper<'tcx> { - bb: mir::BasicBlock, - terminator: &'tcx mir::Terminator<'tcx>, - funclet_bb: Option, -} - -impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { - /// Returns the appropriate `Funclet` for the current funclet, if on MSVC, - /// either already previously cached, or newly created, by `landing_pad_for`. - fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>( - &self, - fx: &'b mut FunctionCx<'a, 'tcx, Bx>, - ) -> Option<&'b Bx::Funclet> { - let funclet_bb = self.funclet_bb?; - if base::wants_msvc_seh(fx.cx.tcx().sess) { - // If `landing_pad_for` hasn't been called yet to create the `Funclet`, - // it has to be now. This may not seem necessary, as RPO should lead - // to all the unwind edges being visited (and so to `landing_pad_for` - // getting called for them), before building any of the blocks inside - // the funclet itself - however, if MIR contains edges that end up not - // being needed in the LLVM IR after monomorphization, the funclet may - // be unreachable, and we don't have yet a way to skip building it in - // such an eventuality (which may be a better solution than this). - if fx.funclets[funclet_bb].is_none() { - fx.landing_pad_for(funclet_bb); - } - - Some( - fx.funclets[funclet_bb] - .as_ref() - .expect("landing_pad_for didn't also create funclets entry"), - ) - } else { - None - } - } - - fn lltarget>( - &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, - target: mir::BasicBlock, - ) -> (Bx::BasicBlock, bool) { - let span = self.terminator.source_info.span; - let lltarget = fx.llbb(target); - let target_funclet = fx.cleanup_kinds[target].funclet_bb(target); - match (self.funclet_bb, target_funclet) { - (None, None) => (lltarget, false), - (Some(f), Some(t_f)) if f == t_f || !base::wants_msvc_seh(fx.cx.tcx().sess) => { - (lltarget, false) - } - // jump *into* cleanup - need a landing pad if GNU, cleanup pad if MSVC - (None, Some(_)) => (fx.landing_pad_for(target), false), - (Some(_), None) => span_bug!(span, "{:?} - jump out of cleanup?", self.terminator), - (Some(_), Some(_)) => (fx.landing_pad_for(target), true), - } - } - - /// Create a basic block. - fn llblock>( - &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, - target: mir::BasicBlock, - ) -> Bx::BasicBlock { - let (lltarget, is_cleanupret) = self.lltarget(fx, target); - if is_cleanupret { - // MSVC cross-funclet jump - need a trampoline - - debug!("llblock: creating cleanup trampoline for {:?}", target); - let name = &format!("{:?}_cleanup_trampoline_{:?}", self.bb, target); - let trampoline = Bx::append_block(fx.cx, fx.llfn, name); - let mut trampoline_bx = Bx::build(fx.cx, trampoline); - trampoline_bx.cleanup_ret(self.funclet(fx).unwrap(), Some(lltarget)); - trampoline - } else { - lltarget - } - } - - fn funclet_br>( - &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, - bx: &mut Bx, - target: mir::BasicBlock, - ) { - let (lltarget, is_cleanupret) = self.lltarget(fx, target); - if is_cleanupret { - // micro-optimization: generate a `ret` rather than a jump - // to a trampoline. - bx.cleanup_ret(self.funclet(fx).unwrap(), Some(lltarget)); - } else { - bx.br(lltarget); - } - } - - /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional - /// return destination `destination` and the cleanup function `cleanup`. - fn do_call>( - &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, - bx: &mut Bx, - fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, - fn_ptr: Bx::Value, - llargs: &[Bx::Value], - destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>, - cleanup: Option, - copied_constant_arguments: &[PlaceRef<'tcx, ::Value>], - ) { - // If there is a cleanup block and the function we're calling can unwind, then - // do an invoke, otherwise do a call. - let fn_ty = bx.fn_decl_backend_type(&fn_abi); - - let unwind_block = if let Some(cleanup) = cleanup.filter(|_| fn_abi.can_unwind) { - Some(self.llblock(fx, cleanup)) - } else if fx.mir[self.bb].is_cleanup - && fn_abi.can_unwind - && !base::wants_msvc_seh(fx.cx.tcx().sess) - { - // Exception must not propagate out of the execution of a cleanup (doing so - // can cause undefined behaviour). We insert a double unwind guard for - // functions that can potentially unwind to protect against this. - // - // This is not necessary for SEH which does not use successive unwinding - // like Itanium EH. EH frames in SEH are different from normal function - // frames and SEH will abort automatically if an exception tries to - // propagate out from cleanup. - Some(fx.double_unwind_guard()) - } else { - None - }; - - if let Some(unwind_block) = unwind_block { - let ret_llbb = if let Some((_, target)) = destination { - fx.llbb(target) - } else { - fx.unreachable_block() - }; - let invokeret = - bx.invoke(fn_ty, fn_ptr, &llargs, ret_llbb, unwind_block, self.funclet(fx)); - bx.apply_attrs_callsite(&fn_abi, invokeret); - if fx.mir[self.bb].is_cleanup { - bx.do_not_inline(invokeret); - } - - if let Some((ret_dest, target)) = destination { - bx.switch_to_block(fx.llbb(target)); - fx.set_debug_loc(bx, self.terminator.source_info); - for tmp in copied_constant_arguments { - bx.lifetime_end(tmp.llval, tmp.layout.size); - } - fx.store_return(bx, ret_dest, &fn_abi.ret, invokeret); - } - } else { - let llret = bx.call(fn_ty, fn_ptr, &llargs, self.funclet(fx)); - bx.apply_attrs_callsite(&fn_abi, llret); - if fx.mir[self.bb].is_cleanup { - // Cleanup is always the cold path. Don't inline - // drop glue. Also, when there is a deeply-nested - // struct, there are "symmetry" issues that cause - // exponential inlining - see issue #41696. - bx.do_not_inline(llret); - } - - if let Some((ret_dest, target)) = destination { - for tmp in copied_constant_arguments { - bx.lifetime_end(tmp.llval, tmp.layout.size); - } - fx.store_return(bx, ret_dest, &fn_abi.ret, llret); - self.funclet_br(fx, bx, target); - } else { - bx.unreachable(); - } - } - } - - /// Generates inline assembly with optional `destination` and `cleanup`. - fn do_inlineasm>( - &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, - bx: &mut Bx, - template: &[InlineAsmTemplatePiece], - operands: &[InlineAsmOperandRef<'tcx, Bx>], - options: InlineAsmOptions, - line_spans: &[Span], - destination: Option, - cleanup: Option, - instance: Instance<'_>, - ) { - if let Some(cleanup) = cleanup { - let ret_llbb = if let Some(target) = destination { - fx.llbb(target) - } else { - fx.unreachable_block() - }; - - bx.codegen_inline_asm( - template, - &operands, - options, - line_spans, - instance, - Some((ret_llbb, self.llblock(fx, cleanup), self.funclet(fx))), - ); - } else { - bx.codegen_inline_asm(template, &operands, options, line_spans, instance, None); - - if let Some(target) = destination { - self.funclet_br(fx, bx, target); - } else { - bx.unreachable(); - } - } - } -} - -/// Codegen implementations for some terminator variants. -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - /// Generates code for a `Resume` terminator. - fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, mut bx: Bx) { - if let Some(funclet) = helper.funclet(self) { - bx.cleanup_ret(funclet, None); - } else { - let slot = self.get_personality_slot(&mut bx); - let lp0 = slot.project_field(&mut bx, 0); - let lp0 = bx.load_operand(lp0).immediate(); - let lp1 = slot.project_field(&mut bx, 1); - let lp1 = bx.load_operand(lp1).immediate(); - slot.storage_dead(&mut bx); - - let mut lp = bx.const_undef(self.landing_pad_type()); - lp = bx.insert_value(lp, lp0, 0); - lp = bx.insert_value(lp, lp1, 1); - bx.resume(lp); - } - } - - fn codegen_switchint_terminator( - &mut self, - helper: TerminatorCodegenHelper<'tcx>, - mut bx: Bx, - discr: &mir::Operand<'tcx>, - switch_ty: Ty<'tcx>, - targets: &SwitchTargets, - ) { - let discr = self.codegen_operand(&mut bx, &discr); - // `switch_ty` is redundant, sanity-check that. - assert_eq!(discr.layout.ty, switch_ty); - let mut target_iter = targets.iter(); - if target_iter.len() == 1 { - // If there are two targets (one conditional, one fallback), emit br instead of switch - let (test_value, target) = target_iter.next().unwrap(); - let lltrue = helper.llblock(self, target); - let llfalse = helper.llblock(self, targets.otherwise()); - if switch_ty == bx.tcx().types.bool { - // Don't generate trivial icmps when switching on bool - match test_value { - 0 => bx.cond_br(discr.immediate(), llfalse, lltrue), - 1 => bx.cond_br(discr.immediate(), lltrue, llfalse), - _ => bug!(), - } - } else { - let switch_llty = bx.immediate_backend_type(bx.layout_of(switch_ty)); - let llval = bx.const_uint_big(switch_llty, test_value); - let cmp = bx.icmp(IntPredicate::IntEQ, discr.immediate(), llval); - bx.cond_br(cmp, lltrue, llfalse); - } - } else { - bx.switch( - discr.immediate(), - helper.llblock(self, targets.otherwise()), - target_iter.map(|(value, target)| (value, helper.llblock(self, target))), - ); - } - } - - fn codegen_return_terminator(&mut self, mut bx: Bx) { - // Call `va_end` if this is the definition of a C-variadic function. - if self.fn_abi.c_variadic { - // The `VaList` "spoofed" argument is just after all the real arguments. - let va_list_arg_idx = self.fn_abi.args.len(); - match self.locals[mir::Local::new(1 + va_list_arg_idx)] { - LocalRef::Place(va_list) => { - bx.va_end(va_list.llval); - } - _ => bug!("C-variadic function must have a `VaList` place"), - } - } - if self.fn_abi.ret.layout.abi.is_uninhabited() { - // Functions with uninhabited return values are marked `noreturn`, - // so we should make sure that we never actually do. - // We play it safe by using a well-defined `abort`, but we could go for immediate UB - // if that turns out to be helpful. - bx.abort(); - // `abort` does not terminate the block, so we still need to generate - // an `unreachable` terminator after it. - bx.unreachable(); - return; - } - let llval = match self.fn_abi.ret.mode { - PassMode::Ignore | PassMode::Indirect { .. } => { - bx.ret_void(); - return; - } - - PassMode::Direct(_) | PassMode::Pair(..) => { - let op = self.codegen_consume(&mut bx, mir::Place::return_place().as_ref()); - if let Ref(llval, _, align) = op.val { - bx.load(bx.backend_type(op.layout), llval, align) - } else { - op.immediate_or_packed_pair(&mut bx) - } - } - - PassMode::Cast(cast_ty) => { - let op = match self.locals[mir::RETURN_PLACE] { - LocalRef::Operand(Some(op)) => op, - LocalRef::Operand(None) => bug!("use of return before def"), - LocalRef::Place(cg_place) => OperandRef { - val: Ref(cg_place.llval, None, cg_place.align), - layout: cg_place.layout, - }, - LocalRef::UnsizedPlace(_) => bug!("return type must be sized"), - }; - let llslot = match op.val { - Immediate(_) | Pair(..) => { - let scratch = PlaceRef::alloca(&mut bx, self.fn_abi.ret.layout); - op.val.store(&mut bx, scratch); - scratch.llval - } - Ref(llval, _, align) => { - assert_eq!(align, op.layout.align.abi, "return place is unaligned!"); - llval - } - }; - let ty = bx.cast_backend_type(&cast_ty); - let addr = bx.pointercast(llslot, bx.type_ptr_to(ty)); - bx.load(ty, addr, self.fn_abi.ret.layout.align.abi) - } - }; - bx.ret(llval); - } - - fn codegen_drop_terminator( - &mut self, - helper: TerminatorCodegenHelper<'tcx>, - mut bx: Bx, - location: mir::Place<'tcx>, - target: mir::BasicBlock, - unwind: Option, - ) { - let ty = location.ty(self.mir, bx.tcx()).ty; - let ty = self.monomorphize(ty); - let drop_fn = Instance::resolve_drop_in_place(bx.tcx(), ty); - - if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def { - // we don't actually need to drop anything. - helper.funclet_br(self, &mut bx, target); - return; - } - - let place = self.codegen_place(&mut bx, location.as_ref()); - let (args1, args2); - let mut args = if let Some(llextra) = place.llextra { - args2 = [place.llval, llextra]; - &args2[..] - } else { - args1 = [place.llval]; - &args1[..] - }; - let (drop_fn, fn_abi) = match ty.kind() { - // FIXME(eddyb) perhaps move some of this logic into - // `Instance::resolve_drop_in_place`? - ty::Dynamic(..) => { - let virtual_drop = Instance { - def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), - substs: drop_fn.substs, - }; - let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty()); - let vtable = args[1]; - args = &args[..1]; - ( - meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE) - .get_fn(&mut bx, vtable, ty, &fn_abi), - fn_abi, - ) - } - _ => (bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty())), - }; - helper.do_call( - self, - &mut bx, - fn_abi, - drop_fn, - args, - Some((ReturnDest::Nothing, target)), - unwind, - &[], - ); - } - - fn codegen_assert_terminator( - &mut self, - helper: TerminatorCodegenHelper<'tcx>, - mut bx: Bx, - terminator: &mir::Terminator<'tcx>, - cond: &mir::Operand<'tcx>, - expected: bool, - msg: &mir::AssertMessage<'tcx>, - target: mir::BasicBlock, - cleanup: Option, - ) { - let span = terminator.source_info.span; - let cond = self.codegen_operand(&mut bx, cond).immediate(); - let mut const_cond = bx.const_to_opt_u128(cond, false).map(|c| c == 1); - - // This case can currently arise only from functions marked - // with #[rustc_inherit_overflow_checks] and inlined from - // another crate (mostly core::num generic/#[inline] fns), - // while the current crate doesn't use overflow checks. - // NOTE: Unlike binops, negation doesn't have its own - // checked operation, just a comparison with the minimum - // value, so we have to check for the assert message. - if !bx.check_overflow() { - if let AssertKind::OverflowNeg(_) = *msg { - const_cond = Some(expected); - } - } - - // Don't codegen the panic block if success if known. - if const_cond == Some(expected) { - helper.funclet_br(self, &mut bx, target); - return; - } - - // Pass the condition through llvm.expect for branch hinting. - let cond = bx.expect(cond, expected); - - // Create the failure block and the conditional branch to it. - let lltarget = helper.llblock(self, target); - let panic_block = bx.append_sibling_block("panic"); - if expected { - bx.cond_br(cond, lltarget, panic_block); - } else { - bx.cond_br(cond, panic_block, lltarget); - } - - // After this point, bx is the block for the call to panic. - bx.switch_to_block(panic_block); - self.set_debug_loc(&mut bx, terminator.source_info); - - // Get the location information. - let location = self.get_caller_location(&mut bx, terminator.source_info).immediate(); - - // Put together the arguments to the panic entry point. - let (lang_item, args) = match msg { - AssertKind::BoundsCheck { ref len, ref index } => { - let len = self.codegen_operand(&mut bx, len).immediate(); - let index = self.codegen_operand(&mut bx, index).immediate(); - // It's `fn panic_bounds_check(index: usize, len: usize)`, - // and `#[track_caller]` adds an implicit third argument. - (LangItem::PanicBoundsCheck, vec![index, len, location]) - } - _ => { - let msg = bx.const_str(msg.description()); - // It's `pub fn panic(expr: &str)`, with the wide reference being passed - // as two arguments, and `#[track_caller]` adds an implicit third argument. - (LangItem::Panic, vec![msg.0, msg.1, location]) - } - }; - - let (fn_abi, llfn) = common::build_langcall(&bx, Some(span), lang_item); - - // Codegen the actual panic invoke/call. - helper.do_call(self, &mut bx, fn_abi, llfn, &args, None, cleanup, &[]); - } - - fn codegen_abort_terminator( - &mut self, - helper: TerminatorCodegenHelper<'tcx>, - mut bx: Bx, - terminator: &mir::Terminator<'tcx>, - ) { - let span = terminator.source_info.span; - self.set_debug_loc(&mut bx, terminator.source_info); - - // Obtain the panic entry point. - let (fn_abi, llfn) = common::build_langcall(&bx, Some(span), LangItem::PanicNoUnwind); - - // Codegen the actual panic invoke/call. - helper.do_call(self, &mut bx, fn_abi, llfn, &[], None, None, &[]); - } - - /// Returns `true` if this is indeed a panic intrinsic and codegen is done. - fn codegen_panic_intrinsic( - &mut self, - helper: &TerminatorCodegenHelper<'tcx>, - bx: &mut Bx, - intrinsic: Option, - instance: Option>, - source_info: mir::SourceInfo, - target: Option, - cleanup: Option, - ) -> bool { - // Emit a panic or a no-op for `assert_*` intrinsics. - // These are intrinsics that compile to panics so that we can get a message - // which mentions the offending type, even from a const context. - #[derive(Debug, PartialEq)] - enum AssertIntrinsic { - Inhabited, - ZeroValid, - UninitValid, - } - let panic_intrinsic = intrinsic.and_then(|i| match i { - sym::assert_inhabited => Some(AssertIntrinsic::Inhabited), - sym::assert_zero_valid => Some(AssertIntrinsic::ZeroValid), - sym::assert_uninit_valid => Some(AssertIntrinsic::UninitValid), - _ => None, - }); - if let Some(intrinsic) = panic_intrinsic { - use AssertIntrinsic::*; - - let ty = instance.unwrap().substs.type_at(0); - let layout = bx.layout_of(ty); - let do_panic = match intrinsic { - Inhabited => layout.abi.is_uninhabited(), - ZeroValid => !bx.tcx().permits_zero_init(layout), - UninitValid => !bx.tcx().permits_uninit_init(layout), - }; - if do_panic { - let msg_str = with_no_visible_paths!({ - with_no_trimmed_paths!({ - if layout.abi.is_uninhabited() { - // Use this error even for the other intrinsics as it is more precise. - format!("attempted to instantiate uninhabited type `{}`", ty) - } else if intrinsic == ZeroValid { - format!("attempted to zero-initialize type `{}`, which is invalid", ty) - } else { - format!( - "attempted to leave type `{}` uninitialized, which is invalid", - ty - ) - } - }) - }); - let msg = bx.const_str(&msg_str); - let location = self.get_caller_location(bx, source_info).immediate(); - - // Obtain the panic entry point. - let (fn_abi, llfn) = - common::build_langcall(bx, Some(source_info.span), LangItem::Panic); - - // Codegen the actual panic invoke/call. - helper.do_call( - self, - bx, - fn_abi, - llfn, - &[msg.0, msg.1, location], - target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)), - cleanup, - &[], - ); - } else { - // a NOP - let target = target.unwrap(); - helper.funclet_br(self, bx, target) - } - true - } else { - false - } - } - - fn codegen_call_terminator( - &mut self, - helper: TerminatorCodegenHelper<'tcx>, - mut bx: Bx, - terminator: &mir::Terminator<'tcx>, - func: &mir::Operand<'tcx>, - args: &[mir::Operand<'tcx>], - destination: mir::Place<'tcx>, - target: Option, - cleanup: Option, - fn_span: Span, - ) { - let source_info = terminator.source_info; - let span = source_info.span; - - // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar. - let callee = self.codegen_operand(&mut bx, func); - - let (instance, mut llfn) = match *callee.layout.ty.kind() { - ty::FnDef(def_id, substs) => ( - Some( - ty::Instance::resolve(bx.tcx(), ty::ParamEnv::reveal_all(), def_id, substs) - .unwrap() - .unwrap() - .polymorphize(bx.tcx()), - ), - None, - ), - ty::FnPtr(_) => (None, Some(callee.immediate())), - _ => bug!("{} is not callable", callee.layout.ty), - }; - let def = instance.map(|i| i.def); - - if let Some(ty::InstanceDef::DropGlue(_, None)) = def { - // Empty drop glue; a no-op. - let target = target.unwrap(); - helper.funclet_br(self, &mut bx, target); - return; - } - - // FIXME(eddyb) avoid computing this if possible, when `instance` is - // available - right now `sig` is only needed for getting the `abi` - // and figuring out how many extra args were passed to a C-variadic `fn`. - let sig = callee.layout.ty.fn_sig(bx.tcx()); - let abi = sig.abi(); - - // Handle intrinsics old codegen wants Expr's for, ourselves. - let intrinsic = match def { - Some(ty::InstanceDef::Intrinsic(def_id)) => Some(bx.tcx().item_name(def_id)), - _ => None, - }; - - let extra_args = &args[sig.inputs().skip_binder().len()..]; - let extra_args = bx.tcx().mk_type_list(extra_args.iter().map(|op_arg| { - let op_ty = op_arg.ty(self.mir, bx.tcx()); - self.monomorphize(op_ty) - })); - - let fn_abi = match instance { - Some(instance) => bx.fn_abi_of_instance(instance, extra_args), - None => bx.fn_abi_of_fn_ptr(sig, extra_args), - }; - - if intrinsic == Some(sym::transmute) { - if let Some(target) = target { - self.codegen_transmute(&mut bx, &args[0], destination); - helper.funclet_br(self, &mut bx, target); - } else { - // If we are trying to transmute to an uninhabited type, - // it is likely there is no allotted destination. In fact, - // transmuting to an uninhabited type is UB, which means - // we can do what we like. Here, we declare that transmuting - // into an uninhabited type is impossible, so anything following - // it must be unreachable. - assert_eq!(fn_abi.ret.layout.abi, abi::Abi::Uninhabited); - bx.unreachable(); - } - return; - } - - if self.codegen_panic_intrinsic( - &helper, - &mut bx, - intrinsic, - instance, - source_info, - target, - cleanup, - ) { - return; - } - - // The arguments we'll be passing. Plus one to account for outptr, if used. - let arg_count = fn_abi.args.len() + fn_abi.ret.is_indirect() as usize; - let mut llargs = Vec::with_capacity(arg_count); - - // Prepare the return value destination - let ret_dest = if target.is_some() { - let is_intrinsic = intrinsic.is_some(); - self.make_return_dest(&mut bx, destination, &fn_abi.ret, &mut llargs, is_intrinsic) - } else { - ReturnDest::Nothing - }; - - if intrinsic == Some(sym::caller_location) { - if let Some(target) = target { - let location = self - .get_caller_location(&mut bx, mir::SourceInfo { span: fn_span, ..source_info }); - - if let ReturnDest::IndirectOperand(tmp, _) = ret_dest { - location.val.store(&mut bx, tmp); - } - self.store_return(&mut bx, ret_dest, &fn_abi.ret, location.immediate()); - helper.funclet_br(self, &mut bx, target); - } - return; - } - - match intrinsic { - None | Some(sym::drop_in_place) => {} - Some(sym::copy_nonoverlapping) => unreachable!(), - Some(intrinsic) => { - let dest = match ret_dest { - _ if fn_abi.ret.is_indirect() => llargs[0], - ReturnDest::Nothing => { - bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret))) - } - ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval, - ReturnDest::DirectOperand(_) => { - bug!("Cannot use direct operand with an intrinsic call") - } - }; - - let args: Vec<_> = args - .iter() - .enumerate() - .map(|(i, arg)| { - // The indices passed to simd_shuffle* in the - // third argument must be constant. This is - // checked by const-qualification, which also - // promotes any complex rvalues to constants. - if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") { - if let mir::Operand::Constant(constant) = arg { - let c = self.eval_mir_constant(constant); - let (llval, ty) = self.simd_shuffle_indices( - &bx, - constant.span, - self.monomorphize(constant.ty()), - c, - ); - return OperandRef { - val: Immediate(llval), - layout: bx.layout_of(ty), - }; - } else { - span_bug!(span, "shuffle indices must be constant"); - } - } - - self.codegen_operand(&mut bx, arg) - }) - .collect(); - - Self::codegen_intrinsic_call( - &mut bx, - *instance.as_ref().unwrap(), - &fn_abi, - &args, - dest, - span, - ); - - if let ReturnDest::IndirectOperand(dst, _) = ret_dest { - self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval); - } - - if let Some(target) = target { - helper.funclet_br(self, &mut bx, target); - } else { - bx.unreachable(); - } - - return; - } - } - - // Split the rust-call tupled arguments off. - let (first_args, untuple) = if abi == Abi::RustCall && !args.is_empty() { - let (tup, args) = args.split_last().unwrap(); - (args, Some(tup)) - } else { - (args, None) - }; - - let mut copied_constant_arguments = vec![]; - 'make_args: for (i, arg) in first_args.iter().enumerate() { - let mut op = self.codegen_operand(&mut bx, arg); - - if let (0, Some(ty::InstanceDef::Virtual(_, idx))) = (i, def) { - if let Pair(..) = op.val { - // In the case of Rc, we need to explicitly pass a - // *mut RcBox with a Scalar (not ScalarPair) ABI. This is a hack - // that is understood elsewhere in the compiler as a method on - // `dyn Trait`. - // To get a `*mut RcBox`, we just keep unwrapping newtypes until - // we get a value of a built-in pointer type - 'descend_newtypes: while !op.layout.ty.is_unsafe_ptr() - && !op.layout.ty.is_region_ptr() - { - for i in 0..op.layout.fields.count() { - let field = op.extract_field(&mut bx, i); - if !field.layout.is_zst() { - // we found the one non-zero-sized field that is allowed - // now find *its* non-zero-sized field, or stop if it's a - // pointer - op = field; - continue 'descend_newtypes; - } - } - - span_bug!(span, "receiver has no non-zero-sized fields {:?}", op); - } - - // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its - // data pointer and vtable. Look up the method in the vtable, and pass - // the data pointer as the first argument - match op.val { - Pair(data_ptr, meta) => { - llfn = Some(meth::VirtualIndex::from_index(idx).get_fn( - &mut bx, - meta, - op.layout.ty, - &fn_abi, - )); - llargs.push(data_ptr); - continue 'make_args; - } - other => bug!("expected a Pair, got {:?}", other), - } - } else if let Ref(data_ptr, Some(meta), _) = op.val { - // by-value dynamic dispatch - llfn = Some(meth::VirtualIndex::from_index(idx).get_fn( - &mut bx, - meta, - op.layout.ty, - &fn_abi, - )); - llargs.push(data_ptr); - continue; - } else { - span_bug!(span, "can't codegen a virtual call on {:?}", op); - } - } - - // The callee needs to own the argument memory if we pass it - // by-ref, so make a local copy of non-immediate constants. - match (arg, op.val) { - (&mir::Operand::Copy(_), Ref(_, None, _)) - | (&mir::Operand::Constant(_), Ref(_, None, _)) => { - let tmp = PlaceRef::alloca(&mut bx, op.layout); - bx.lifetime_start(tmp.llval, tmp.layout.size); - op.val.store(&mut bx, tmp); - op.val = Ref(tmp.llval, None, tmp.align); - copied_constant_arguments.push(tmp); - } - _ => {} - } - - self.codegen_argument(&mut bx, op, &mut llargs, &fn_abi.args[i]); - } - let num_untupled = untuple.map(|tup| { - self.codegen_arguments_untupled( - &mut bx, - tup, - &mut llargs, - &fn_abi.args[first_args.len()..], - ) - }); - - let needs_location = - instance.map_or(false, |i| i.def.requires_caller_location(self.cx.tcx())); - if needs_location { - let mir_args = if let Some(num_untupled) = num_untupled { - first_args.len() + num_untupled - } else { - args.len() - }; - assert_eq!( - fn_abi.args.len(), - mir_args + 1, - "#[track_caller] fn's must have 1 more argument in their ABI than in their MIR: {:?} {:?} {:?}", - instance, - fn_span, - fn_abi, - ); - let location = - self.get_caller_location(&mut bx, mir::SourceInfo { span: fn_span, ..source_info }); - debug!( - "codegen_call_terminator({:?}): location={:?} (fn_span {:?})", - terminator, location, fn_span - ); - - let last_arg = fn_abi.args.last().unwrap(); - self.codegen_argument(&mut bx, location, &mut llargs, last_arg); - } - - let (is_indirect_call, fn_ptr) = match (llfn, instance) { - (Some(llfn), _) => (true, llfn), - (None, Some(instance)) => (false, bx.get_fn_addr(instance)), - _ => span_bug!(span, "no llfn for call"), - }; - - // For backends that support CFI using type membership (i.e., testing whether a given - // pointer is associated with a type identifier). - if bx.tcx().sess.is_sanitizer_cfi_enabled() && is_indirect_call { - // Emit type metadata and checks. - // FIXME(rcvalle): Add support for generalized identifiers. - // FIXME(rcvalle): Create distinct unnamed MDNodes for internal identifiers. - let typeid = typeid_for_fnabi(bx.tcx(), fn_abi); - let typeid_metadata = self.cx.typeid_metadata(typeid); - - // Test whether the function pointer is associated with the type identifier. - let cond = bx.type_test(fn_ptr, typeid_metadata); - let bb_pass = bx.append_sibling_block("type_test.pass"); - let bb_fail = bx.append_sibling_block("type_test.fail"); - bx.cond_br(cond, bb_pass, bb_fail); - - bx.switch_to_block(bb_pass); - helper.do_call( - self, - &mut bx, - fn_abi, - fn_ptr, - &llargs, - target.as_ref().map(|&target| (ret_dest, target)), - cleanup, - &copied_constant_arguments, - ); - - bx.switch_to_block(bb_fail); - bx.abort(); - bx.unreachable(); - - return; - } - - helper.do_call( - self, - &mut bx, - fn_abi, - fn_ptr, - &llargs, - target.as_ref().map(|&target| (ret_dest, target)), - cleanup, - &copied_constant_arguments, - ); - } - - fn codegen_asm_terminator( - &mut self, - helper: TerminatorCodegenHelper<'tcx>, - mut bx: Bx, - terminator: &mir::Terminator<'tcx>, - template: &[ast::InlineAsmTemplatePiece], - operands: &[mir::InlineAsmOperand<'tcx>], - options: ast::InlineAsmOptions, - line_spans: &[Span], - destination: Option, - cleanup: Option, - instance: Instance<'_>, - ) { - let span = terminator.source_info.span; - - let operands: Vec<_> = operands - .iter() - .map(|op| match *op { - mir::InlineAsmOperand::In { reg, ref value } => { - let value = self.codegen_operand(&mut bx, value); - InlineAsmOperandRef::In { reg, value } - } - mir::InlineAsmOperand::Out { reg, late, ref place } => { - let place = place.map(|place| self.codegen_place(&mut bx, place.as_ref())); - InlineAsmOperandRef::Out { reg, late, place } - } - mir::InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => { - let in_value = self.codegen_operand(&mut bx, in_value); - let out_place = - out_place.map(|out_place| self.codegen_place(&mut bx, out_place.as_ref())); - InlineAsmOperandRef::InOut { reg, late, in_value, out_place } - } - mir::InlineAsmOperand::Const { ref value } => { - let const_value = self - .eval_mir_constant(value) - .unwrap_or_else(|_| span_bug!(span, "asm const cannot be resolved")); - let string = common::asm_const_to_str( - bx.tcx(), - span, - const_value, - bx.layout_of(value.ty()), - ); - InlineAsmOperandRef::Const { string } - } - mir::InlineAsmOperand::SymFn { ref value } => { - let literal = self.monomorphize(value.literal); - if let ty::FnDef(def_id, substs) = *literal.ty().kind() { - let instance = ty::Instance::resolve_for_fn_ptr( - bx.tcx(), - ty::ParamEnv::reveal_all(), - def_id, - substs, - ) - .unwrap(); - InlineAsmOperandRef::SymFn { instance } - } else { - span_bug!(span, "invalid type for asm sym (fn)"); - } - } - mir::InlineAsmOperand::SymStatic { def_id } => { - InlineAsmOperandRef::SymStatic { def_id } - } - }) - .collect(); - - helper.do_inlineasm( - self, - &mut bx, - template, - &operands, - options, - line_spans, - destination, - cleanup, - instance, - ); - } -} - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - pub fn codegen_block(&mut self, bb: mir::BasicBlock) { - let llbb = self.llbb(bb); - let mut bx = Bx::build(self.cx, llbb); - let mir = self.mir; - let data = &mir[bb]; - - debug!("codegen_block({:?}={:?})", bb, data); - - for statement in &data.statements { - bx = self.codegen_statement(bx, statement); - } - - self.codegen_terminator(bx, bb, data.terminator()); - } - - fn codegen_terminator( - &mut self, - mut bx: Bx, - bb: mir::BasicBlock, - terminator: &'tcx mir::Terminator<'tcx>, - ) { - debug!("codegen_terminator: {:?}", terminator); - - // Create the cleanup bundle, if needed. - let funclet_bb = self.cleanup_kinds[bb].funclet_bb(bb); - let helper = TerminatorCodegenHelper { bb, terminator, funclet_bb }; - - self.set_debug_loc(&mut bx, terminator.source_info); - match terminator.kind { - mir::TerminatorKind::Resume => self.codegen_resume_terminator(helper, bx), - - mir::TerminatorKind::Abort => { - self.codegen_abort_terminator(helper, bx, terminator); - } - - mir::TerminatorKind::Goto { target } => { - helper.funclet_br(self, &mut bx, target); - } - - mir::TerminatorKind::SwitchInt { ref discr, switch_ty, ref targets } => { - self.codegen_switchint_terminator(helper, bx, discr, switch_ty, targets); - } - - mir::TerminatorKind::Return => { - self.codegen_return_terminator(bx); - } - - mir::TerminatorKind::Unreachable => { - bx.unreachable(); - } - - mir::TerminatorKind::Drop { place, target, unwind } => { - self.codegen_drop_terminator(helper, bx, place, target, unwind); - } - - mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, cleanup } => { - self.codegen_assert_terminator( - helper, bx, terminator, cond, expected, msg, target, cleanup, - ); - } - - mir::TerminatorKind::DropAndReplace { .. } => { - bug!("undesugared DropAndReplace in codegen: {:?}", terminator); - } - - mir::TerminatorKind::Call { - ref func, - ref args, - destination, - target, - cleanup, - from_hir_call: _, - fn_span, - } => { - self.codegen_call_terminator( - helper, - bx, - terminator, - func, - args, - destination, - target, - cleanup, - fn_span, - ); - } - mir::TerminatorKind::GeneratorDrop | mir::TerminatorKind::Yield { .. } => { - bug!("generator ops in codegen") - } - mir::TerminatorKind::FalseEdge { .. } | mir::TerminatorKind::FalseUnwind { .. } => { - bug!("borrowck false edges in codegen") - } - - mir::TerminatorKind::InlineAsm { - template, - ref operands, - options, - line_spans, - destination, - cleanup, - } => { - self.codegen_asm_terminator( - helper, - bx, - terminator, - template, - operands, - options, - line_spans, - destination, - cleanup, - self.instance, - ); - } - } - } - - fn codegen_argument( - &mut self, - bx: &mut Bx, - op: OperandRef<'tcx, Bx::Value>, - llargs: &mut Vec, - arg: &ArgAbi<'tcx, Ty<'tcx>>, - ) { - // Fill padding with undef value, where applicable. - if let Some(ty) = arg.pad { - llargs.push(bx.const_undef(bx.reg_backend_type(&ty))) - } - - if arg.is_ignore() { - return; - } - - if let PassMode::Pair(..) = arg.mode { - match op.val { - Pair(a, b) => { - llargs.push(a); - llargs.push(b); - return; - } - _ => bug!("codegen_argument: {:?} invalid for pair argument", op), - } - } else if arg.is_unsized_indirect() { - match op.val { - Ref(a, Some(b), _) => { - llargs.push(a); - llargs.push(b); - return; - } - _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op), - } - } - - // Force by-ref if we have to load through a cast pointer. - let (mut llval, align, by_ref) = match op.val { - Immediate(_) | Pair(..) => match arg.mode { - PassMode::Indirect { .. } | PassMode::Cast(_) => { - let scratch = PlaceRef::alloca(bx, arg.layout); - op.val.store(bx, scratch); - (scratch.llval, scratch.align, true) - } - _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi, false), - }, - Ref(llval, _, align) => { - if arg.is_indirect() && align < arg.layout.align.abi { - // `foo(packed.large_field)`. We can't pass the (unaligned) field directly. I - // think that ATM (Rust 1.16) we only pass temporaries, but we shouldn't - // have scary latent bugs around. - - let scratch = PlaceRef::alloca(bx, arg.layout); - base::memcpy_ty( - bx, - scratch.llval, - scratch.align, - llval, - align, - op.layout, - MemFlags::empty(), - ); - (scratch.llval, scratch.align, true) - } else { - (llval, align, true) - } - } - }; - - if by_ref && !arg.is_indirect() { - // Have to load the argument, maybe while casting it. - if let PassMode::Cast(ty) = arg.mode { - let llty = bx.cast_backend_type(&ty); - let addr = bx.pointercast(llval, bx.type_ptr_to(llty)); - llval = bx.load(llty, addr, align.min(arg.layout.align.abi)); - } else { - // We can't use `PlaceRef::load` here because the argument - // may have a type we don't treat as immediate, but the ABI - // used for this call is passing it by-value. In that case, - // the load would just produce `OperandValue::Ref` instead - // of the `OperandValue::Immediate` we need for the call. - llval = bx.load(bx.backend_type(arg.layout), llval, align); - if let abi::Abi::Scalar(scalar) = arg.layout.abi { - if scalar.is_bool() { - bx.range_metadata(llval, WrappingRange { start: 0, end: 1 }); - } - } - // We store bools as `i8` so we need to truncate to `i1`. - llval = bx.to_immediate(llval, arg.layout); - } - } - - llargs.push(llval); - } - - fn codegen_arguments_untupled( - &mut self, - bx: &mut Bx, - operand: &mir::Operand<'tcx>, - llargs: &mut Vec, - args: &[ArgAbi<'tcx, Ty<'tcx>>], - ) -> usize { - let tuple = self.codegen_operand(bx, operand); - - // Handle both by-ref and immediate tuples. - if let Ref(llval, None, align) = tuple.val { - let tuple_ptr = PlaceRef::new_sized_aligned(llval, tuple.layout, align); - for i in 0..tuple.layout.fields.count() { - let field_ptr = tuple_ptr.project_field(bx, i); - let field = bx.load_operand(field_ptr); - self.codegen_argument(bx, field, llargs, &args[i]); - } - } else if let Ref(_, Some(_), _) = tuple.val { - bug!("closure arguments must be sized") - } else { - // If the tuple is immediate, the elements are as well. - for i in 0..tuple.layout.fields.count() { - let op = tuple.extract_field(bx, i); - self.codegen_argument(bx, op, llargs, &args[i]); - } - } - tuple.layout.fields.count() - } - - fn get_caller_location( - &mut self, - bx: &mut Bx, - mut source_info: mir::SourceInfo, - ) -> OperandRef<'tcx, Bx::Value> { - let tcx = bx.tcx(); - - let mut span_to_caller_location = |span: Span| { - let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span); - let caller = tcx.sess.source_map().lookup_char_pos(topmost.lo()); - let const_loc = tcx.const_caller_location(( - Symbol::intern(&caller.file.name.prefer_remapped().to_string_lossy()), - caller.line as u32, - caller.col_display as u32 + 1, - )); - OperandRef::from_const(bx, const_loc, bx.tcx().caller_location_ty()) - }; - - // Walk up the `SourceScope`s, in case some of them are from MIR inlining. - // If so, the starting `source_info.span` is in the innermost inlined - // function, and will be replaced with outer callsite spans as long - // as the inlined functions were `#[track_caller]`. - loop { - let scope_data = &self.mir.source_scopes[source_info.scope]; - - if let Some((callee, callsite_span)) = scope_data.inlined { - // Stop inside the most nested non-`#[track_caller]` function, - // before ever reaching its caller (which is irrelevant). - if !callee.def.requires_caller_location(tcx) { - return span_to_caller_location(source_info.span); - } - source_info.span = callsite_span; - } - - // Skip past all of the parents with `inlined: None`. - match scope_data.inlined_parent_scope { - Some(parent) => source_info.scope = parent, - None => break, - } - } - - // No inlined `SourceScope`s, or all of them were `#[track_caller]`. - self.caller_location.unwrap_or_else(|| span_to_caller_location(source_info.span)) - } - - fn get_personality_slot(&mut self, bx: &mut Bx) -> PlaceRef<'tcx, Bx::Value> { - let cx = bx.cx(); - if let Some(slot) = self.personality_slot { - slot - } else { - let layout = cx.layout_of( - cx.tcx().intern_tup(&[cx.tcx().mk_mut_ptr(cx.tcx().types.u8), cx.tcx().types.i32]), - ); - let slot = PlaceRef::alloca(bx, layout); - self.personality_slot = Some(slot); - slot - } - } - - /// Returns the landing/cleanup pad wrapper around the given basic block. - // FIXME(eddyb) rename this to `eh_pad_for`. - fn landing_pad_for(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock { - if let Some(landing_pad) = self.landing_pads[bb] { - return landing_pad; - } - - let landing_pad = self.landing_pad_for_uncached(bb); - self.landing_pads[bb] = Some(landing_pad); - landing_pad - } - - // FIXME(eddyb) rename this to `eh_pad_for_uncached`. - fn landing_pad_for_uncached(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock { - let llbb = self.llbb(bb); - if base::wants_msvc_seh(self.cx.sess()) { - let funclet; - let ret_llbb; - match self.mir[bb].terminator.as_ref().map(|t| &t.kind) { - // This is a basic block that we're aborting the program for, - // notably in an `extern` function. These basic blocks are inserted - // so that we assert that `extern` functions do indeed not panic, - // and if they do we abort the process. - // - // On MSVC these are tricky though (where we're doing funclets). If - // we were to do a cleanuppad (like below) the normal functions like - // `longjmp` would trigger the abort logic, terminating the - // program. Instead we insert the equivalent of `catch(...)` for C++ - // which magically doesn't trigger when `longjmp` files over this - // frame. - // - // Lots more discussion can be found on #48251 but this codegen is - // modeled after clang's for: - // - // try { - // foo(); - // } catch (...) { - // bar(); - // } - Some(&mir::TerminatorKind::Abort) => { - let cs_bb = - Bx::append_block(self.cx, self.llfn, &format!("cs_funclet{:?}", bb)); - let cp_bb = - Bx::append_block(self.cx, self.llfn, &format!("cp_funclet{:?}", bb)); - ret_llbb = cs_bb; - - let mut cs_bx = Bx::build(self.cx, cs_bb); - let cs = cs_bx.catch_switch(None, None, &[cp_bb]); - - // The "null" here is actually a RTTI type descriptor for the - // C++ personality function, but `catch (...)` has no type so - // it's null. The 64 here is actually a bitfield which - // represents that this is a catch-all block. - let mut cp_bx = Bx::build(self.cx, cp_bb); - let null = cp_bx.const_null( - cp_bx.type_i8p_ext(cp_bx.cx().data_layout().instruction_address_space), - ); - let sixty_four = cp_bx.const_i32(64); - funclet = cp_bx.catch_pad(cs, &[null, sixty_four, null]); - cp_bx.br(llbb); - } - _ => { - let cleanup_bb = - Bx::append_block(self.cx, self.llfn, &format!("funclet_{:?}", bb)); - ret_llbb = cleanup_bb; - let mut cleanup_bx = Bx::build(self.cx, cleanup_bb); - funclet = cleanup_bx.cleanup_pad(None, &[]); - cleanup_bx.br(llbb); - } - } - self.funclets[bb] = Some(funclet); - ret_llbb - } else { - let bb = Bx::append_block(self.cx, self.llfn, "cleanup"); - let mut bx = Bx::build(self.cx, bb); - - let llpersonality = self.cx.eh_personality(); - let llretty = self.landing_pad_type(); - let lp = bx.cleanup_landing_pad(llretty, llpersonality); - - let slot = self.get_personality_slot(&mut bx); - slot.storage_live(&mut bx); - Pair(bx.extract_value(lp, 0), bx.extract_value(lp, 1)).store(&mut bx, slot); - - bx.br(llbb); - bx.llbb() - } - } - - fn landing_pad_type(&self) -> Bx::Type { - let cx = self.cx; - cx.type_struct(&[cx.type_i8p(), cx.type_i32()], false) - } - - fn unreachable_block(&mut self) -> Bx::BasicBlock { - self.unreachable_block.unwrap_or_else(|| { - let llbb = Bx::append_block(self.cx, self.llfn, "unreachable"); - let mut bx = Bx::build(self.cx, llbb); - bx.unreachable(); - self.unreachable_block = Some(llbb); - llbb - }) - } - - fn double_unwind_guard(&mut self) -> Bx::BasicBlock { - self.double_unwind_guard.unwrap_or_else(|| { - assert!(!base::wants_msvc_seh(self.cx.sess())); - - let llbb = Bx::append_block(self.cx, self.llfn, "abort"); - let mut bx = Bx::build(self.cx, llbb); - self.set_debug_loc(&mut bx, mir::SourceInfo::outermost(self.mir.span)); - - let llpersonality = self.cx.eh_personality(); - let llretty = self.landing_pad_type(); - bx.cleanup_landing_pad(llretty, llpersonality); - - let (fn_abi, fn_ptr) = common::build_langcall(&bx, None, LangItem::PanicNoUnwind); - let fn_ty = bx.fn_decl_backend_type(&fn_abi); - - let llret = bx.call(fn_ty, fn_ptr, &[], None); - bx.apply_attrs_callsite(&fn_abi, llret); - bx.do_not_inline(llret); - - bx.unreachable(); - - self.double_unwind_guard = Some(llbb); - llbb - }) - } - - /// Get the backend `BasicBlock` for a MIR `BasicBlock`, either already - /// cached in `self.cached_llbbs`, or created on demand (and cached). - // FIXME(eddyb) rename `llbb` and other `ll`-prefixed things to use a - // more backend-agnostic prefix such as `cg` (i.e. this would be `cgbb`). - pub fn llbb(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock { - self.cached_llbbs[bb].unwrap_or_else(|| { - // FIXME(eddyb) only name the block if `fewer_names` is `false`. - let llbb = Bx::append_block(self.cx, self.llfn, &format!("{:?}", bb)); - self.cached_llbbs[bb] = Some(llbb); - llbb - }) - } - - fn make_return_dest( - &mut self, - bx: &mut Bx, - dest: mir::Place<'tcx>, - fn_ret: &ArgAbi<'tcx, Ty<'tcx>>, - llargs: &mut Vec, - is_intrinsic: bool, - ) -> ReturnDest<'tcx, Bx::Value> { - // If the return is ignored, we can just return a do-nothing `ReturnDest`. - if fn_ret.is_ignore() { - return ReturnDest::Nothing; - } - let dest = if let Some(index) = dest.as_local() { - match self.locals[index] { - LocalRef::Place(dest) => dest, - LocalRef::UnsizedPlace(_) => bug!("return type must be sized"), - LocalRef::Operand(None) => { - // Handle temporary places, specifically `Operand` ones, as - // they don't have `alloca`s. - return if fn_ret.is_indirect() { - // Odd, but possible, case, we have an operand temporary, - // but the calling convention has an indirect return. - let tmp = PlaceRef::alloca(bx, fn_ret.layout); - tmp.storage_live(bx); - llargs.push(tmp.llval); - ReturnDest::IndirectOperand(tmp, index) - } else if is_intrinsic { - // Currently, intrinsics always need a location to store - // the result, so we create a temporary `alloca` for the - // result. - let tmp = PlaceRef::alloca(bx, fn_ret.layout); - tmp.storage_live(bx); - ReturnDest::IndirectOperand(tmp, index) - } else { - ReturnDest::DirectOperand(index) - }; - } - LocalRef::Operand(Some(_)) => { - bug!("place local already assigned to"); - } - } - } else { - self.codegen_place( - bx, - mir::PlaceRef { local: dest.local, projection: &dest.projection }, - ) - }; - if fn_ret.is_indirect() { - if dest.align < dest.layout.align.abi { - // Currently, MIR code generation does not create calls - // that store directly to fields of packed structs (in - // fact, the calls it creates write only to temps). - // - // If someone changes that, please update this code path - // to create a temporary. - span_bug!(self.mir.span, "can't directly store to unaligned value"); - } - llargs.push(dest.llval); - ReturnDest::Nothing - } else { - ReturnDest::Store(dest) - } - } - - fn codegen_transmute(&mut self, bx: &mut Bx, src: &mir::Operand<'tcx>, dst: mir::Place<'tcx>) { - if let Some(index) = dst.as_local() { - match self.locals[index] { - LocalRef::Place(place) => self.codegen_transmute_into(bx, src, place), - LocalRef::UnsizedPlace(_) => bug!("transmute must not involve unsized locals"), - LocalRef::Operand(None) => { - let dst_layout = bx.layout_of(self.monomorphized_place_ty(dst.as_ref())); - assert!(!dst_layout.ty.has_erasable_regions()); - let place = PlaceRef::alloca(bx, dst_layout); - place.storage_live(bx); - self.codegen_transmute_into(bx, src, place); - let op = bx.load_operand(place); - place.storage_dead(bx); - self.locals[index] = LocalRef::Operand(Some(op)); - self.debug_introduce_local(bx, index); - } - LocalRef::Operand(Some(op)) => { - assert!(op.layout.is_zst(), "assigning to initialized SSAtemp"); - } - } - } else { - let dst = self.codegen_place(bx, dst.as_ref()); - self.codegen_transmute_into(bx, src, dst); - } - } - - fn codegen_transmute_into( - &mut self, - bx: &mut Bx, - src: &mir::Operand<'tcx>, - dst: PlaceRef<'tcx, Bx::Value>, - ) { - let src = self.codegen_operand(bx, src); - - // Special-case transmutes between scalars as simple bitcasts. - match (src.layout.abi, dst.layout.abi) { - (abi::Abi::Scalar(src_scalar), abi::Abi::Scalar(dst_scalar)) => { - // HACK(eddyb) LLVM doesn't like `bitcast`s between pointers and non-pointers. - if (src_scalar.primitive() == abi::Pointer) - == (dst_scalar.primitive() == abi::Pointer) - { - assert_eq!(src.layout.size, dst.layout.size); - - // NOTE(eddyb) the `from_immediate` and `to_immediate_scalar` - // conversions allow handling `bool`s the same as `u8`s. - let src = bx.from_immediate(src.immediate()); - let src_as_dst = bx.bitcast(src, bx.backend_type(dst.layout)); - Immediate(bx.to_immediate_scalar(src_as_dst, dst_scalar)).store(bx, dst); - return; - } - } - _ => {} - } - - let llty = bx.backend_type(src.layout); - let cast_ptr = bx.pointercast(dst.llval, bx.type_ptr_to(llty)); - let align = src.layout.align.abi.min(dst.align); - src.val.store(bx, PlaceRef::new_sized_aligned(cast_ptr, src.layout, align)); - } - - // Stores the return value of a function call into it's final location. - fn store_return( - &mut self, - bx: &mut Bx, - dest: ReturnDest<'tcx, Bx::Value>, - ret_abi: &ArgAbi<'tcx, Ty<'tcx>>, - llval: Bx::Value, - ) { - use self::ReturnDest::*; - - match dest { - Nothing => (), - Store(dst) => bx.store_arg(&ret_abi, llval, dst), - IndirectOperand(tmp, index) => { - let op = bx.load_operand(tmp); - tmp.storage_dead(bx); - self.locals[index] = LocalRef::Operand(Some(op)); - self.debug_introduce_local(bx, index); - } - DirectOperand(index) => { - // If there is a cast, we have to store and reload. - let op = if let PassMode::Cast(_) = ret_abi.mode { - let tmp = PlaceRef::alloca(bx, ret_abi.layout); - tmp.storage_live(bx); - bx.store_arg(&ret_abi, llval, tmp); - let op = bx.load_operand(tmp); - tmp.storage_dead(bx); - op - } else { - OperandRef::from_immediate_or_packed_pair(bx, llval, ret_abi.layout) - }; - self.locals[index] = LocalRef::Operand(Some(op)); - self.debug_introduce_local(bx, index); - } - } - } -} - -enum ReturnDest<'tcx, V> { - // Do nothing; the return value is indirect or ignored. - Nothing, - // Store the return value to the pointer. - Store(PlaceRef<'tcx, V>), - // Store an indirect return value to an operand local place. - IndirectOperand(PlaceRef<'tcx, V>, mir::Local), - // Store a direct return value to an operand local place. - DirectOperand(mir::Local), -} diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs deleted file mode 100644 index 9a995fbf65c..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/constant.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::mir::operand::OperandRef; -use crate::traits::*; -use rustc_middle::mir; -use rustc_middle::mir::interpret::{ConstValue, ErrorHandled}; -use rustc_middle::ty::layout::HasTyCtxt; -use rustc_middle::ty::{self, Ty}; -use rustc_span::source_map::Span; -use rustc_target::abi::Abi; - -use super::FunctionCx; - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - pub fn eval_mir_constant_to_operand( - &self, - bx: &mut Bx, - constant: &mir::Constant<'tcx>, - ) -> Result, ErrorHandled> { - let val = self.eval_mir_constant(constant)?; - let ty = self.monomorphize(constant.ty()); - Ok(OperandRef::from_const(bx, val, ty)) - } - - pub fn eval_mir_constant( - &self, - constant: &mir::Constant<'tcx>, - ) -> Result, ErrorHandled> { - let ct = self.monomorphize(constant.literal); - let ct = match ct { - mir::ConstantKind::Ty(ct) => ct, - mir::ConstantKind::Val(val, _) => return Ok(val), - }; - match ct.kind() { - ty::ConstKind::Unevaluated(ct) => self - .cx - .tcx() - .const_eval_resolve(ty::ParamEnv::reveal_all(), ct, None) - .map_err(|err| { - self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered"); - err - }), - ty::ConstKind::Value(val) => Ok(self.cx.tcx().valtree_to_const_val((ct.ty(), val))), - err => span_bug!( - constant.span, - "encountered bad ConstKind after monomorphizing: {:?}", - err - ), - } - } - - /// process constant containing SIMD shuffle indices - pub fn simd_shuffle_indices( - &mut self, - bx: &Bx, - span: Span, - ty: Ty<'tcx>, - constant: Result, ErrorHandled>, - ) -> (Bx::Value, Ty<'tcx>) { - constant - .map(|val| { - let field_ty = ty.builtin_index().unwrap(); - let c = mir::ConstantKind::from_value(val, ty); - let values: Vec<_> = bx - .tcx() - .destructure_mir_constant(ty::ParamEnv::reveal_all(), c) - .fields - .iter() - .map(|field| { - if let Some(prim) = field.try_to_scalar() { - let layout = bx.layout_of(field_ty); - let Abi::Scalar(scalar) = layout.abi else { - bug!("from_const: invalid ByVal layout: {:#?}", layout); - }; - bx.scalar_to_backend(prim, scalar, bx.immediate_backend_type(layout)) - } else { - bug!("simd shuffle field {:?}", field) - } - }) - .collect(); - let llval = bx.const_struct(&values, false); - (llval, c.ty()) - }) - .unwrap_or_else(|_| { - bx.tcx().sess.span_err(span, "could not evaluate shuffle_indices at compile time"); - // We've errored, so we don't have to produce working code. - let ty = self.monomorphize(ty); - let llty = bx.backend_type(bx.layout_of(ty)); - (bx.const_undef(llty), ty) - }) - } -} diff --git a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs deleted file mode 100644 index f1fe495282a..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs +++ /dev/null @@ -1,55 +0,0 @@ -use crate::traits::*; - -use rustc_middle::mir::coverage::*; -use rustc_middle::mir::Coverage; -use rustc_middle::mir::SourceScope; - -use super::FunctionCx; - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage, scope: SourceScope) { - // Determine the instance that coverage data was originally generated for. - let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) { - self.monomorphize(inlined) - } else { - self.instance - }; - - let Coverage { kind, code_region } = coverage; - match kind { - CoverageKind::Counter { function_source_hash, id } => { - if bx.set_function_source_hash(instance, function_source_hash) { - // If `set_function_source_hash()` returned true, the coverage map is enabled, - // so continue adding the counter. - if let Some(code_region) = code_region { - // Note: Some counters do not have code regions, but may still be referenced - // from expressions. In that case, don't add the counter to the coverage map, - // but do inject the counter intrinsic. - bx.add_coverage_counter(instance, id, code_region); - } - - let coverageinfo = bx.tcx().coverageinfo(instance.def); - - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(function_source_hash); - let num_counters = bx.const_u32(coverageinfo.num_counters); - let index = bx.const_u32(id.zero_based_index()); - debug!( - "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", - fn_name, hash, num_counters, index, - ); - bx.instrprof_increment(fn_name, hash, num_counters, index); - } - } - CoverageKind::Expression { id, lhs, op, rhs } => { - bx.add_coverage_counter_expression(instance, id, lhs, op, rhs, code_region); - } - CoverageKind::Unreachable => { - bx.add_coverage_unreachable( - instance, - code_region.expect("unreachable regions always have code regions"), - ); - } - } - } -} diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs deleted file mode 100644 index 8c3186efc63..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ /dev/null @@ -1,418 +0,0 @@ -use crate::traits::*; -use rustc_index::vec::IndexVec; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir; -use rustc_middle::ty; -use rustc_middle::ty::layout::LayoutOf; -use rustc_session::config::DebugInfo; -use rustc_span::symbol::{kw, Symbol}; -use rustc_span::{BytePos, Span}; -use rustc_target::abi::Abi; -use rustc_target::abi::Size; - -use super::operand::{OperandRef, OperandValue}; -use super::place::PlaceRef; -use super::{FunctionCx, LocalRef}; - -pub struct FunctionDebugContext { - pub scopes: IndexVec>, -} - -#[derive(Copy, Clone)] -pub enum VariableKind { - ArgumentVariable(usize /*index*/), - LocalVariable, -} - -/// Like `mir::VarDebugInfo`, but within a `mir::Local`. -#[derive(Copy, Clone)] -pub struct PerLocalVarDebugInfo<'tcx, D> { - pub name: Symbol, - pub source_info: mir::SourceInfo, - - /// `DIVariable` returned by `create_dbg_var`. - pub dbg_var: Option, - - /// `.place.projection` from `mir::VarDebugInfo`. - pub projection: &'tcx ty::List>, -} - -#[derive(Clone, Copy, Debug)] -pub struct DebugScope { - pub dbg_scope: S, - - /// Call site location, if this scope was inlined from another function. - pub inlined_at: Option, - - // Start and end offsets of the file to which this DIScope belongs. - // These are used to quickly determine whether some span refers to the same file. - pub file_start_pos: BytePos, - pub file_end_pos: BytePos, -} - -impl<'tcx, S: Copy, L: Copy> DebugScope { - /// DILocations inherit source file name from the parent DIScope. Due to macro expansions - /// it may so happen that the current span belongs to a different file than the DIScope - /// corresponding to span's containing source scope. If so, we need to create a DIScope - /// "extension" into that file. - pub fn adjust_dbg_scope_for_span>( - &self, - cx: &Cx, - span: Span, - ) -> S { - let pos = span.lo(); - if pos < self.file_start_pos || pos >= self.file_end_pos { - let sm = cx.sess().source_map(); - cx.extend_scope_to_file(self.dbg_scope, &sm.lookup_char_pos(pos).file) - } else { - self.dbg_scope - } - } -} - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) { - bx.set_span(source_info.span); - if let Some(dbg_loc) = self.dbg_loc(source_info) { - bx.set_dbg_loc(dbg_loc); - } - } - - fn dbg_loc(&self, source_info: mir::SourceInfo) -> Option { - let (dbg_scope, inlined_at, span) = self.adjusted_span_and_dbg_scope(source_info)?; - Some(self.cx.dbg_loc(dbg_scope, inlined_at, span)) - } - - fn adjusted_span_and_dbg_scope( - &self, - source_info: mir::SourceInfo, - ) -> Option<(Bx::DIScope, Option, Span)> { - let span = self.adjust_span_for_debugging(source_info.span); - let scope = &self.debug_context.as_ref()?.scopes[source_info.scope]; - Some((scope.adjust_dbg_scope_for_span(self.cx, span), scope.inlined_at, span)) - } - - /// In order to have a good line stepping behavior in debugger, we overwrite debug - /// locations of macro expansions with that of the outermost expansion site - /// (unless the crate is being compiled with `-Z debug-macros`). - fn adjust_span_for_debugging(&self, mut span: Span) -> Span { - // Bail out if debug info emission is not enabled. - if self.debug_context.is_none() { - return span; - } - - if span.from_expansion() && !self.cx.sess().opts.unstable_opts.debug_macros { - // Walk up the macro expansion chain until we reach a non-expanded span. - // We also stop at the function body level because no line stepping can occur - // at the level above that. - // Use span of the outermost expansion site, while keeping the original lexical scope. - span = rustc_span::hygiene::walk_chain(span, self.mir.span.ctxt()); - } - - span - } - - fn spill_operand_to_stack( - operand: &OperandRef<'tcx, Bx::Value>, - name: Option, - bx: &mut Bx, - ) -> PlaceRef<'tcx, Bx::Value> { - // "Spill" the value onto the stack, for debuginfo, - // without forcing non-debuginfo uses of the local - // to also load from the stack every single time. - // FIXME(#68817) use `llvm.dbg.value` instead, - // at least for the cases which LLVM handles correctly. - let spill_slot = PlaceRef::alloca(bx, operand.layout); - if let Some(name) = name { - bx.set_var_name(spill_slot.llval, &(name + ".dbg.spill")); - } - operand.val.store(bx, spill_slot); - spill_slot - } - - /// Apply debuginfo and/or name, after creating the `alloca` for a local, - /// or initializing the local with an operand (whichever applies). - pub fn debug_introduce_local(&self, bx: &mut Bx, local: mir::Local) { - let full_debug_info = bx.sess().opts.debuginfo == DebugInfo::Full; - - // FIXME(eddyb) maybe name the return place as `_0` or `return`? - if local == mir::RETURN_PLACE && !self.mir.local_decls[mir::RETURN_PLACE].is_user_variable() - { - return; - } - - let vars = match &self.per_local_var_debug_info { - Some(per_local) => &per_local[local], - None => return, - }; - let whole_local_var = vars.iter().find(|var| var.projection.is_empty()).copied(); - let has_proj = || vars.iter().any(|var| !var.projection.is_empty()); - - let fallback_var = if self.mir.local_kind(local) == mir::LocalKind::Arg { - let arg_index = local.index() - 1; - - // Add debuginfo even to unnamed arguments. - // FIXME(eddyb) is this really needed? - if arg_index == 0 && has_proj() { - // Hide closure environments from debuginfo. - // FIXME(eddyb) shouldn't `ArgumentVariable` indices - // be offset to account for the hidden environment? - None - } else if whole_local_var.is_some() { - // No need to make up anything, there is a `mir::VarDebugInfo` - // covering the whole local. - // FIXME(eddyb) take `whole_local_var.source_info.scope` into - // account, just in case it doesn't use `ArgumentVariable` - // (after #67586 gets fixed). - None - } else { - let name = kw::Empty; - let decl = &self.mir.local_decls[local]; - let dbg_var = if full_debug_info { - self.adjusted_span_and_dbg_scope(decl.source_info).map( - |(dbg_scope, _, span)| { - // FIXME(eddyb) is this `+ 1` needed at all? - let kind = VariableKind::ArgumentVariable(arg_index + 1); - - let arg_ty = self.monomorphize(decl.ty); - - self.cx.create_dbg_var(name, arg_ty, dbg_scope, kind, span) - }, - ) - } else { - None - }; - - Some(PerLocalVarDebugInfo { - name, - source_info: decl.source_info, - dbg_var, - projection: ty::List::empty(), - }) - } - } else { - None - }; - - let local_ref = &self.locals[local]; - - let name = if bx.sess().fewer_names() { - None - } else { - Some(match whole_local_var.or(fallback_var) { - Some(var) if var.name != kw::Empty => var.name.to_string(), - _ => format!("{:?}", local), - }) - }; - - if let Some(name) = &name { - match local_ref { - LocalRef::Place(place) | LocalRef::UnsizedPlace(place) => { - bx.set_var_name(place.llval, name); - } - LocalRef::Operand(Some(operand)) => match operand.val { - OperandValue::Ref(x, ..) | OperandValue::Immediate(x) => { - bx.set_var_name(x, name); - } - OperandValue::Pair(a, b) => { - // FIXME(eddyb) these are scalar components, - // maybe extract the high-level fields? - bx.set_var_name(a, &(name.clone() + ".0")); - bx.set_var_name(b, &(name.clone() + ".1")); - } - }, - LocalRef::Operand(None) => {} - } - } - - if !full_debug_info || vars.is_empty() && fallback_var.is_none() { - return; - } - - let base = match local_ref { - LocalRef::Operand(None) => return, - - LocalRef::Operand(Some(operand)) => { - // Don't spill operands onto the stack in naked functions. - // See: https://github.com/rust-lang/rust/issues/42779 - let attrs = bx.tcx().codegen_fn_attrs(self.instance.def_id()); - if attrs.flags.contains(CodegenFnAttrFlags::NAKED) { - return; - } - - Self::spill_operand_to_stack(operand, name, bx) - } - - LocalRef::Place(place) => *place, - - // FIXME(eddyb) add debuginfo for unsized places too. - LocalRef::UnsizedPlace(_) => return, - }; - - let vars = vars.iter().copied().chain(fallback_var); - - for var in vars { - let Some(dbg_var) = var.dbg_var else { continue }; - let Some(dbg_loc) = self.dbg_loc(var.source_info) else { continue }; - - let mut direct_offset = Size::ZERO; - // FIXME(eddyb) use smallvec here. - let mut indirect_offsets = vec![]; - let mut place = base; - - for elem in &var.projection[..] { - match *elem { - mir::ProjectionElem::Deref => { - indirect_offsets.push(Size::ZERO); - place = bx.load_operand(place).deref(bx.cx()); - } - mir::ProjectionElem::Field(field, _) => { - let i = field.index(); - let offset = indirect_offsets.last_mut().unwrap_or(&mut direct_offset); - *offset += place.layout.fields.offset(i); - place = place.project_field(bx, i); - } - mir::ProjectionElem::Downcast(_, variant) => { - place = place.project_downcast(bx, variant); - } - _ => span_bug!( - var.source_info.span, - "unsupported var debuginfo place `{:?}`", - mir::Place { local, projection: var.projection }, - ), - } - } - - // When targeting MSVC, create extra allocas for arguments instead of pointing multiple - // dbg_var_addr() calls into the same alloca with offsets. MSVC uses CodeView records - // not DWARF and LLVM doesn't support translating the resulting - // [DW_OP_deref, DW_OP_plus_uconst, offset, DW_OP_deref] debug info to CodeView. - // Creating extra allocas on the stack makes the resulting debug info simple enough - // that LLVM can generate correct CodeView records and thus the values appear in the - // debugger. (#83709) - let should_create_individual_allocas = bx.cx().sess().target.is_like_msvc - && self.mir.local_kind(local) == mir::LocalKind::Arg - // LLVM can handle simple things but anything more complex than just a direct - // offset or one indirect offset of 0 is too complex for it to generate CV records - // correctly. - && (direct_offset != Size::ZERO - || !matches!(&indirect_offsets[..], [Size::ZERO] | [])); - - if should_create_individual_allocas { - // Create a variable which will be a pointer to the actual value - let ptr_ty = bx.tcx().mk_ty(ty::RawPtr(ty::TypeAndMut { - mutbl: mir::Mutability::Mut, - ty: place.layout.ty, - })); - let ptr_layout = bx.layout_of(ptr_ty); - let alloca = PlaceRef::alloca(bx, ptr_layout); - bx.set_var_name(alloca.llval, &(var.name.to_string() + ".dbg.spill")); - - // Write the pointer to the variable - bx.store(place.llval, alloca.llval, alloca.align); - - // Point the debug info to `*alloca` for the current variable - bx.dbg_var_addr(dbg_var, dbg_loc, alloca.llval, Size::ZERO, &[Size::ZERO]); - } else { - bx.dbg_var_addr(dbg_var, dbg_loc, base.llval, direct_offset, &indirect_offsets); - } - } - } - - pub fn debug_introduce_locals(&self, bx: &mut Bx) { - if bx.sess().opts.debuginfo == DebugInfo::Full || !bx.sess().fewer_names() { - for local in self.locals.indices() { - self.debug_introduce_local(bx, local); - } - } - } - - /// Partition all `VarDebugInfo` in `self.mir`, by their base `Local`. - pub fn compute_per_local_var_debug_info( - &self, - bx: &mut Bx, - ) -> Option>>> { - let full_debug_info = self.cx.sess().opts.debuginfo == DebugInfo::Full; - - let target_is_msvc = self.cx.sess().target.is_like_msvc; - - if !full_debug_info && self.cx.sess().fewer_names() { - return None; - } - - let mut per_local = IndexVec::from_elem(vec![], &self.mir.local_decls); - for var in &self.mir.var_debug_info { - let dbg_scope_and_span = if full_debug_info { - self.adjusted_span_and_dbg_scope(var.source_info) - } else { - None - }; - - let dbg_var = dbg_scope_and_span.map(|(dbg_scope, _, span)| { - let (var_ty, var_kind) = match var.value { - mir::VarDebugInfoContents::Place(place) => { - let var_ty = self.monomorphized_place_ty(place.as_ref()); - let var_kind = if self.mir.local_kind(place.local) == mir::LocalKind::Arg - && place.projection.is_empty() - && var.source_info.scope == mir::OUTERMOST_SOURCE_SCOPE - { - let arg_index = place.local.index() - 1; - if target_is_msvc { - // ScalarPair parameters are spilled to the stack so they need to - // be marked as a `LocalVariable` for MSVC debuggers to visualize - // their data correctly. (See #81894 & #88625) - let var_ty_layout = self.cx.layout_of(var_ty); - if let Abi::ScalarPair(_, _) = var_ty_layout.abi { - VariableKind::LocalVariable - } else { - VariableKind::ArgumentVariable(arg_index + 1) - } - } else { - // FIXME(eddyb) shouldn't `ArgumentVariable` indices be - // offset in closures to account for the hidden environment? - // Also, is this `+ 1` needed at all? - VariableKind::ArgumentVariable(arg_index + 1) - } - } else { - VariableKind::LocalVariable - }; - (var_ty, var_kind) - } - mir::VarDebugInfoContents::Const(c) => { - let ty = self.monomorphize(c.ty()); - (ty, VariableKind::LocalVariable) - } - }; - - self.cx.create_dbg_var(var.name, var_ty, dbg_scope, var_kind, span) - }); - - match var.value { - mir::VarDebugInfoContents::Place(place) => { - per_local[place.local].push(PerLocalVarDebugInfo { - name: var.name, - source_info: var.source_info, - dbg_var, - projection: place.projection, - }); - } - mir::VarDebugInfoContents::Const(c) => { - if let Some(dbg_var) = dbg_var { - let Some(dbg_loc) = self.dbg_loc(var.source_info) else { continue }; - - if let Ok(operand) = self.eval_mir_constant_to_operand(bx, &c) { - let base = Self::spill_operand_to_stack( - &operand, - Some(var.name.to_string()), - bx, - ); - - bx.dbg_var_addr(dbg_var, dbg_loc, base.llval, Size::ZERO, &[]); - } - } - } - } - } - Some(per_local) - } -} diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs deleted file mode 100644 index 94ac71a4dd2..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ /dev/null @@ -1,636 +0,0 @@ -use super::operand::{OperandRef, OperandValue}; -use super::place::PlaceRef; -use super::FunctionCx; -use crate::common::{span_invalid_monomorphization_error, IntPredicate}; -use crate::glue; -use crate::meth; -use crate::traits::*; -use crate::MemFlags; - -use rustc_middle::ty::{self, Ty, TyCtxt}; -use rustc_span::{sym, Span}; -use rustc_target::abi::{ - call::{FnAbi, PassMode}, - WrappingRange, -}; - -fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - allow_overlap: bool, - volatile: bool, - ty: Ty<'tcx>, - dst: Bx::Value, - src: Bx::Value, - count: Bx::Value, -) { - let layout = bx.layout_of(ty); - let size = layout.size; - let align = layout.align.abi; - let size = bx.mul(bx.const_usize(size.bytes()), count); - let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() }; - if allow_overlap { - bx.memmove(dst, align, src, align, size, flags); - } else { - bx.memcpy(dst, align, src, align, size, flags); - } -} - -fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - volatile: bool, - ty: Ty<'tcx>, - dst: Bx::Value, - val: Bx::Value, - count: Bx::Value, -) { - let layout = bx.layout_of(ty); - let size = layout.size; - let align = layout.align.abi; - let size = bx.mul(bx.const_usize(size.bytes()), count); - let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() }; - bx.memset(dst, val, size, align, flags); -} - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - pub fn codegen_intrinsic_call( - bx: &mut Bx, - instance: ty::Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - args: &[OperandRef<'tcx, Bx::Value>], - llresult: Bx::Value, - span: Span, - ) { - let callee_ty = instance.ty(bx.tcx(), ty::ParamEnv::reveal_all()); - - let ty::FnDef(def_id, substs) = *callee_ty.kind() else { - bug!("expected fn item type, found {}", callee_ty); - }; - - let sig = callee_ty.fn_sig(bx.tcx()); - let sig = bx.tcx().normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig); - let arg_tys = sig.inputs(); - let ret_ty = sig.output(); - let name = bx.tcx().item_name(def_id); - let name_str = name.as_str(); - - let llret_ty = bx.backend_type(bx.layout_of(ret_ty)); - let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); - - let llval = match name { - sym::assume => { - bx.assume(args[0].immediate()); - return; - } - sym::abort => { - bx.abort(); - return; - } - - sym::va_start => bx.va_start(args[0].immediate()), - sym::va_end => bx.va_end(args[0].immediate()), - sym::size_of_val => { - let tp_ty = substs.type_at(0); - if let OperandValue::Pair(_, meta) = args[0].val { - let (llsize, _) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta)); - llsize - } else { - bx.const_usize(bx.layout_of(tp_ty).size.bytes()) - } - } - sym::min_align_of_val => { - let tp_ty = substs.type_at(0); - if let OperandValue::Pair(_, meta) = args[0].val { - let (_, llalign) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta)); - llalign - } else { - bx.const_usize(bx.layout_of(tp_ty).align.abi.bytes()) - } - } - sym::vtable_size | sym::vtable_align => { - let vtable = args[0].immediate(); - let idx = match name { - sym::vtable_size => ty::COMMON_VTABLE_ENTRIES_SIZE, - sym::vtable_align => ty::COMMON_VTABLE_ENTRIES_ALIGN, - _ => bug!(), - }; - let value = meth::VirtualIndex::from_index(idx).get_usize(bx, vtable); - if name == sym::vtable_align { - // Alignment is always nonzero. - bx.range_metadata(value, WrappingRange { start: 1, end: !0 }); - }; - value - } - sym::pref_align_of - | sym::needs_drop - | sym::type_id - | sym::type_name - | sym::variant_count => { - let value = bx - .tcx() - .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None) - .unwrap(); - OperandRef::from_const(bx, value, ret_ty).immediate_or_packed_pair(bx) - } - sym::offset => { - let ty = substs.type_at(0); - let layout = bx.layout_of(ty); - let ptr = args[0].immediate(); - let offset = args[1].immediate(); - bx.inbounds_gep(bx.backend_type(layout), ptr, &[offset]) - } - sym::arith_offset => { - let ty = substs.type_at(0); - let layout = bx.layout_of(ty); - let ptr = args[0].immediate(); - let offset = args[1].immediate(); - bx.gep(bx.backend_type(layout), ptr, &[offset]) - } - sym::copy => { - copy_intrinsic( - bx, - true, - false, - substs.type_at(0), - args[1].immediate(), - args[0].immediate(), - args[2].immediate(), - ); - return; - } - sym::write_bytes => { - memset_intrinsic( - bx, - false, - substs.type_at(0), - args[0].immediate(), - args[1].immediate(), - args[2].immediate(), - ); - return; - } - - sym::volatile_copy_nonoverlapping_memory => { - copy_intrinsic( - bx, - false, - true, - substs.type_at(0), - args[0].immediate(), - args[1].immediate(), - args[2].immediate(), - ); - return; - } - sym::volatile_copy_memory => { - copy_intrinsic( - bx, - true, - true, - substs.type_at(0), - args[0].immediate(), - args[1].immediate(), - args[2].immediate(), - ); - return; - } - sym::volatile_set_memory => { - memset_intrinsic( - bx, - true, - substs.type_at(0), - args[0].immediate(), - args[1].immediate(), - args[2].immediate(), - ); - return; - } - sym::volatile_store => { - let dst = args[0].deref(bx.cx()); - args[1].val.volatile_store(bx, dst); - return; - } - sym::unaligned_volatile_store => { - let dst = args[0].deref(bx.cx()); - args[1].val.unaligned_volatile_store(bx, dst); - return; - } - sym::add_with_overflow - | sym::sub_with_overflow - | sym::mul_with_overflow - | sym::unchecked_div - | sym::unchecked_rem - | sym::unchecked_shl - | sym::unchecked_shr - | sym::unchecked_add - | sym::unchecked_sub - | sym::unchecked_mul - | sym::exact_div => { - let ty = arg_tys[0]; - match int_type_width_signed(ty, bx.tcx()) { - Some((_width, signed)) => match name { - sym::add_with_overflow - | sym::sub_with_overflow - | sym::mul_with_overflow => { - let op = match name { - sym::add_with_overflow => OverflowOp::Add, - sym::sub_with_overflow => OverflowOp::Sub, - sym::mul_with_overflow => OverflowOp::Mul, - _ => bug!(), - }; - let (val, overflow) = - bx.checked_binop(op, ty, args[0].immediate(), args[1].immediate()); - // Convert `i1` to a `bool`, and write it to the out parameter - let val = bx.from_immediate(val); - let overflow = bx.from_immediate(overflow); - - let dest = result.project_field(bx, 0); - bx.store(val, dest.llval, dest.align); - let dest = result.project_field(bx, 1); - bx.store(overflow, dest.llval, dest.align); - - return; - } - sym::exact_div => { - if signed { - bx.exactsdiv(args[0].immediate(), args[1].immediate()) - } else { - bx.exactudiv(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_div => { - if signed { - bx.sdiv(args[0].immediate(), args[1].immediate()) - } else { - bx.udiv(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_rem => { - if signed { - bx.srem(args[0].immediate(), args[1].immediate()) - } else { - bx.urem(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_shl => bx.shl(args[0].immediate(), args[1].immediate()), - sym::unchecked_shr => { - if signed { - bx.ashr(args[0].immediate(), args[1].immediate()) - } else { - bx.lshr(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_add => { - if signed { - bx.unchecked_sadd(args[0].immediate(), args[1].immediate()) - } else { - bx.unchecked_uadd(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_sub => { - if signed { - bx.unchecked_ssub(args[0].immediate(), args[1].immediate()) - } else { - bx.unchecked_usub(args[0].immediate(), args[1].immediate()) - } - } - sym::unchecked_mul => { - if signed { - bx.unchecked_smul(args[0].immediate(), args[1].immediate()) - } else { - bx.unchecked_umul(args[0].immediate(), args[1].immediate()) - } - } - _ => bug!(), - }, - None => { - span_invalid_monomorphization_error( - bx.tcx().sess, - span, - &format!( - "invalid monomorphization of `{}` intrinsic: \ - expected basic integer type, found `{}`", - name, ty - ), - ); - return; - } - } - } - sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => { - match float_type_width(arg_tys[0]) { - Some(_width) => match name { - sym::fadd_fast => bx.fadd_fast(args[0].immediate(), args[1].immediate()), - sym::fsub_fast => bx.fsub_fast(args[0].immediate(), args[1].immediate()), - sym::fmul_fast => bx.fmul_fast(args[0].immediate(), args[1].immediate()), - sym::fdiv_fast => bx.fdiv_fast(args[0].immediate(), args[1].immediate()), - sym::frem_fast => bx.frem_fast(args[0].immediate(), args[1].immediate()), - _ => bug!(), - }, - None => { - span_invalid_monomorphization_error( - bx.tcx().sess, - span, - &format!( - "invalid monomorphization of `{}` intrinsic: \ - expected basic float type, found `{}`", - name, arg_tys[0] - ), - ); - return; - } - } - } - - sym::float_to_int_unchecked => { - if float_type_width(arg_tys[0]).is_none() { - span_invalid_monomorphization_error( - bx.tcx().sess, - span, - &format!( - "invalid monomorphization of `float_to_int_unchecked` \ - intrinsic: expected basic float type, \ - found `{}`", - arg_tys[0] - ), - ); - return; - } - let Some((_width, signed)) = int_type_width_signed(ret_ty, bx.tcx()) else { - span_invalid_monomorphization_error( - bx.tcx().sess, - span, - &format!( - "invalid monomorphization of `float_to_int_unchecked` \ - intrinsic: expected basic integer type, \ - found `{}`", - ret_ty - ), - ); - return; - }; - if signed { - bx.fptosi(args[0].immediate(), llret_ty) - } else { - bx.fptoui(args[0].immediate(), llret_ty) - } - } - - sym::discriminant_value => { - if ret_ty.is_integral() { - args[0].deref(bx.cx()).codegen_get_discr(bx, ret_ty) - } else { - span_bug!(span, "Invalid discriminant type for `{:?}`", arg_tys[0]) - } - } - - sym::const_allocate => { - // returns a null pointer at runtime. - bx.const_null(bx.type_i8p()) - } - - sym::const_deallocate => { - // nop at runtime. - return; - } - - // This requires that atomic intrinsics follow a specific naming pattern: - // "atomic_[_]" - name if let Some(atomic) = name_str.strip_prefix("atomic_") => { - use crate::common::AtomicOrdering::*; - use crate::common::{AtomicRmwBinOp, SynchronizationScope}; - - let Some((instruction, ordering)) = atomic.split_once('_') else { - bx.sess().fatal("Atomic intrinsic missing memory ordering"); - }; - - let parse_ordering = |bx: &Bx, s| match s { - "unordered" => Unordered, - "relaxed" => Relaxed, - "acquire" => Acquire, - "release" => Release, - "acqrel" => AcquireRelease, - "seqcst" => SequentiallyConsistent, - _ => bx.sess().fatal("unknown ordering in atomic intrinsic"), - }; - - let invalid_monomorphization = |ty| { - span_invalid_monomorphization_error( - bx.tcx().sess, - span, - &format!( - "invalid monomorphization of `{}` intrinsic: \ - expected basic integer type, found `{}`", - name, ty - ), - ); - }; - - match instruction { - "cxchg" | "cxchgweak" => { - let Some((success, failure)) = ordering.split_once('_') else { - bx.sess().fatal("Atomic compare-exchange intrinsic missing failure memory ordering"); - }; - let ty = substs.type_at(0); - if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { - let weak = instruction == "cxchgweak"; - let mut dst = args[0].immediate(); - let mut cmp = args[1].immediate(); - let mut src = args[2].immediate(); - if ty.is_unsafe_ptr() { - // Some platforms do not support atomic operations on pointers, - // so we cast to integer first. - let ptr_llty = bx.type_ptr_to(bx.type_isize()); - dst = bx.pointercast(dst, ptr_llty); - cmp = bx.ptrtoint(cmp, bx.type_isize()); - src = bx.ptrtoint(src, bx.type_isize()); - } - let pair = bx.atomic_cmpxchg(dst, cmp, src, parse_ordering(bx, success), parse_ordering(bx, failure), weak); - let val = bx.extract_value(pair, 0); - let success = bx.extract_value(pair, 1); - let val = bx.from_immediate(val); - let success = bx.from_immediate(success); - - let dest = result.project_field(bx, 0); - bx.store(val, dest.llval, dest.align); - let dest = result.project_field(bx, 1); - bx.store(success, dest.llval, dest.align); - return; - } else { - return invalid_monomorphization(ty); - } - } - - "load" => { - let ty = substs.type_at(0); - if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { - let layout = bx.layout_of(ty); - let size = layout.size; - let mut source = args[0].immediate(); - if ty.is_unsafe_ptr() { - // Some platforms do not support atomic operations on pointers, - // so we cast to integer first... - let llty = bx.type_isize(); - let ptr_llty = bx.type_ptr_to(llty); - source = bx.pointercast(source, ptr_llty); - let result = bx.atomic_load(llty, source, parse_ordering(bx, ordering), size); - // ... and then cast the result back to a pointer - bx.inttoptr(result, bx.backend_type(layout)) - } else { - bx.atomic_load(bx.backend_type(layout), source, parse_ordering(bx, ordering), size) - } - } else { - return invalid_monomorphization(ty); - } - } - - "store" => { - let ty = substs.type_at(0); - if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { - let size = bx.layout_of(ty).size; - let mut val = args[1].immediate(); - let mut ptr = args[0].immediate(); - if ty.is_unsafe_ptr() { - // Some platforms do not support atomic operations on pointers, - // so we cast to integer first. - let ptr_llty = bx.type_ptr_to(bx.type_isize()); - ptr = bx.pointercast(ptr, ptr_llty); - val = bx.ptrtoint(val, bx.type_isize()); - } - bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size); - return; - } else { - return invalid_monomorphization(ty); - } - } - - "fence" => { - bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::CrossThread); - return; - } - - "singlethreadfence" => { - bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::SingleThread); - return; - } - - // These are all AtomicRMW ops - op => { - let atom_op = match op { - "xchg" => AtomicRmwBinOp::AtomicXchg, - "xadd" => AtomicRmwBinOp::AtomicAdd, - "xsub" => AtomicRmwBinOp::AtomicSub, - "and" => AtomicRmwBinOp::AtomicAnd, - "nand" => AtomicRmwBinOp::AtomicNand, - "or" => AtomicRmwBinOp::AtomicOr, - "xor" => AtomicRmwBinOp::AtomicXor, - "max" => AtomicRmwBinOp::AtomicMax, - "min" => AtomicRmwBinOp::AtomicMin, - "umax" => AtomicRmwBinOp::AtomicUMax, - "umin" => AtomicRmwBinOp::AtomicUMin, - _ => bx.sess().fatal("unknown atomic operation"), - }; - - let ty = substs.type_at(0); - if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { - let mut ptr = args[0].immediate(); - let mut val = args[1].immediate(); - if ty.is_unsafe_ptr() { - // Some platforms do not support atomic operations on pointers, - // so we cast to integer first. - let ptr_llty = bx.type_ptr_to(bx.type_isize()); - ptr = bx.pointercast(ptr, ptr_llty); - val = bx.ptrtoint(val, bx.type_isize()); - } - bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering)) - } else { - return invalid_monomorphization(ty); - } - } - } - } - - sym::nontemporal_store => { - let dst = args[0].deref(bx.cx()); - args[1].val.nontemporal_store(bx, dst); - return; - } - - sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => { - let a = args[0].immediate(); - let b = args[1].immediate(); - if name == sym::ptr_guaranteed_eq { - bx.icmp(IntPredicate::IntEQ, a, b) - } else { - bx.icmp(IntPredicate::IntNE, a, b) - } - } - - sym::ptr_offset_from | sym::ptr_offset_from_unsigned => { - let ty = substs.type_at(0); - let pointee_size = bx.layout_of(ty).size; - - let a = args[0].immediate(); - let b = args[1].immediate(); - let a = bx.ptrtoint(a, bx.type_isize()); - let b = bx.ptrtoint(b, bx.type_isize()); - let pointee_size = bx.const_usize(pointee_size.bytes()); - if name == sym::ptr_offset_from { - // This is the same sequence that Clang emits for pointer subtraction. - // It can be neither `nsw` nor `nuw` because the input is treated as - // unsigned but then the output is treated as signed, so neither works. - let d = bx.sub(a, b); - // this is where the signed magic happens (notice the `s` in `exactsdiv`) - bx.exactsdiv(d, pointee_size) - } else { - // The `_unsigned` version knows the relative ordering of the pointers, - // so can use `sub nuw` and `udiv exact` instead of dealing in signed. - let d = bx.unchecked_usub(a, b); - bx.exactudiv(d, pointee_size) - } - } - - _ => { - // Need to use backend-specific things in the implementation. - bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span); - return; - } - }; - - if !fn_abi.ret.is_ignore() { - if let PassMode::Cast(ty) = fn_abi.ret.mode { - let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(&ty)); - let ptr = bx.pointercast(result.llval, ptr_llty); - bx.store(llval, ptr, result.align); - } else { - OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout) - .val - .store(bx, result); - } - } - } -} - -// Returns the width of an int Ty, and if it's signed or not -// Returns None if the type is not an integer -// FIXME: there’s multiple of this functions, investigate using some of the already existing -// stuffs. -fn int_type_width_signed(ty: Ty<'_>, tcx: TyCtxt<'_>) -> Option<(u64, bool)> { - match ty.kind() { - ty::Int(t) => { - Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), true)) - } - ty::Uint(t) => { - Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), false)) - } - _ => None, - } -} - -// Returns the width of a float Ty -// Returns None if the type is not a float -fn float_type_width(ty: Ty<'_>) -> Option { - match ty.kind() { - ty::Float(t) => Some(t.bit_width()), - _ => None, - } -} diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs deleted file mode 100644 index 8ee375fa9e3..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ /dev/null @@ -1,410 +0,0 @@ -use crate::traits::*; -use rustc_middle::mir; -use rustc_middle::mir::interpret::ErrorHandled; -use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout}; -use rustc_middle::ty::{self, Instance, Ty, TypeFoldable, TypeVisitable}; -use rustc_target::abi::call::{FnAbi, PassMode}; - -use std::iter; - -use rustc_index::bit_set::BitSet; -use rustc_index::vec::IndexVec; - -use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo}; -use self::place::PlaceRef; -use rustc_middle::mir::traversal; - -use self::operand::{OperandRef, OperandValue}; - -/// Master context for codegenning from MIR. -pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { - instance: Instance<'tcx>, - - mir: &'tcx mir::Body<'tcx>, - - debug_context: Option>, - - llfn: Bx::Function, - - cx: &'a Bx::CodegenCx, - - fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, - - /// When unwinding is initiated, we have to store this personality - /// value somewhere so that we can load it and re-use it in the - /// resume instruction. The personality is (afaik) some kind of - /// value used for C++ unwinding, which must filter by type: we - /// don't really care about it very much. Anyway, this value - /// contains an alloca into which the personality is stored and - /// then later loaded when generating the DIVERGE_BLOCK. - personality_slot: Option>, - - /// A backend `BasicBlock` for each MIR `BasicBlock`, created lazily - /// as-needed (e.g. RPO reaching it or another block branching to it). - // FIXME(eddyb) rename `llbbs` and other `ll`-prefixed things to use a - // more backend-agnostic prefix such as `cg` (i.e. this would be `cgbbs`). - cached_llbbs: IndexVec>, - - /// The funclet status of each basic block - cleanup_kinds: IndexVec, - - /// When targeting MSVC, this stores the cleanup info for each funclet BB. - /// This is initialized at the same time as the `landing_pads` entry for the - /// funclets' head block, i.e. when needed by an unwind / `cleanup_ret` edge. - funclets: IndexVec>, - - /// This stores the cached landing/cleanup pad block for a given BB. - // FIXME(eddyb) rename this to `eh_pads`. - landing_pads: IndexVec>, - - /// Cached unreachable block - unreachable_block: Option, - - /// Cached double unwind guarding block - double_unwind_guard: Option, - - /// The location where each MIR arg/var/tmp/ret is stored. This is - /// usually an `PlaceRef` representing an alloca, but not always: - /// sometimes we can skip the alloca and just store the value - /// directly using an `OperandRef`, which makes for tighter LLVM - /// IR. The conditions for using an `OperandRef` are as follows: - /// - /// - the type of the local must be judged "immediate" by `is_llvm_immediate` - /// - the operand must never be referenced indirectly - /// - we should not take its address using the `&` operator - /// - nor should it appear in a place path like `tmp.a` - /// - the operand must be defined by an rvalue that can generate immediate - /// values - /// - /// Avoiding allocs can also be important for certain intrinsics, - /// notably `expect`. - locals: IndexVec>, - - /// All `VarDebugInfo` from the MIR body, partitioned by `Local`. - /// This is `None` if no var`#[non_exhaustive]`iable debuginfo/names are needed. - per_local_var_debug_info: - Option>>>, - - /// Caller location propagated if this function has `#[track_caller]`. - caller_location: Option>, -} - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - pub fn monomorphize(&self, value: T) -> T - where - T: Copy + TypeFoldable<'tcx>, - { - debug!("monomorphize: self.instance={:?}", self.instance); - self.instance.subst_mir_and_normalize_erasing_regions( - self.cx.tcx(), - ty::ParamEnv::reveal_all(), - value, - ) - } -} - -enum LocalRef<'tcx, V> { - Place(PlaceRef<'tcx, V>), - /// `UnsizedPlace(p)`: `p` itself is a thin pointer (indirect place). - /// `*p` is the fat pointer that references the actual unsized place. - /// Every time it is initialized, we have to reallocate the place - /// and update the fat pointer. That's the reason why it is indirect. - UnsizedPlace(PlaceRef<'tcx, V>), - Operand(Option>), -} - -impl<'a, 'tcx, V: CodegenObject> LocalRef<'tcx, V> { - fn new_operand>( - bx: &mut Bx, - layout: TyAndLayout<'tcx>, - ) -> LocalRef<'tcx, V> { - if layout.is_zst() { - // Zero-size temporaries aren't always initialized, which - // doesn't matter because they don't contain data, but - // we need something in the operand. - LocalRef::Operand(Some(OperandRef::new_zst(bx, layout))) - } else { - LocalRef::Operand(None) - } - } -} - -/////////////////////////////////////////////////////////////////////////// - -#[instrument(level = "debug", skip(cx))] -pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - cx: &'a Bx::CodegenCx, - instance: Instance<'tcx>, -) { - assert!(!instance.substs.needs_infer()); - - let llfn = cx.get_fn(instance); - - let mir = cx.tcx().instance_mir(instance.def); - - let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); - debug!("fn_abi: {:?}", fn_abi); - - let debug_context = cx.create_function_debug_context(instance, &fn_abi, llfn, &mir); - - let start_llbb = Bx::append_block(cx, llfn, "start"); - let mut bx = Bx::build(cx, start_llbb); - - if mir.basic_blocks().iter().any(|bb| bb.is_cleanup) { - bx.set_personality_fn(cx.eh_personality()); - } - - let cleanup_kinds = analyze::cleanup_kinds(&mir); - let cached_llbbs: IndexVec> = mir - .basic_blocks() - .indices() - .map(|bb| if bb == mir::START_BLOCK { Some(start_llbb) } else { None }) - .collect(); - - let mut fx = FunctionCx { - instance, - mir, - llfn, - fn_abi, - cx, - personality_slot: None, - cached_llbbs, - unreachable_block: None, - double_unwind_guard: None, - cleanup_kinds, - landing_pads: IndexVec::from_elem(None, mir.basic_blocks()), - funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks().len()), - locals: IndexVec::new(), - debug_context, - per_local_var_debug_info: None, - caller_location: None, - }; - - fx.per_local_var_debug_info = fx.compute_per_local_var_debug_info(&mut bx); - - // Evaluate all required consts; codegen later assumes that CTFE will never fail. - let mut all_consts_ok = true; - for const_ in &mir.required_consts { - if let Err(err) = fx.eval_mir_constant(const_) { - all_consts_ok = false; - match err { - // errored or at least linted - ErrorHandled::Reported(_) | ErrorHandled::Linted => {} - ErrorHandled::TooGeneric => { - span_bug!(const_.span, "codgen encountered polymorphic constant: {:?}", err) - } - } - } - } - if !all_consts_ok { - // We leave the IR in some half-built state here, and rely on this code not even being - // submitted to LLVM once an error was raised. - return; - } - - let memory_locals = analyze::non_ssa_locals(&fx); - - // Allocate variable and temp allocas - fx.locals = { - let args = arg_local_refs(&mut bx, &mut fx, &memory_locals); - - let mut allocate_local = |local| { - let decl = &mir.local_decls[local]; - let layout = bx.layout_of(fx.monomorphize(decl.ty)); - assert!(!layout.ty.has_erasable_regions()); - - if local == mir::RETURN_PLACE && fx.fn_abi.ret.is_indirect() { - debug!("alloc: {:?} (return place) -> place", local); - let llretptr = bx.get_param(0); - return LocalRef::Place(PlaceRef::new_sized(llretptr, layout)); - } - - if memory_locals.contains(local) { - debug!("alloc: {:?} -> place", local); - if layout.is_unsized() { - LocalRef::UnsizedPlace(PlaceRef::alloca_unsized_indirect(&mut bx, layout)) - } else { - LocalRef::Place(PlaceRef::alloca(&mut bx, layout)) - } - } else { - debug!("alloc: {:?} -> operand", local); - LocalRef::new_operand(&mut bx, layout) - } - }; - - let retptr = allocate_local(mir::RETURN_PLACE); - iter::once(retptr) - .chain(args.into_iter()) - .chain(mir.vars_and_temps_iter().map(allocate_local)) - .collect() - }; - - // Apply debuginfo to the newly allocated locals. - fx.debug_introduce_locals(&mut bx); - - // Codegen the body of each block using reverse postorder - for (bb, _) in traversal::reverse_postorder(&mir) { - fx.codegen_block(bb); - } -} - -/// Produces, for each argument, a `Value` pointing at the -/// argument's value. As arguments are places, these are always -/// indirect. -fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - fx: &mut FunctionCx<'a, 'tcx, Bx>, - memory_locals: &BitSet, -) -> Vec> { - let mir = fx.mir; - let mut idx = 0; - let mut llarg_idx = fx.fn_abi.ret.is_indirect() as usize; - - let mut num_untupled = None; - - let args = mir - .args_iter() - .enumerate() - .map(|(arg_index, local)| { - let arg_decl = &mir.local_decls[local]; - - if Some(local) == mir.spread_arg { - // This argument (e.g., the last argument in the "rust-call" ABI) - // is a tuple that was spread at the ABI level and now we have - // to reconstruct it into a tuple local variable, from multiple - // individual LLVM function arguments. - - let arg_ty = fx.monomorphize(arg_decl.ty); - let ty::Tuple(tupled_arg_tys) = arg_ty.kind() else { - bug!("spread argument isn't a tuple?!"); - }; - - let place = PlaceRef::alloca(bx, bx.layout_of(arg_ty)); - for i in 0..tupled_arg_tys.len() { - let arg = &fx.fn_abi.args[idx]; - idx += 1; - if arg.pad.is_some() { - llarg_idx += 1; - } - let pr_field = place.project_field(bx, i); - bx.store_fn_arg(arg, &mut llarg_idx, pr_field); - } - assert_eq!( - None, - num_untupled.replace(tupled_arg_tys.len()), - "Replaced existing num_tupled" - ); - - return LocalRef::Place(place); - } - - if fx.fn_abi.c_variadic && arg_index == fx.fn_abi.args.len() { - let arg_ty = fx.monomorphize(arg_decl.ty); - - let va_list = PlaceRef::alloca(bx, bx.layout_of(arg_ty)); - bx.va_start(va_list.llval); - - return LocalRef::Place(va_list); - } - - let arg = &fx.fn_abi.args[idx]; - idx += 1; - if arg.pad.is_some() { - llarg_idx += 1; - } - - if !memory_locals.contains(local) { - // We don't have to cast or keep the argument in the alloca. - // FIXME(eddyb): We should figure out how to use llvm.dbg.value instead - // of putting everything in allocas just so we can use llvm.dbg.declare. - let local = |op| LocalRef::Operand(Some(op)); - match arg.mode { - PassMode::Ignore => { - return local(OperandRef::new_zst(bx, arg.layout)); - } - PassMode::Direct(_) => { - let llarg = bx.get_param(llarg_idx); - llarg_idx += 1; - return local(OperandRef::from_immediate_or_packed_pair( - bx, llarg, arg.layout, - )); - } - PassMode::Pair(..) => { - let (a, b) = (bx.get_param(llarg_idx), bx.get_param(llarg_idx + 1)); - llarg_idx += 2; - - return local(OperandRef { - val: OperandValue::Pair(a, b), - layout: arg.layout, - }); - } - _ => {} - } - } - - if arg.is_sized_indirect() { - // Don't copy an indirect argument to an alloca, the caller - // already put it in a temporary alloca and gave it up. - // FIXME: lifetimes - let llarg = bx.get_param(llarg_idx); - llarg_idx += 1; - LocalRef::Place(PlaceRef::new_sized(llarg, arg.layout)) - } else if arg.is_unsized_indirect() { - // As the storage for the indirect argument lives during - // the whole function call, we just copy the fat pointer. - let llarg = bx.get_param(llarg_idx); - llarg_idx += 1; - let llextra = bx.get_param(llarg_idx); - llarg_idx += 1; - let indirect_operand = OperandValue::Pair(llarg, llextra); - - let tmp = PlaceRef::alloca_unsized_indirect(bx, arg.layout); - indirect_operand.store(bx, tmp); - LocalRef::UnsizedPlace(tmp) - } else { - let tmp = PlaceRef::alloca(bx, arg.layout); - bx.store_fn_arg(arg, &mut llarg_idx, tmp); - LocalRef::Place(tmp) - } - }) - .collect::>(); - - if fx.instance.def.requires_caller_location(bx.tcx()) { - let mir_args = if let Some(num_untupled) = num_untupled { - // Subtract off the tupled argument that gets 'expanded' - args.len() - 1 + num_untupled - } else { - args.len() - }; - assert_eq!( - fx.fn_abi.args.len(), - mir_args + 1, - "#[track_caller] instance {:?} must have 1 more argument in their ABI than in their MIR", - fx.instance - ); - - let arg = fx.fn_abi.args.last().unwrap(); - match arg.mode { - PassMode::Direct(_) => (), - _ => bug!("caller location must be PassMode::Direct, found {:?}", arg.mode), - } - - fx.caller_location = Some(OperandRef { - val: OperandValue::Immediate(bx.get_param(llarg_idx)), - layout: arg.layout, - }); - } - - args -} - -mod analyze; -mod block; -pub mod constant; -pub mod coverageinfo; -pub mod debuginfo; -mod intrinsic; -pub mod operand; -pub mod place; -mod rvalue; -mod statement; diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs deleted file mode 100644 index c612634fce2..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ /dev/null @@ -1,461 +0,0 @@ -use super::place::PlaceRef; -use super::{FunctionCx, LocalRef}; - -use crate::base; -use crate::glue; -use crate::traits::*; -use crate::MemFlags; - -use rustc_middle::mir; -use rustc_middle::mir::interpret::{ConstValue, Pointer, Scalar}; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_middle::ty::Ty; -use rustc_target::abi::{Abi, Align, Size}; - -use std::fmt; - -/// The representation of a Rust value. The enum variant is in fact -/// uniquely determined by the value's type, but is kept as a -/// safety check. -#[derive(Copy, Clone, Debug)] -pub enum OperandValue { - /// A reference to the actual operand. The data is guaranteed - /// to be valid for the operand's lifetime. - /// The second value, if any, is the extra data (vtable or length) - /// which indicates that it refers to an unsized rvalue. - Ref(V, Option, Align), - /// A single LLVM value. - Immediate(V), - /// A pair of immediate LLVM values. Used by fat pointers too. - Pair(V, V), -} - -/// An `OperandRef` is an "SSA" reference to a Rust value, along with -/// its type. -/// -/// NOTE: unless you know a value's type exactly, you should not -/// generate LLVM opcodes acting on it and instead act via methods, -/// to avoid nasty edge cases. In particular, using `Builder::store` -/// directly is sure to cause problems -- use `OperandRef::store` -/// instead. -#[derive(Copy, Clone)] -pub struct OperandRef<'tcx, V> { - // The value. - pub val: OperandValue, - - // The layout of value, based on its Rust type. - pub layout: TyAndLayout<'tcx>, -} - -impl fmt::Debug for OperandRef<'_, V> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout) - } -} - -impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { - pub fn new_zst>( - bx: &mut Bx, - layout: TyAndLayout<'tcx>, - ) -> OperandRef<'tcx, V> { - assert!(layout.is_zst()); - OperandRef { - val: OperandValue::Immediate(bx.const_undef(bx.immediate_backend_type(layout))), - layout, - } - } - - pub fn from_const>( - bx: &mut Bx, - val: ConstValue<'tcx>, - ty: Ty<'tcx>, - ) -> Self { - let layout = bx.layout_of(ty); - - if layout.is_zst() { - return OperandRef::new_zst(bx, layout); - } - - let val = match val { - ConstValue::Scalar(x) => { - let Abi::Scalar(scalar) = layout.abi else { - bug!("from_const: invalid ByVal layout: {:#?}", layout); - }; - let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout)); - OperandValue::Immediate(llval) - } - ConstValue::ZeroSized => { - let llval = bx.zst_to_backend(bx.immediate_backend_type(layout)); - OperandValue::Immediate(llval) - } - ConstValue::Slice { data, start, end } => { - let Abi::ScalarPair(a_scalar, _) = layout.abi else { - bug!("from_const: invalid ScalarPair layout: {:#?}", layout); - }; - let a = Scalar::from_pointer( - Pointer::new(bx.tcx().create_memory_alloc(data), Size::from_bytes(start)), - &bx.tcx(), - ); - let a_llval = bx.scalar_to_backend( - a, - a_scalar, - bx.scalar_pair_element_backend_type(layout, 0, true), - ); - let b_llval = bx.const_usize((end - start) as u64); - OperandValue::Pair(a_llval, b_llval) - } - ConstValue::ByRef { alloc, offset } => { - return bx.load_operand(bx.from_const_alloc(layout, alloc, offset)); - } - }; - - OperandRef { val, layout } - } - - /// Asserts that this operand refers to a scalar and returns - /// a reference to its value. - pub fn immediate(self) -> V { - match self.val { - OperandValue::Immediate(s) => s, - _ => bug!("not immediate: {:?}", self), - } - } - - pub fn deref>(self, cx: &Cx) -> PlaceRef<'tcx, V> { - if self.layout.ty.is_box() { - bug!("dereferencing {:?} in codegen", self.layout.ty); - } - - let projected_ty = self - .layout - .ty - .builtin_deref(true) - .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self)) - .ty; - - let (llptr, llextra) = match self.val { - OperandValue::Immediate(llptr) => (llptr, None), - OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)), - OperandValue::Ref(..) => bug!("Deref of by-Ref operand {:?}", self), - }; - let layout = cx.layout_of(projected_ty); - PlaceRef { llval: llptr, llextra, layout, align: layout.align.abi } - } - - /// If this operand is a `Pair`, we return an aggregate with the two values. - /// For other cases, see `immediate`. - pub fn immediate_or_packed_pair>( - self, - bx: &mut Bx, - ) -> V { - if let OperandValue::Pair(a, b) = self.val { - let llty = bx.cx().backend_type(self.layout); - debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty); - // Reconstruct the immediate aggregate. - let mut llpair = bx.cx().const_undef(llty); - let imm_a = bx.from_immediate(a); - let imm_b = bx.from_immediate(b); - llpair = bx.insert_value(llpair, imm_a, 0); - llpair = bx.insert_value(llpair, imm_b, 1); - llpair - } else { - self.immediate() - } - } - - /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`. - pub fn from_immediate_or_packed_pair>( - bx: &mut Bx, - llval: V, - layout: TyAndLayout<'tcx>, - ) -> Self { - let val = if let Abi::ScalarPair(a, b) = layout.abi { - debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval, layout); - - // Deconstruct the immediate aggregate. - let a_llval = bx.extract_value(llval, 0); - let a_llval = bx.to_immediate_scalar(a_llval, a); - let b_llval = bx.extract_value(llval, 1); - let b_llval = bx.to_immediate_scalar(b_llval, b); - OperandValue::Pair(a_llval, b_llval) - } else { - OperandValue::Immediate(llval) - }; - OperandRef { val, layout } - } - - pub fn extract_field>( - &self, - bx: &mut Bx, - i: usize, - ) -> Self { - let field = self.layout.field(bx.cx(), i); - let offset = self.layout.fields.offset(i); - - let mut val = match (self.val, self.layout.abi) { - // If the field is ZST, it has no data. - _ if field.is_zst() => { - return OperandRef::new_zst(bx, field); - } - - // Newtype of a scalar, scalar pair or vector. - (OperandValue::Immediate(_) | OperandValue::Pair(..), _) - if field.size == self.layout.size => - { - assert_eq!(offset.bytes(), 0); - self.val - } - - // Extract a scalar component from a pair. - (OperandValue::Pair(a_llval, b_llval), Abi::ScalarPair(a, b)) => { - if offset.bytes() == 0 { - assert_eq!(field.size, a.size(bx.cx())); - OperandValue::Immediate(a_llval) - } else { - assert_eq!(offset, a.size(bx.cx()).align_to(b.align(bx.cx()).abi)); - assert_eq!(field.size, b.size(bx.cx())); - OperandValue::Immediate(b_llval) - } - } - - // `#[repr(simd)]` types are also immediate. - (OperandValue::Immediate(llval), Abi::Vector { .. }) => { - OperandValue::Immediate(bx.extract_element(llval, bx.cx().const_usize(i as u64))) - } - - _ => bug!("OperandRef::extract_field({:?}): not applicable", self), - }; - - match (&mut val, field.abi) { - (OperandValue::Immediate(llval), _) => { - // Bools in union fields needs to be truncated. - *llval = bx.to_immediate(*llval, field); - // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. - *llval = bx.bitcast(*llval, bx.cx().immediate_backend_type(field)); - } - (OperandValue::Pair(a, b), Abi::ScalarPair(a_abi, b_abi)) => { - // Bools in union fields needs to be truncated. - *a = bx.to_immediate_scalar(*a, a_abi); - *b = bx.to_immediate_scalar(*b, b_abi); - // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types. - *a = bx.bitcast(*a, bx.cx().scalar_pair_element_backend_type(field, 0, true)); - *b = bx.bitcast(*b, bx.cx().scalar_pair_element_backend_type(field, 1, true)); - } - (OperandValue::Pair(..), _) => bug!(), - (OperandValue::Ref(..), _) => bug!(), - } - - OperandRef { val, layout: field } - } -} - -impl<'a, 'tcx, V: CodegenObject> OperandValue { - pub fn store>( - self, - bx: &mut Bx, - dest: PlaceRef<'tcx, V>, - ) { - self.store_with_flags(bx, dest, MemFlags::empty()); - } - - pub fn volatile_store>( - self, - bx: &mut Bx, - dest: PlaceRef<'tcx, V>, - ) { - self.store_with_flags(bx, dest, MemFlags::VOLATILE); - } - - pub fn unaligned_volatile_store>( - self, - bx: &mut Bx, - dest: PlaceRef<'tcx, V>, - ) { - self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED); - } - - pub fn nontemporal_store>( - self, - bx: &mut Bx, - dest: PlaceRef<'tcx, V>, - ) { - self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL); - } - - fn store_with_flags>( - self, - bx: &mut Bx, - dest: PlaceRef<'tcx, V>, - flags: MemFlags, - ) { - debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest); - // Avoid generating stores of zero-sized values, because the only way to have a zero-sized - // value is through `undef`, and store itself is useless. - if dest.layout.is_zst() { - return; - } - match self { - OperandValue::Ref(r, None, source_align) => { - if flags.contains(MemFlags::NONTEMPORAL) { - // HACK(nox): This is inefficient but there is no nontemporal memcpy. - let ty = bx.backend_type(dest.layout); - let ptr = bx.pointercast(r, bx.type_ptr_to(ty)); - let val = bx.load(ty, ptr, source_align); - bx.store_with_flags(val, dest.llval, dest.align, flags); - return; - } - base::memcpy_ty(bx, dest.llval, dest.align, r, source_align, dest.layout, flags) - } - OperandValue::Ref(_, Some(_), _) => { - bug!("cannot directly store unsized values"); - } - OperandValue::Immediate(s) => { - let val = bx.from_immediate(s); - bx.store_with_flags(val, dest.llval, dest.align, flags); - } - OperandValue::Pair(a, b) => { - let Abi::ScalarPair(a_scalar, b_scalar) = dest.layout.abi else { - bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout); - }; - let ty = bx.backend_type(dest.layout); - let b_offset = a_scalar.size(bx).align_to(b_scalar.align(bx).abi); - - let llptr = bx.struct_gep(ty, dest.llval, 0); - let val = bx.from_immediate(a); - let align = dest.align; - bx.store_with_flags(val, llptr, align, flags); - - let llptr = bx.struct_gep(ty, dest.llval, 1); - let val = bx.from_immediate(b); - let align = dest.align.restrict_for_offset(b_offset); - bx.store_with_flags(val, llptr, align, flags); - } - } - } - - pub fn store_unsized>( - self, - bx: &mut Bx, - indirect_dest: PlaceRef<'tcx, V>, - ) { - debug!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest); - let flags = MemFlags::empty(); - - // `indirect_dest` must have `*mut T` type. We extract `T` out of it. - let unsized_ty = indirect_dest - .layout - .ty - .builtin_deref(true) - .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest)) - .ty; - - let OperandValue::Ref(llptr, Some(llextra), _) = self else { - bug!("store_unsized called with a sized value") - }; - - // FIXME: choose an appropriate alignment, or use dynamic align somehow - let max_align = Align::from_bits(128).unwrap(); - let min_align = Align::from_bits(8).unwrap(); - - // Allocate an appropriate region on the stack, and copy the value into it - let (llsize, _) = glue::size_and_align_of_dst(bx, unsized_ty, Some(llextra)); - let lldst = bx.array_alloca(bx.cx().type_i8(), llsize, max_align); - bx.memcpy(lldst, max_align, llptr, min_align, llsize, flags); - - // Store the allocated region and the extra to the indirect place. - let indirect_operand = OperandValue::Pair(lldst, llextra); - indirect_operand.store(bx, indirect_dest); - } -} - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - fn maybe_codegen_consume_direct( - &mut self, - bx: &mut Bx, - place_ref: mir::PlaceRef<'tcx>, - ) -> Option> { - debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref); - - match self.locals[place_ref.local] { - LocalRef::Operand(Some(mut o)) => { - // Moves out of scalar and scalar pair fields are trivial. - for elem in place_ref.projection.iter() { - match elem { - mir::ProjectionElem::Field(ref f, _) => { - o = o.extract_field(bx, f.index()); - } - mir::ProjectionElem::Index(_) - | mir::ProjectionElem::ConstantIndex { .. } => { - // ZSTs don't require any actual memory access. - // FIXME(eddyb) deduplicate this with the identical - // checks in `codegen_consume` and `extract_field`. - let elem = o.layout.field(bx.cx(), 0); - if elem.is_zst() { - o = OperandRef::new_zst(bx, elem); - } else { - return None; - } - } - _ => return None, - } - } - - Some(o) - } - LocalRef::Operand(None) => { - bug!("use of {:?} before def", place_ref); - } - LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => { - // watch out for locals that do not have an - // alloca; they are handled somewhat differently - None - } - } - } - - pub fn codegen_consume( - &mut self, - bx: &mut Bx, - place_ref: mir::PlaceRef<'tcx>, - ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_consume(place_ref={:?})", place_ref); - - let ty = self.monomorphized_place_ty(place_ref); - let layout = bx.cx().layout_of(ty); - - // ZSTs don't require any actual memory access. - if layout.is_zst() { - return OperandRef::new_zst(bx, layout); - } - - if let Some(o) = self.maybe_codegen_consume_direct(bx, place_ref) { - return o; - } - - // for most places, to consume them we just load them - // out from their home - let place = self.codegen_place(bx, place_ref); - bx.load_operand(place) - } - - pub fn codegen_operand( - &mut self, - bx: &mut Bx, - operand: &mir::Operand<'tcx>, - ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_operand(operand={:?})", operand); - - match *operand { - mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => { - self.codegen_consume(bx, place.as_ref()) - } - - mir::Operand::Constant(ref constant) => { - // This cannot fail because we checked all required_consts in advance. - self.eval_mir_constant_to_operand(bx, constant).unwrap_or_else(|_err| { - span_bug!(constant.span, "erroneous constant not captured by required_consts") - }) - } - } - } -} diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs deleted file mode 100644 index 268c4d76503..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ /dev/null @@ -1,549 +0,0 @@ -use super::operand::OperandValue; -use super::{FunctionCx, LocalRef}; - -use crate::common::IntPredicate; -use crate::glue; -use crate::traits::*; -use crate::MemFlags; - -use rustc_middle::mir; -use rustc_middle::mir::tcx::PlaceTy; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; -use rustc_middle::ty::{self, Ty}; -use rustc_target::abi::{Abi, Align, FieldsShape, Int, TagEncoding}; -use rustc_target::abi::{VariantIdx, Variants}; - -#[derive(Copy, Clone, Debug)] -pub struct PlaceRef<'tcx, V> { - /// A pointer to the contents of the place. - pub llval: V, - - /// This place's extra data if it is unsized, or `None` if null. - pub llextra: Option, - - /// The monomorphized type of this place, including variant information. - pub layout: TyAndLayout<'tcx>, - - /// The alignment we know for this place. - pub align: Align, -} - -impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { - pub fn new_sized(llval: V, layout: TyAndLayout<'tcx>) -> PlaceRef<'tcx, V> { - assert!(!layout.is_unsized()); - PlaceRef { llval, llextra: None, layout, align: layout.align.abi } - } - - pub fn new_sized_aligned( - llval: V, - layout: TyAndLayout<'tcx>, - align: Align, - ) -> PlaceRef<'tcx, V> { - assert!(!layout.is_unsized()); - PlaceRef { llval, llextra: None, layout, align } - } - - // FIXME(eddyb) pass something else for the name so no work is done - // unless LLVM IR names are turned on (e.g. for `--emit=llvm-ir`). - pub fn alloca>( - bx: &mut Bx, - layout: TyAndLayout<'tcx>, - ) -> Self { - assert!(!layout.is_unsized(), "tried to statically allocate unsized place"); - let tmp = bx.alloca(bx.cx().backend_type(layout), layout.align.abi); - Self::new_sized(tmp, layout) - } - - /// Returns a place for an indirect reference to an unsized place. - // FIXME(eddyb) pass something else for the name so no work is done - // unless LLVM IR names are turned on (e.g. for `--emit=llvm-ir`). - pub fn alloca_unsized_indirect>( - bx: &mut Bx, - layout: TyAndLayout<'tcx>, - ) -> Self { - assert!(layout.is_unsized(), "tried to allocate indirect place for sized values"); - let ptr_ty = bx.cx().tcx().mk_mut_ptr(layout.ty); - let ptr_layout = bx.cx().layout_of(ptr_ty); - Self::alloca(bx, ptr_layout) - } - - pub fn len>(&self, cx: &Cx) -> V { - if let FieldsShape::Array { count, .. } = self.layout.fields { - if self.layout.is_unsized() { - assert_eq!(count, 0); - self.llextra.unwrap() - } else { - cx.const_usize(count) - } - } else { - bug!("unexpected layout `{:#?}` in PlaceRef::len", self.layout) - } - } -} - -impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { - /// Access a field, at a point when the value's case is known. - pub fn project_field>( - self, - bx: &mut Bx, - ix: usize, - ) -> Self { - let field = self.layout.field(bx.cx(), ix); - let offset = self.layout.fields.offset(ix); - let effective_field_align = self.align.restrict_for_offset(offset); - - let mut simple = || { - let llval = match self.layout.abi { - _ if offset.bytes() == 0 => { - // Unions and newtypes only use an offset of 0. - // Also handles the first field of Scalar, ScalarPair, and Vector layouts. - self.llval - } - Abi::ScalarPair(a, b) - if offset == a.size(bx.cx()).align_to(b.align(bx.cx()).abi) => - { - // Offset matches second field. - let ty = bx.backend_type(self.layout); - bx.struct_gep(ty, self.llval, 1) - } - Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } if field.is_zst() => { - // ZST fields are not included in Scalar, ScalarPair, and Vector layouts, so manually offset the pointer. - let byte_ptr = bx.pointercast(self.llval, bx.cx().type_i8p()); - bx.gep(bx.cx().type_i8(), byte_ptr, &[bx.const_usize(offset.bytes())]) - } - Abi::Scalar(_) | Abi::ScalarPair(..) => { - // All fields of Scalar and ScalarPair layouts must have been handled by this point. - // Vector layouts have additional fields for each element of the vector, so don't panic in that case. - bug!( - "offset of non-ZST field `{:?}` does not match layout `{:#?}`", - field, - self.layout - ); - } - _ => { - let ty = bx.backend_type(self.layout); - bx.struct_gep(ty, self.llval, bx.cx().backend_field_index(self.layout, ix)) - } - }; - PlaceRef { - // HACK(eddyb): have to bitcast pointers until LLVM removes pointee types. - llval: bx.pointercast(llval, bx.cx().type_ptr_to(bx.cx().backend_type(field))), - llextra: if bx.cx().type_has_metadata(field.ty) { self.llextra } else { None }, - layout: field, - align: effective_field_align, - } - }; - - // Simple cases, which don't need DST adjustment: - // * no metadata available - just log the case - // * known alignment - sized types, `[T]`, `str` or a foreign type - // * packed struct - there is no alignment padding - match field.ty.kind() { - _ if self.llextra.is_none() => { - debug!( - "unsized field `{}`, of `{:?}` has no metadata for adjustment", - ix, self.llval - ); - return simple(); - } - _ if !field.is_unsized() => return simple(), - ty::Slice(..) | ty::Str | ty::Foreign(..) => return simple(), - ty::Adt(def, _) => { - if def.repr().packed() { - // FIXME(eddyb) generalize the adjustment when we - // start supporting packing to larger alignments. - assert_eq!(self.layout.align.abi.bytes(), 1); - return simple(); - } - } - _ => {} - } - - // We need to get the pointer manually now. - // We do this by casting to a `*i8`, then offsetting it by the appropriate amount. - // We do this instead of, say, simply adjusting the pointer from the result of a GEP - // because the field may have an arbitrary alignment in the LLVM representation - // anyway. - // - // To demonstrate: - // - // struct Foo { - // x: u16, - // y: T - // } - // - // The type `Foo>` is represented in LLVM as `{ u16, { u16, u8 }}`, meaning that - // the `y` field has 16-bit alignment. - - let meta = self.llextra; - - let unaligned_offset = bx.cx().const_usize(offset.bytes()); - - // Get the alignment of the field - let (_, unsized_align) = glue::size_and_align_of_dst(bx, field.ty, meta); - - // Bump the unaligned offset up to the appropriate alignment - let offset = round_up_const_value_to_alignment(bx, unaligned_offset, unsized_align); - - debug!("struct_field_ptr: DST field offset: {:?}", offset); - - // Cast and adjust pointer. - let byte_ptr = bx.pointercast(self.llval, bx.cx().type_i8p()); - let byte_ptr = bx.gep(bx.cx().type_i8(), byte_ptr, &[offset]); - - // Finally, cast back to the type expected. - let ll_fty = bx.cx().backend_type(field); - debug!("struct_field_ptr: Field type is {:?}", ll_fty); - - PlaceRef { - llval: bx.pointercast(byte_ptr, bx.cx().type_ptr_to(ll_fty)), - llextra: self.llextra, - layout: field, - align: effective_field_align, - } - } - - /// Obtain the actual discriminant of a value. - #[instrument(level = "trace", skip(bx))] - pub fn codegen_get_discr>( - self, - bx: &mut Bx, - cast_to: Ty<'tcx>, - ) -> V { - let cast_to = bx.cx().immediate_backend_type(bx.cx().layout_of(cast_to)); - if self.layout.abi.is_uninhabited() { - return bx.cx().const_undef(cast_to); - } - let (tag_scalar, tag_encoding, tag_field) = match self.layout.variants { - Variants::Single { index } => { - let discr_val = self - .layout - .ty - .discriminant_for_variant(bx.cx().tcx(), index) - .map_or(index.as_u32() as u128, |discr| discr.val); - return bx.cx().const_uint_big(cast_to, discr_val); - } - Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => { - (tag, tag_encoding, tag_field) - } - }; - - // Read the tag/niche-encoded discriminant from memory. - let tag = self.project_field(bx, tag_field); - let tag = bx.load_operand(tag); - - // Decode the discriminant (specifically if it's niche-encoded). - match *tag_encoding { - TagEncoding::Direct => { - let signed = match tag_scalar.primitive() { - // We use `i1` for bytes that are always `0` or `1`, - // e.g., `#[repr(i8)] enum E { A, B }`, but we can't - // let LLVM interpret the `i1` as signed, because - // then `i1 1` (i.e., `E::B`) is effectively `i8 -1`. - Int(_, signed) => !tag_scalar.is_bool() && signed, - _ => false, - }; - bx.intcast(tag.immediate(), cast_to, signed) - } - TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => { - // Rebase from niche values to discriminants, and check - // whether the result is in range for the niche variants. - let niche_llty = bx.cx().immediate_backend_type(tag.layout); - let tag = tag.immediate(); - - // We first compute the "relative discriminant" (wrt `niche_variants`), - // that is, if `n = niche_variants.end() - niche_variants.start()`, - // we remap `niche_start..=niche_start + n` (which may wrap around) - // to (non-wrap-around) `0..=n`, to be able to check whether the - // discriminant corresponds to a niche variant with one comparison. - // We also can't go directly to the (variant index) discriminant - // and check that it is in the range `niche_variants`, because - // that might not fit in the same type, on top of needing an extra - // comparison (see also the comment on `let niche_discr`). - let relative_discr = if niche_start == 0 { - // Avoid subtracting `0`, which wouldn't work for pointers. - // FIXME(eddyb) check the actual primitive type here. - tag - } else { - bx.sub(tag, bx.cx().const_uint_big(niche_llty, niche_start)) - }; - let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32(); - let is_niche = if relative_max == 0 { - // Avoid calling `const_uint`, which wouldn't work for pointers. - // Also use canonical == 0 instead of non-canonical u<= 0. - // FIXME(eddyb) check the actual primitive type here. - bx.icmp(IntPredicate::IntEQ, relative_discr, bx.cx().const_null(niche_llty)) - } else { - let relative_max = bx.cx().const_uint(niche_llty, relative_max as u64); - bx.icmp(IntPredicate::IntULE, relative_discr, relative_max) - }; - - // NOTE(eddyb) this addition needs to be performed on the final - // type, in case the niche itself can't represent all variant - // indices (e.g. `u8` niche with more than `256` variants, - // but enough uninhabited variants so that the remaining variants - // fit in the niche). - // In other words, `niche_variants.end - niche_variants.start` - // is representable in the niche, but `niche_variants.end` - // might not be, in extreme cases. - let niche_discr = { - let relative_discr = if relative_max == 0 { - // HACK(eddyb) since we have only one niche, we know which - // one it is, and we can avoid having a dynamic value here. - bx.cx().const_uint(cast_to, 0) - } else { - bx.intcast(relative_discr, cast_to, false) - }; - bx.add( - relative_discr, - bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64), - ) - }; - - bx.select( - is_niche, - niche_discr, - bx.cx().const_uint(cast_to, dataful_variant.as_u32() as u64), - ) - } - } - } - - /// Sets the discriminant for a new value of the given case of the given - /// representation. - pub fn codegen_set_discr>( - &self, - bx: &mut Bx, - variant_index: VariantIdx, - ) { - if self.layout.for_variant(bx.cx(), variant_index).abi.is_uninhabited() { - // We play it safe by using a well-defined `abort`, but we could go for immediate UB - // if that turns out to be helpful. - bx.abort(); - return; - } - match self.layout.variants { - Variants::Single { index } => { - assert_eq!(index, variant_index); - } - Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } => { - let ptr = self.project_field(bx, tag_field); - let to = - self.layout.ty.discriminant_for_variant(bx.tcx(), variant_index).unwrap().val; - bx.store( - bx.cx().const_uint_big(bx.cx().backend_type(ptr.layout), to), - ptr.llval, - ptr.align, - ); - } - Variants::Multiple { - tag_encoding: - TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start }, - tag_field, - .. - } => { - if variant_index != dataful_variant { - if bx.cx().sess().target.arch == "arm" - || bx.cx().sess().target.arch == "aarch64" - { - // FIXME(#34427): as workaround for LLVM bug on ARM, - // use memset of 0 before assigning niche value. - let fill_byte = bx.cx().const_u8(0); - let size = bx.cx().const_usize(self.layout.size.bytes()); - bx.memset(self.llval, fill_byte, size, self.align, MemFlags::empty()); - } - - let niche = self.project_field(bx, tag_field); - let niche_llty = bx.cx().immediate_backend_type(niche.layout); - let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); - let niche_value = (niche_value as u128).wrapping_add(niche_start); - // FIXME(eddyb): check the actual primitive type here. - let niche_llval = if niche_value == 0 { - // HACK(eddyb): using `c_null` as it works on all types. - bx.cx().const_null(niche_llty) - } else { - bx.cx().const_uint_big(niche_llty, niche_value) - }; - OperandValue::Immediate(niche_llval).store(bx, niche); - } - } - } - } - - pub fn project_index>( - &self, - bx: &mut Bx, - llindex: V, - ) -> Self { - // Statically compute the offset if we can, otherwise just use the element size, - // as this will yield the lowest alignment. - let layout = self.layout.field(bx, 0); - let offset = if let Some(llindex) = bx.const_to_opt_uint(llindex) { - layout.size.checked_mul(llindex, bx).unwrap_or(layout.size) - } else { - layout.size - }; - - PlaceRef { - llval: bx.inbounds_gep( - bx.cx().backend_type(self.layout), - self.llval, - &[bx.cx().const_usize(0), llindex], - ), - llextra: None, - layout, - align: self.align.restrict_for_offset(offset), - } - } - - pub fn project_downcast>( - &self, - bx: &mut Bx, - variant_index: VariantIdx, - ) -> Self { - let mut downcast = *self; - downcast.layout = self.layout.for_variant(bx.cx(), variant_index); - - // Cast to the appropriate variant struct type. - let variant_ty = bx.cx().backend_type(downcast.layout); - downcast.llval = bx.pointercast(downcast.llval, bx.cx().type_ptr_to(variant_ty)); - - downcast - } - - pub fn storage_live>(&self, bx: &mut Bx) { - bx.lifetime_start(self.llval, self.layout.size); - } - - pub fn storage_dead>(&self, bx: &mut Bx) { - bx.lifetime_end(self.llval, self.layout.size); - } -} - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - #[instrument(level = "trace", skip(self, bx))] - pub fn codegen_place( - &mut self, - bx: &mut Bx, - place_ref: mir::PlaceRef<'tcx>, - ) -> PlaceRef<'tcx, Bx::Value> { - let cx = self.cx; - let tcx = self.cx.tcx(); - - let mut base = 0; - let mut cg_base = match self.locals[place_ref.local] { - LocalRef::Place(place) => place, - LocalRef::UnsizedPlace(place) => bx.load_operand(place).deref(cx), - LocalRef::Operand(..) => { - if place_ref.has_deref() { - base = 1; - let cg_base = self.codegen_consume( - bx, - mir::PlaceRef { projection: &place_ref.projection[..0], ..place_ref }, - ); - cg_base.deref(bx.cx()) - } else { - bug!("using operand local {:?} as place", place_ref); - } - } - }; - for elem in place_ref.projection[base..].iter() { - cg_base = match *elem { - mir::ProjectionElem::Deref => bx.load_operand(cg_base).deref(bx.cx()), - mir::ProjectionElem::Field(ref field, _) => { - cg_base.project_field(bx, field.index()) - } - mir::ProjectionElem::Index(index) => { - let index = &mir::Operand::Copy(mir::Place::from(index)); - let index = self.codegen_operand(bx, index); - let llindex = index.immediate(); - cg_base.project_index(bx, llindex) - } - mir::ProjectionElem::ConstantIndex { offset, from_end: false, min_length: _ } => { - let lloffset = bx.cx().const_usize(offset as u64); - cg_base.project_index(bx, lloffset) - } - mir::ProjectionElem::ConstantIndex { offset, from_end: true, min_length: _ } => { - let lloffset = bx.cx().const_usize(offset as u64); - let lllen = cg_base.len(bx.cx()); - let llindex = bx.sub(lllen, lloffset); - cg_base.project_index(bx, llindex) - } - mir::ProjectionElem::Subslice { from, to, from_end } => { - let mut subslice = cg_base.project_index(bx, bx.cx().const_usize(from as u64)); - let projected_ty = - PlaceTy::from_ty(cg_base.layout.ty).projection_ty(tcx, *elem).ty; - subslice.layout = bx.cx().layout_of(self.monomorphize(projected_ty)); - - if subslice.layout.is_unsized() { - assert!(from_end, "slice subslices should be `from_end`"); - subslice.llextra = Some(bx.sub( - cg_base.llextra.unwrap(), - bx.cx().const_usize((from as u64) + (to as u64)), - )); - } - - // Cast the place pointer type to the new - // array or slice type (`*[%_; new_len]`). - subslice.llval = bx.pointercast( - subslice.llval, - bx.cx().type_ptr_to(bx.cx().backend_type(subslice.layout)), - ); - - subslice - } - mir::ProjectionElem::Downcast(_, v) => cg_base.project_downcast(bx, v), - }; - } - debug!("codegen_place(place={:?}) => {:?}", place_ref, cg_base); - cg_base - } - - pub fn monomorphized_place_ty(&self, place_ref: mir::PlaceRef<'tcx>) -> Ty<'tcx> { - let tcx = self.cx.tcx(); - let place_ty = place_ref.ty(self.mir, tcx); - self.monomorphize(place_ty.ty) - } -} - -fn round_up_const_value_to_alignment<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - value: Bx::Value, - align: Bx::Value, -) -> Bx::Value { - // In pseudo code: - // - // if value & (align - 1) == 0 { - // value - // } else { - // (value & !(align - 1)) + align - // } - // - // Usually this is written without branches as - // - // (value + align - 1) & !(align - 1) - // - // But this formula cannot take advantage of constant `value`. E.g. if `value` is known - // at compile time to be `1`, this expression should be optimized to `align`. However, - // optimization only holds if `align` is a power of two. Since the optimizer doesn't know - // that `align` is a power of two, it cannot perform this optimization. - // - // Instead we use - // - // value + (-value & (align - 1)) - // - // Since `align` is used only once, the expression can be optimized. For `value = 0` - // its optimized to `0` even in debug mode. - // - // NB: The previous version of this code used - // - // (value + align - 1) & -align - // - // Even though `-align == !(align - 1)`, LLVM failed to optimize this even for - // `value = 0`. Bug report: https://bugs.llvm.org/show_bug.cgi?id=48559 - let one = bx.const_usize(1); - let align_minus_1 = bx.sub(align, one); - let neg_value = bx.neg(value); - let offset = bx.and(neg_value, align_minus_1); - bx.add(value, offset) -} diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs deleted file mode 100644 index 26b9fbf4428..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ /dev/null @@ -1,729 +0,0 @@ -use super::operand::{OperandRef, OperandValue}; -use super::place::PlaceRef; -use super::{FunctionCx, LocalRef}; - -use crate::base; -use crate::common::{self, IntPredicate}; -use crate::traits::*; -use crate::MemFlags; - -use rustc_middle::mir; -use rustc_middle::mir::Operand; -use rustc_middle::ty::cast::{CastTy, IntTy}; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; -use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt}; -use rustc_span::source_map::{Span, DUMMY_SP}; - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - #[instrument(level = "trace", skip(self, bx))] - pub fn codegen_rvalue( - &mut self, - mut bx: Bx, - dest: PlaceRef<'tcx, Bx::Value>, - rvalue: &mir::Rvalue<'tcx>, - ) -> Bx { - match *rvalue { - mir::Rvalue::Use(ref operand) => { - let cg_operand = self.codegen_operand(&mut bx, operand); - // FIXME: consider not copying constants through stack. (Fixable by codegen'ing - // constants into `OperandValue::Ref`; why don’t we do that yet if we don’t?) - cg_operand.val.store(&mut bx, dest); - bx - } - - mir::Rvalue::Cast(mir::CastKind::Pointer(PointerCast::Unsize), ref source, _) => { - // The destination necessarily contains a fat pointer, so if - // it's a scalar pair, it's a fat pointer or newtype thereof. - if bx.cx().is_backend_scalar_pair(dest.layout) { - // Into-coerce of a thin pointer to a fat pointer -- just - // use the operand path. - let (mut bx, temp) = self.codegen_rvalue_operand(bx, rvalue); - temp.val.store(&mut bx, dest); - return bx; - } - - // Unsize of a nontrivial struct. I would prefer for - // this to be eliminated by MIR building, but - // `CoerceUnsized` can be passed by a where-clause, - // so the (generic) MIR may not be able to expand it. - let operand = self.codegen_operand(&mut bx, source); - match operand.val { - OperandValue::Pair(..) | OperandValue::Immediate(_) => { - // Unsize from an immediate structure. We don't - // really need a temporary alloca here, but - // avoiding it would require us to have - // `coerce_unsized_into` use `extractvalue` to - // index into the struct, and this case isn't - // important enough for it. - debug!("codegen_rvalue: creating ugly alloca"); - let scratch = PlaceRef::alloca(&mut bx, operand.layout); - scratch.storage_live(&mut bx); - operand.val.store(&mut bx, scratch); - base::coerce_unsized_into(&mut bx, scratch, dest); - scratch.storage_dead(&mut bx); - } - OperandValue::Ref(llref, None, align) => { - let source = PlaceRef::new_sized_aligned(llref, operand.layout, align); - base::coerce_unsized_into(&mut bx, source, dest); - } - OperandValue::Ref(_, Some(_), _) => { - bug!("unsized coercion on an unsized rvalue"); - } - } - bx - } - - mir::Rvalue::Repeat(ref elem, count) => { - let cg_elem = self.codegen_operand(&mut bx, elem); - - // Do not generate the loop for zero-sized elements or empty arrays. - if dest.layout.is_zst() { - return bx; - } - - if let OperandValue::Immediate(v) = cg_elem.val { - let zero = bx.const_usize(0); - let start = dest.project_index(&mut bx, zero).llval; - let size = bx.const_usize(dest.layout.size.bytes()); - - // Use llvm.memset.p0i8.* to initialize all zero arrays - if bx.cx().const_to_opt_uint(v) == Some(0) { - let fill = bx.cx().const_u8(0); - bx.memset(start, fill, size, dest.align, MemFlags::empty()); - return bx; - } - - // Use llvm.memset.p0i8.* to initialize byte arrays - let v = bx.from_immediate(v); - if bx.cx().val_ty(v) == bx.cx().type_i8() { - bx.memset(start, v, size, dest.align, MemFlags::empty()); - return bx; - } - } - - let count = - self.monomorphize(count).eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all()); - - bx.write_operand_repeatedly(cg_elem, count, dest) - } - - mir::Rvalue::Aggregate(ref kind, ref operands) => { - let (dest, active_field_index) = match **kind { - mir::AggregateKind::Adt(adt_did, variant_index, _, _, active_field_index) => { - dest.codegen_set_discr(&mut bx, variant_index); - if bx.tcx().adt_def(adt_did).is_enum() { - (dest.project_downcast(&mut bx, variant_index), active_field_index) - } else { - (dest, active_field_index) - } - } - _ => (dest, None), - }; - for (i, operand) in operands.iter().enumerate() { - let op = self.codegen_operand(&mut bx, operand); - // Do not generate stores and GEPis for zero-sized fields. - if !op.layout.is_zst() { - let field_index = active_field_index.unwrap_or(i); - let field = if let mir::AggregateKind::Array(_) = **kind { - let llindex = bx.cx().const_usize(field_index as u64); - dest.project_index(&mut bx, llindex) - } else { - dest.project_field(&mut bx, field_index) - }; - op.val.store(&mut bx, field); - } - } - bx - } - - _ => { - assert!(self.rvalue_creates_operand(rvalue, DUMMY_SP)); - let (mut bx, temp) = self.codegen_rvalue_operand(bx, rvalue); - temp.val.store(&mut bx, dest); - bx - } - } - } - - pub fn codegen_rvalue_unsized( - &mut self, - mut bx: Bx, - indirect_dest: PlaceRef<'tcx, Bx::Value>, - rvalue: &mir::Rvalue<'tcx>, - ) -> Bx { - debug!( - "codegen_rvalue_unsized(indirect_dest.llval={:?}, rvalue={:?})", - indirect_dest.llval, rvalue - ); - - match *rvalue { - mir::Rvalue::Use(ref operand) => { - let cg_operand = self.codegen_operand(&mut bx, operand); - cg_operand.val.store_unsized(&mut bx, indirect_dest); - bx - } - - _ => bug!("unsized assignment other than `Rvalue::Use`"), - } - } - - pub fn codegen_rvalue_operand( - &mut self, - mut bx: Bx, - rvalue: &mir::Rvalue<'tcx>, - ) -> (Bx, OperandRef<'tcx, Bx::Value>) { - assert!( - self.rvalue_creates_operand(rvalue, DUMMY_SP), - "cannot codegen {:?} to operand", - rvalue, - ); - - match *rvalue { - mir::Rvalue::Cast(ref kind, ref source, mir_cast_ty) => { - let operand = self.codegen_operand(&mut bx, source); - debug!("cast operand is {:?}", operand); - let cast = bx.cx().layout_of(self.monomorphize(mir_cast_ty)); - - let val = match *kind { - mir::CastKind::PointerExposeAddress => { - assert!(bx.cx().is_backend_immediate(cast)); - let llptr = operand.immediate(); - let llcast_ty = bx.cx().immediate_backend_type(cast); - let lladdr = bx.ptrtoint(llptr, llcast_ty); - OperandValue::Immediate(lladdr) - } - mir::CastKind::Pointer(PointerCast::ReifyFnPointer) => { - match *operand.layout.ty.kind() { - ty::FnDef(def_id, substs) => { - let instance = ty::Instance::resolve_for_fn_ptr( - bx.tcx(), - ty::ParamEnv::reveal_all(), - def_id, - substs, - ) - .unwrap() - .polymorphize(bx.cx().tcx()); - OperandValue::Immediate(bx.get_fn_addr(instance)) - } - _ => bug!("{} cannot be reified to a fn ptr", operand.layout.ty), - } - } - mir::CastKind::Pointer(PointerCast::ClosureFnPointer(_)) => { - match *operand.layout.ty.kind() { - ty::Closure(def_id, substs) => { - let instance = Instance::resolve_closure( - bx.cx().tcx(), - def_id, - substs, - ty::ClosureKind::FnOnce, - ) - .expect("failed to normalize and resolve closure during codegen") - .polymorphize(bx.cx().tcx()); - OperandValue::Immediate(bx.cx().get_fn_addr(instance)) - } - _ => bug!("{} cannot be cast to a fn ptr", operand.layout.ty), - } - } - mir::CastKind::Pointer(PointerCast::UnsafeFnPointer) => { - // This is a no-op at the LLVM level. - operand.val - } - mir::CastKind::Pointer(PointerCast::Unsize) => { - assert!(bx.cx().is_backend_scalar_pair(cast)); - let (lldata, llextra) = match operand.val { - OperandValue::Pair(lldata, llextra) => { - // unsize from a fat pointer -- this is a - // "trait-object-to-supertrait" coercion. - (lldata, Some(llextra)) - } - OperandValue::Immediate(lldata) => { - // "standard" unsize - (lldata, None) - } - OperandValue::Ref(..) => { - bug!("by-ref operand {:?} in `codegen_rvalue_operand`", operand); - } - }; - let (lldata, llextra) = - base::unsize_ptr(&mut bx, lldata, operand.layout.ty, cast.ty, llextra); - OperandValue::Pair(lldata, llextra) - } - mir::CastKind::Pointer(PointerCast::MutToConstPointer) - | mir::CastKind::Misc - if bx.cx().is_backend_scalar_pair(operand.layout) => - { - if let OperandValue::Pair(data_ptr, meta) = operand.val { - if bx.cx().is_backend_scalar_pair(cast) { - let data_cast = bx.pointercast( - data_ptr, - bx.cx().scalar_pair_element_backend_type(cast, 0, true), - ); - OperandValue::Pair(data_cast, meta) - } else { - // cast to thin-ptr - // Cast of fat-ptr to thin-ptr is an extraction of data-ptr and - // pointer-cast of that pointer to desired pointer type. - let llcast_ty = bx.cx().immediate_backend_type(cast); - let llval = bx.pointercast(data_ptr, llcast_ty); - OperandValue::Immediate(llval) - } - } else { - bug!("unexpected non-pair operand"); - } - } - mir::CastKind::Pointer( - PointerCast::MutToConstPointer | PointerCast::ArrayToPointer, - ) - | mir::CastKind::Misc - // Since int2ptr can have arbitrary integer types as input (so we have to do - // sign extension and all that), it is currently best handled in the same code - // path as the other integer-to-X casts. - | mir::CastKind::PointerFromExposedAddress => { - assert!(bx.cx().is_backend_immediate(cast)); - let ll_t_out = bx.cx().immediate_backend_type(cast); - if operand.layout.abi.is_uninhabited() { - let val = OperandValue::Immediate(bx.cx().const_undef(ll_t_out)); - return (bx, OperandRef { val, layout: cast }); - } - let r_t_in = - CastTy::from_ty(operand.layout.ty).expect("bad input type for cast"); - let r_t_out = CastTy::from_ty(cast.ty).expect("bad output type for cast"); - let ll_t_in = bx.cx().immediate_backend_type(operand.layout); - let llval = operand.immediate(); - - let newval = match (r_t_in, r_t_out) { - (CastTy::Int(i), CastTy::Int(_)) => { - bx.intcast(llval, ll_t_out, i.is_signed()) - } - (CastTy::Float, CastTy::Float) => { - let srcsz = bx.cx().float_width(ll_t_in); - let dstsz = bx.cx().float_width(ll_t_out); - if dstsz > srcsz { - bx.fpext(llval, ll_t_out) - } else if srcsz > dstsz { - bx.fptrunc(llval, ll_t_out) - } else { - llval - } - } - (CastTy::Int(i), CastTy::Float) => { - if i.is_signed() { - bx.sitofp(llval, ll_t_out) - } else { - bx.uitofp(llval, ll_t_out) - } - } - (CastTy::Ptr(_) | CastTy::FnPtr, CastTy::Ptr(_)) => { - bx.pointercast(llval, ll_t_out) - } - (CastTy::Int(i), CastTy::Ptr(_)) => { - let usize_llval = - bx.intcast(llval, bx.cx().type_isize(), i.is_signed()); - bx.inttoptr(usize_llval, ll_t_out) - } - (CastTy::Float, CastTy::Int(IntTy::I)) => { - bx.cast_float_to_int(true, llval, ll_t_out) - } - (CastTy::Float, CastTy::Int(_)) => { - bx.cast_float_to_int(false, llval, ll_t_out) - } - _ => bug!("unsupported cast: {:?} to {:?}", operand.layout.ty, cast.ty), - }; - OperandValue::Immediate(newval) - } - }; - (bx, OperandRef { val, layout: cast }) - } - - mir::Rvalue::Ref(_, bk, place) => { - let mk_ref = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { - tcx.mk_ref( - tcx.lifetimes.re_erased, - ty::TypeAndMut { ty, mutbl: bk.to_mutbl_lossy() }, - ) - }; - self.codegen_place_to_pointer(bx, place, mk_ref) - } - - mir::Rvalue::CopyForDeref(place) => { - let operand = self.codegen_operand(&mut bx, &Operand::Copy(place)); - (bx, operand) - } - mir::Rvalue::AddressOf(mutability, place) => { - let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { - tcx.mk_ptr(ty::TypeAndMut { ty, mutbl: mutability }) - }; - self.codegen_place_to_pointer(bx, place, mk_ptr) - } - - mir::Rvalue::Len(place) => { - let size = self.evaluate_array_len(&mut bx, place); - let operand = OperandRef { - val: OperandValue::Immediate(size), - layout: bx.cx().layout_of(bx.tcx().types.usize), - }; - (bx, operand) - } - - mir::Rvalue::BinaryOp(op, box (ref lhs, ref rhs)) => { - let lhs = self.codegen_operand(&mut bx, lhs); - let rhs = self.codegen_operand(&mut bx, rhs); - let llresult = match (lhs.val, rhs.val) { - ( - OperandValue::Pair(lhs_addr, lhs_extra), - OperandValue::Pair(rhs_addr, rhs_extra), - ) => self.codegen_fat_ptr_binop( - &mut bx, - op, - lhs_addr, - lhs_extra, - rhs_addr, - rhs_extra, - lhs.layout.ty, - ), - - (OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => { - self.codegen_scalar_binop(&mut bx, op, lhs_val, rhs_val, lhs.layout.ty) - } - - _ => bug!(), - }; - let operand = OperandRef { - val: OperandValue::Immediate(llresult), - layout: bx.cx().layout_of(op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty)), - }; - (bx, operand) - } - mir::Rvalue::CheckedBinaryOp(op, box (ref lhs, ref rhs)) => { - let lhs = self.codegen_operand(&mut bx, lhs); - let rhs = self.codegen_operand(&mut bx, rhs); - let result = self.codegen_scalar_checked_binop( - &mut bx, - op, - lhs.immediate(), - rhs.immediate(), - lhs.layout.ty, - ); - let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty); - let operand_ty = bx.tcx().intern_tup(&[val_ty, bx.tcx().types.bool]); - let operand = OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) }; - - (bx, operand) - } - - mir::Rvalue::UnaryOp(op, ref operand) => { - let operand = self.codegen_operand(&mut bx, operand); - let lloperand = operand.immediate(); - let is_float = operand.layout.ty.is_floating_point(); - let llval = match op { - mir::UnOp::Not => bx.not(lloperand), - mir::UnOp::Neg => { - if is_float { - bx.fneg(lloperand) - } else { - bx.neg(lloperand) - } - } - }; - (bx, OperandRef { val: OperandValue::Immediate(llval), layout: operand.layout }) - } - - mir::Rvalue::Discriminant(ref place) => { - let discr_ty = rvalue.ty(self.mir, bx.tcx()); - let discr_ty = self.monomorphize(discr_ty); - let discr = self - .codegen_place(&mut bx, place.as_ref()) - .codegen_get_discr(&mut bx, discr_ty); - ( - bx, - OperandRef { - val: OperandValue::Immediate(discr), - layout: self.cx.layout_of(discr_ty), - }, - ) - } - - mir::Rvalue::NullaryOp(null_op, ty) => { - let ty = self.monomorphize(ty); - assert!(bx.cx().type_is_sized(ty)); - let layout = bx.cx().layout_of(ty); - let val = match null_op { - mir::NullOp::SizeOf => layout.size.bytes(), - mir::NullOp::AlignOf => layout.align.abi.bytes(), - }; - let val = bx.cx().const_usize(val); - let tcx = self.cx.tcx(); - ( - bx, - OperandRef { - val: OperandValue::Immediate(val), - layout: self.cx.layout_of(tcx.types.usize), - }, - ) - } - - mir::Rvalue::ThreadLocalRef(def_id) => { - assert!(bx.cx().tcx().is_static(def_id)); - let static_ = bx.get_static(def_id); - let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id)); - let operand = OperandRef::from_immediate_or_packed_pair(&mut bx, static_, layout); - (bx, operand) - } - mir::Rvalue::Use(ref operand) => { - let operand = self.codegen_operand(&mut bx, operand); - (bx, operand) - } - mir::Rvalue::Repeat(..) | mir::Rvalue::Aggregate(..) => { - // According to `rvalue_creates_operand`, only ZST - // aggregate rvalues are allowed to be operands. - let ty = rvalue.ty(self.mir, self.cx.tcx()); - let operand = - OperandRef::new_zst(&mut bx, self.cx.layout_of(self.monomorphize(ty))); - (bx, operand) - } - mir::Rvalue::ShallowInitBox(ref operand, content_ty) => { - let operand = self.codegen_operand(&mut bx, operand); - let lloperand = operand.immediate(); - - let content_ty = self.monomorphize(content_ty); - let box_layout = bx.cx().layout_of(bx.tcx().mk_box(content_ty)); - let llty_ptr = bx.cx().backend_type(box_layout); - - let val = bx.pointercast(lloperand, llty_ptr); - let operand = OperandRef { val: OperandValue::Immediate(val), layout: box_layout }; - (bx, operand) - } - } - } - - fn evaluate_array_len(&mut self, bx: &mut Bx, place: mir::Place<'tcx>) -> Bx::Value { - // ZST are passed as operands and require special handling - // because codegen_place() panics if Local is operand. - if let Some(index) = place.as_local() { - if let LocalRef::Operand(Some(op)) = self.locals[index] { - if let ty::Array(_, n) = op.layout.ty.kind() { - let n = n.eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all()); - return bx.cx().const_usize(n); - } - } - } - // use common size calculation for non zero-sized types - let cg_value = self.codegen_place(bx, place.as_ref()); - cg_value.len(bx.cx()) - } - - /// Codegen an `Rvalue::AddressOf` or `Rvalue::Ref` - fn codegen_place_to_pointer( - &mut self, - mut bx: Bx, - place: mir::Place<'tcx>, - mk_ptr_ty: impl FnOnce(TyCtxt<'tcx>, Ty<'tcx>) -> Ty<'tcx>, - ) -> (Bx, OperandRef<'tcx, Bx::Value>) { - let cg_place = self.codegen_place(&mut bx, place.as_ref()); - - let ty = cg_place.layout.ty; - - // Note: places are indirect, so storing the `llval` into the - // destination effectively creates a reference. - let val = if !bx.cx().type_has_metadata(ty) { - OperandValue::Immediate(cg_place.llval) - } else { - OperandValue::Pair(cg_place.llval, cg_place.llextra.unwrap()) - }; - (bx, OperandRef { val, layout: self.cx.layout_of(mk_ptr_ty(self.cx.tcx(), ty)) }) - } - - pub fn codegen_scalar_binop( - &mut self, - bx: &mut Bx, - op: mir::BinOp, - lhs: Bx::Value, - rhs: Bx::Value, - input_ty: Ty<'tcx>, - ) -> Bx::Value { - let is_float = input_ty.is_floating_point(); - let is_signed = input_ty.is_signed(); - match op { - mir::BinOp::Add => { - if is_float { - bx.fadd(lhs, rhs) - } else { - bx.add(lhs, rhs) - } - } - mir::BinOp::Sub => { - if is_float { - bx.fsub(lhs, rhs) - } else { - bx.sub(lhs, rhs) - } - } - mir::BinOp::Mul => { - if is_float { - bx.fmul(lhs, rhs) - } else { - bx.mul(lhs, rhs) - } - } - mir::BinOp::Div => { - if is_float { - bx.fdiv(lhs, rhs) - } else if is_signed { - bx.sdiv(lhs, rhs) - } else { - bx.udiv(lhs, rhs) - } - } - mir::BinOp::Rem => { - if is_float { - bx.frem(lhs, rhs) - } else if is_signed { - bx.srem(lhs, rhs) - } else { - bx.urem(lhs, rhs) - } - } - mir::BinOp::BitOr => bx.or(lhs, rhs), - mir::BinOp::BitAnd => bx.and(lhs, rhs), - mir::BinOp::BitXor => bx.xor(lhs, rhs), - mir::BinOp::Offset => { - let pointee_type = input_ty - .builtin_deref(true) - .unwrap_or_else(|| bug!("deref of non-pointer {:?}", input_ty)) - .ty; - let llty = bx.cx().backend_type(bx.cx().layout_of(pointee_type)); - bx.inbounds_gep(llty, lhs, &[rhs]) - } - mir::BinOp::Shl => common::build_unchecked_lshift(bx, lhs, rhs), - mir::BinOp::Shr => common::build_unchecked_rshift(bx, input_ty, lhs, rhs), - mir::BinOp::Ne - | mir::BinOp::Lt - | mir::BinOp::Gt - | mir::BinOp::Eq - | mir::BinOp::Le - | mir::BinOp::Ge => { - if is_float { - bx.fcmp(base::bin_op_to_fcmp_predicate(op.to_hir_binop()), lhs, rhs) - } else { - bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs) - } - } - } - } - - pub fn codegen_fat_ptr_binop( - &mut self, - bx: &mut Bx, - op: mir::BinOp, - lhs_addr: Bx::Value, - lhs_extra: Bx::Value, - rhs_addr: Bx::Value, - rhs_extra: Bx::Value, - _input_ty: Ty<'tcx>, - ) -> Bx::Value { - match op { - mir::BinOp::Eq => { - let lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr); - let rhs = bx.icmp(IntPredicate::IntEQ, lhs_extra, rhs_extra); - bx.and(lhs, rhs) - } - mir::BinOp::Ne => { - let lhs = bx.icmp(IntPredicate::IntNE, lhs_addr, rhs_addr); - let rhs = bx.icmp(IntPredicate::IntNE, lhs_extra, rhs_extra); - bx.or(lhs, rhs) - } - mir::BinOp::Le | mir::BinOp::Lt | mir::BinOp::Ge | mir::BinOp::Gt => { - // a OP b ~ a.0 STRICT(OP) b.0 | (a.0 == b.0 && a.1 OP a.1) - let (op, strict_op) = match op { - mir::BinOp::Lt => (IntPredicate::IntULT, IntPredicate::IntULT), - mir::BinOp::Le => (IntPredicate::IntULE, IntPredicate::IntULT), - mir::BinOp::Gt => (IntPredicate::IntUGT, IntPredicate::IntUGT), - mir::BinOp::Ge => (IntPredicate::IntUGE, IntPredicate::IntUGT), - _ => bug!(), - }; - let lhs = bx.icmp(strict_op, lhs_addr, rhs_addr); - let and_lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr); - let and_rhs = bx.icmp(op, lhs_extra, rhs_extra); - let rhs = bx.and(and_lhs, and_rhs); - bx.or(lhs, rhs) - } - _ => { - bug!("unexpected fat ptr binop"); - } - } - } - - pub fn codegen_scalar_checked_binop( - &mut self, - bx: &mut Bx, - op: mir::BinOp, - lhs: Bx::Value, - rhs: Bx::Value, - input_ty: Ty<'tcx>, - ) -> OperandValue { - // This case can currently arise only from functions marked - // with #[rustc_inherit_overflow_checks] and inlined from - // another crate (mostly core::num generic/#[inline] fns), - // while the current crate doesn't use overflow checks. - if !bx.cx().check_overflow() { - let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty); - return OperandValue::Pair(val, bx.cx().const_bool(false)); - } - - let (val, of) = match op { - // These are checked using intrinsics - mir::BinOp::Add | mir::BinOp::Sub | mir::BinOp::Mul => { - let oop = match op { - mir::BinOp::Add => OverflowOp::Add, - mir::BinOp::Sub => OverflowOp::Sub, - mir::BinOp::Mul => OverflowOp::Mul, - _ => unreachable!(), - }; - bx.checked_binop(oop, input_ty, lhs, rhs) - } - mir::BinOp::Shl | mir::BinOp::Shr => { - let lhs_llty = bx.cx().val_ty(lhs); - let rhs_llty = bx.cx().val_ty(rhs); - let invert_mask = common::shift_mask_val(bx, lhs_llty, rhs_llty, true); - let outer_bits = bx.and(rhs, invert_mask); - - let of = bx.icmp(IntPredicate::IntNE, outer_bits, bx.cx().const_null(rhs_llty)); - let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty); - - (val, of) - } - _ => bug!("Operator `{:?}` is not a checkable operator", op), - }; - - OperandValue::Pair(val, of) - } -} - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool { - match *rvalue { - mir::Rvalue::Ref(..) | - mir::Rvalue::CopyForDeref(..) | - mir::Rvalue::AddressOf(..) | - mir::Rvalue::Len(..) | - mir::Rvalue::Cast(..) | // (*) - mir::Rvalue::ShallowInitBox(..) | // (*) - mir::Rvalue::BinaryOp(..) | - mir::Rvalue::CheckedBinaryOp(..) | - mir::Rvalue::UnaryOp(..) | - mir::Rvalue::Discriminant(..) | - mir::Rvalue::NullaryOp(..) | - mir::Rvalue::ThreadLocalRef(_) | - mir::Rvalue::Use(..) => // (*) - true, - mir::Rvalue::Repeat(..) | - mir::Rvalue::Aggregate(..) => { - let ty = rvalue.ty(self.mir, self.cx.tcx()); - let ty = self.monomorphize(ty); - self.cx.spanned_layout_of(ty, span).is_zst() - } - } - - // (*) this is only true if the type is suitable - } -} diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs deleted file mode 100644 index f452f29883f..00000000000 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ /dev/null @@ -1,102 +0,0 @@ -use rustc_middle::mir; - -use super::FunctionCx; -use super::LocalRef; -use crate::traits::BuilderMethods; -use crate::traits::*; - -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - #[instrument(level = "debug", skip(self, bx))] - pub fn codegen_statement(&mut self, mut bx: Bx, statement: &mir::Statement<'tcx>) -> Bx { - self.set_debug_loc(&mut bx, statement.source_info); - match statement.kind { - mir::StatementKind::Assign(box (ref place, ref rvalue)) => { - if let Some(index) = place.as_local() { - match self.locals[index] { - LocalRef::Place(cg_dest) => self.codegen_rvalue(bx, cg_dest, rvalue), - LocalRef::UnsizedPlace(cg_indirect_dest) => { - self.codegen_rvalue_unsized(bx, cg_indirect_dest, rvalue) - } - LocalRef::Operand(None) => { - let (mut bx, operand) = self.codegen_rvalue_operand(bx, rvalue); - self.locals[index] = LocalRef::Operand(Some(operand)); - self.debug_introduce_local(&mut bx, index); - bx - } - LocalRef::Operand(Some(op)) => { - if !op.layout.is_zst() { - span_bug!( - statement.source_info.span, - "operand {:?} already assigned", - rvalue - ); - } - - // If the type is zero-sized, it's already been set here, - // but we still need to make sure we codegen the operand - self.codegen_rvalue_operand(bx, rvalue).0 - } - } - } else { - let cg_dest = self.codegen_place(&mut bx, place.as_ref()); - self.codegen_rvalue(bx, cg_dest, rvalue) - } - } - mir::StatementKind::SetDiscriminant { box ref place, variant_index } => { - self.codegen_place(&mut bx, place.as_ref()) - .codegen_set_discr(&mut bx, variant_index); - bx - } - mir::StatementKind::Deinit(..) => { - // For now, don't codegen this to anything. In the future it may be worth - // experimenting with what kind of information we can emit to LLVM without hurting - // perf here - bx - } - mir::StatementKind::StorageLive(local) => { - if let LocalRef::Place(cg_place) = self.locals[local] { - cg_place.storage_live(&mut bx); - } else if let LocalRef::UnsizedPlace(cg_indirect_place) = self.locals[local] { - cg_indirect_place.storage_live(&mut bx); - } - bx - } - mir::StatementKind::StorageDead(local) => { - if let LocalRef::Place(cg_place) = self.locals[local] { - cg_place.storage_dead(&mut bx); - } else if let LocalRef::UnsizedPlace(cg_indirect_place) = self.locals[local] { - cg_indirect_place.storage_dead(&mut bx); - } - bx - } - mir::StatementKind::Coverage(box ref coverage) => { - self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope); - bx - } - mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping { - ref src, - ref dst, - ref count, - }) => { - let dst_val = self.codegen_operand(&mut bx, dst); - let src_val = self.codegen_operand(&mut bx, src); - let count = self.codegen_operand(&mut bx, count).immediate(); - let pointee_layout = dst_val - .layout - .pointee_info_at(&bx, rustc_target::abi::Size::ZERO) - .expect("Expected pointer"); - let bytes = bx.mul(count, bx.const_usize(pointee_layout.size.bytes())); - - let align = pointee_layout.align; - let dst = dst_val.immediate(); - let src = src_val.immediate(); - bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty()); - bx - } - mir::StatementKind::FakeRead(..) - | mir::StatementKind::Retag { .. } - | mir::StatementKind::AscribeUserType(..) - | mir::StatementKind::Nop => bx, - } - } -} diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs deleted file mode 100644 index 5006a2157fc..00000000000 --- a/compiler/rustc_codegen_ssa/src/mono_item.rs +++ /dev/null @@ -1,147 +0,0 @@ -use crate::base; -use crate::common; -use crate::traits::*; -use rustc_hir as hir; -use rustc_middle::mir::mono::MonoItem; -use rustc_middle::mir::mono::{Linkage, Visibility}; -use rustc_middle::ty; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; -use rustc_middle::ty::Instance; - -pub trait MonoItemExt<'a, 'tcx> { - fn define>(&self, cx: &'a Bx::CodegenCx); - fn predefine>( - &self, - cx: &'a Bx::CodegenCx, - linkage: Linkage, - visibility: Visibility, - ); - fn to_raw_string(&self) -> String; -} - -impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> { - fn define>(&self, cx: &'a Bx::CodegenCx) { - debug!( - "BEGIN IMPLEMENTING '{} ({})' in cgu {}", - self, - self.to_raw_string(), - cx.codegen_unit().name() - ); - - match *self { - MonoItem::Static(def_id) => { - cx.codegen_static(def_id, cx.tcx().is_mutable_static(def_id)); - } - MonoItem::GlobalAsm(item_id) => { - let item = cx.tcx().hir().item(item_id); - if let hir::ItemKind::GlobalAsm(ref asm) = item.kind { - let operands: Vec<_> = asm - .operands - .iter() - .map(|(op, op_sp)| match *op { - hir::InlineAsmOperand::Const { ref anon_const } => { - let anon_const_def_id = - cx.tcx().hir().local_def_id(anon_const.hir_id).to_def_id(); - let const_value = - cx.tcx().const_eval_poly(anon_const_def_id).unwrap_or_else( - |_| span_bug!(*op_sp, "asm const cannot be resolved"), - ); - let ty = cx - .tcx() - .typeck_body(anon_const.body) - .node_type(anon_const.hir_id); - let string = common::asm_const_to_str( - cx.tcx(), - *op_sp, - const_value, - cx.layout_of(ty), - ); - GlobalAsmOperandRef::Const { string } - } - hir::InlineAsmOperand::SymFn { ref anon_const } => { - let ty = cx - .tcx() - .typeck_body(anon_const.body) - .node_type(anon_const.hir_id); - let instance = match ty.kind() { - &ty::FnDef(def_id, substs) => Instance::new(def_id, substs), - _ => span_bug!(*op_sp, "asm sym is not a function"), - }; - - GlobalAsmOperandRef::SymFn { instance } - } - hir::InlineAsmOperand::SymStatic { path: _, def_id } => { - GlobalAsmOperandRef::SymStatic { def_id } - } - hir::InlineAsmOperand::In { .. } - | hir::InlineAsmOperand::Out { .. } - | hir::InlineAsmOperand::InOut { .. } - | hir::InlineAsmOperand::SplitInOut { .. } => { - span_bug!(*op_sp, "invalid operand type for global_asm!") - } - }) - .collect(); - - cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans); - } else { - span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type") - } - } - MonoItem::Fn(instance) => { - base::codegen_instance::(&cx, instance); - } - } - - debug!( - "END IMPLEMENTING '{} ({})' in cgu {}", - self, - self.to_raw_string(), - cx.codegen_unit().name() - ); - } - - fn predefine>( - &self, - cx: &'a Bx::CodegenCx, - linkage: Linkage, - visibility: Visibility, - ) { - debug!( - "BEGIN PREDEFINING '{} ({})' in cgu {}", - self, - self.to_raw_string(), - cx.codegen_unit().name() - ); - - let symbol_name = self.symbol_name(cx.tcx()).name; - - debug!("symbol {}", &symbol_name); - - match *self { - MonoItem::Static(def_id) => { - cx.predefine_static(def_id, linkage, visibility, &symbol_name); - } - MonoItem::Fn(instance) => { - cx.predefine_fn(instance, linkage, visibility, &symbol_name); - } - MonoItem::GlobalAsm(..) => {} - } - - debug!( - "END PREDEFINING '{} ({})' in cgu {}", - self, - self.to_raw_string(), - cx.codegen_unit().name() - ); - } - - fn to_raw_string(&self) -> String { - match *self { - MonoItem::Fn(instance) => { - format!("Fn({:?}, {})", instance.def, instance.substs.as_ptr().addr()) - } - MonoItem::Static(id) => format!("Static({:?})", id), - MonoItem::GlobalAsm(id) => format!("GlobalAsm({:?})", id), - } - } -} diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs deleted file mode 100644 index ecad0518533..00000000000 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ /dev/null @@ -1,308 +0,0 @@ -use rustc_hir::def_id::LOCAL_CRATE; -use rustc_middle::ty::query::Providers; -use rustc_session::Session; -use rustc_span::symbol::sym; -use rustc_span::symbol::Symbol; - -/// Features that control behaviour of rustc, rather than the codegen. -pub const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"]; - -// When adding features to the below lists -// check whether they're named already elsewhere in rust -// e.g. in stdarch and whether the given name matches LLVM's -// if it doesn't, to_llvm_feature in llvm_util in rustc_codegen_llvm needs to be adapted - -const ARM_ALLOWED_FEATURES: &[(&str, Option)] = &[ - ("aclass", Some(sym::arm_target_feature)), - ("mclass", Some(sym::arm_target_feature)), - ("rclass", Some(sym::arm_target_feature)), - ("dsp", Some(sym::arm_target_feature)), - ("neon", Some(sym::arm_target_feature)), - ("crc", Some(sym::arm_target_feature)), - ("crypto", Some(sym::arm_target_feature)), - ("aes", Some(sym::arm_target_feature)), - ("sha2", Some(sym::arm_target_feature)), - ("i8mm", Some(sym::arm_target_feature)), - ("dotprod", Some(sym::arm_target_feature)), - ("v5te", Some(sym::arm_target_feature)), - ("v6", Some(sym::arm_target_feature)), - ("v6k", Some(sym::arm_target_feature)), - ("v6t2", Some(sym::arm_target_feature)), - ("v7", Some(sym::arm_target_feature)), - ("v8", Some(sym::arm_target_feature)), - ("vfp2", Some(sym::arm_target_feature)), - ("vfp3", Some(sym::arm_target_feature)), - ("vfp4", Some(sym::arm_target_feature)), - ("fp-armv8", Some(sym::arm_target_feature)), - // This is needed for inline assembly, but shouldn't be stabilized as-is - // since it should be enabled per-function using #[instruction_set], not - // #[target_feature]. - ("thumb-mode", Some(sym::arm_target_feature)), - ("thumb2", Some(sym::arm_target_feature)), - ("d32", Some(sym::arm_target_feature)), -]; - -const AARCH64_ALLOWED_FEATURES: &[(&str, Option)] = &[ - // FEAT_AdvSimd & FEAT_FP - ("neon", None), - // FEAT_FP16 - ("fp16", None), - // FEAT_SVE - ("sve", None), - // FEAT_CRC - ("crc", None), - // FEAT_RAS - ("ras", None), - // FEAT_LSE - ("lse", None), - // FEAT_RDM - ("rdm", None), - // FEAT_RCPC - ("rcpc", None), - // FEAT_RCPC2 - ("rcpc2", None), - // FEAT_DotProd - ("dotprod", None), - // FEAT_TME - ("tme", None), - // FEAT_FHM - ("fhm", None), - // FEAT_DIT - ("dit", None), - // FEAT_FLAGM - ("flagm", None), - // FEAT_SSBS - ("ssbs", None), - // FEAT_SB - ("sb", None), - // FEAT_PAUTH (address authentication) - ("paca", None), - // FEAT_PAUTH (generic authentication) - ("pacg", None), - // FEAT_DPB - ("dpb", None), - // FEAT_DPB2 - ("dpb2", None), - // FEAT_SVE2 - ("sve2", None), - // FEAT_SVE2_AES - ("sve2-aes", None), - // FEAT_SVE2_SM4 - ("sve2-sm4", None), - // FEAT_SVE2_SHA3 - ("sve2-sha3", None), - // FEAT_SVE2_BitPerm - ("sve2-bitperm", None), - // FEAT_FRINTTS - ("frintts", None), - // FEAT_I8MM - ("i8mm", None), - // FEAT_F32MM - ("f32mm", None), - // FEAT_F64MM - ("f64mm", None), - // FEAT_BF16 - ("bf16", None), - // FEAT_RAND - ("rand", None), - // FEAT_BTI - ("bti", None), - // FEAT_MTE - ("mte", None), - // FEAT_JSCVT - ("jsconv", None), - // FEAT_FCMA - ("fcma", None), - // FEAT_AES - ("aes", None), - // FEAT_SHA1 & FEAT_SHA256 - ("sha2", None), - // FEAT_SHA512 & FEAT_SHA3 - ("sha3", None), - // FEAT_SM3 & FEAT_SM4 - ("sm4", None), - // FEAT_PAN - ("pan", None), - // FEAT_LOR - ("lor", None), - // FEAT_VHE - ("vh", None), - // FEAT_PMUv3 - ("pmuv3", None), - // FEAT_SPE - ("spe", None), - ("v8.1a", Some(sym::aarch64_ver_target_feature)), - ("v8.2a", Some(sym::aarch64_ver_target_feature)), - ("v8.3a", Some(sym::aarch64_ver_target_feature)), - ("v8.4a", Some(sym::aarch64_ver_target_feature)), - ("v8.5a", Some(sym::aarch64_ver_target_feature)), - ("v8.6a", Some(sym::aarch64_ver_target_feature)), - ("v8.7a", Some(sym::aarch64_ver_target_feature)), -]; - -const AARCH64_TIED_FEATURES: &[&[&str]] = &[ - &["paca", "pacg"], // Together these represent `pauth` in LLVM -]; - -const X86_ALLOWED_FEATURES: &[(&str, Option)] = &[ - ("adx", None), - ("aes", None), - ("avx", None), - ("avx2", None), - ("avx512bf16", Some(sym::avx512_target_feature)), - ("avx512bitalg", Some(sym::avx512_target_feature)), - ("avx512bw", Some(sym::avx512_target_feature)), - ("avx512cd", Some(sym::avx512_target_feature)), - ("avx512dq", Some(sym::avx512_target_feature)), - ("avx512er", Some(sym::avx512_target_feature)), - ("avx512f", Some(sym::avx512_target_feature)), - ("avx512gfni", Some(sym::avx512_target_feature)), - ("avx512ifma", Some(sym::avx512_target_feature)), - ("avx512pf", Some(sym::avx512_target_feature)), - ("avx512vaes", Some(sym::avx512_target_feature)), - ("avx512vbmi", Some(sym::avx512_target_feature)), - ("avx512vbmi2", Some(sym::avx512_target_feature)), - ("avx512vl", Some(sym::avx512_target_feature)), - ("avx512vnni", Some(sym::avx512_target_feature)), - ("avx512vp2intersect", Some(sym::avx512_target_feature)), - ("avx512vpclmulqdq", Some(sym::avx512_target_feature)), - ("avx512vpopcntdq", Some(sym::avx512_target_feature)), - ("bmi1", None), - ("bmi2", None), - ("cmpxchg16b", Some(sym::cmpxchg16b_target_feature)), - ("ermsb", Some(sym::ermsb_target_feature)), - ("f16c", Some(sym::f16c_target_feature)), - ("fma", None), - ("fxsr", None), - ("lzcnt", None), - ("movbe", Some(sym::movbe_target_feature)), - ("pclmulqdq", None), - ("popcnt", None), - ("rdrand", None), - ("rdseed", None), - ("rtm", Some(sym::rtm_target_feature)), - ("sha", None), - ("sse", None), - ("sse2", None), - ("sse3", None), - ("sse4.1", None), - ("sse4.2", None), - ("sse4a", Some(sym::sse4a_target_feature)), - ("ssse3", None), - ("tbm", Some(sym::tbm_target_feature)), - ("xsave", None), - ("xsavec", None), - ("xsaveopt", None), - ("xsaves", None), -]; - -const HEXAGON_ALLOWED_FEATURES: &[(&str, Option)] = &[ - ("hvx", Some(sym::hexagon_target_feature)), - ("hvx-length128b", Some(sym::hexagon_target_feature)), -]; - -const POWERPC_ALLOWED_FEATURES: &[(&str, Option)] = &[ - ("altivec", Some(sym::powerpc_target_feature)), - ("power8-altivec", Some(sym::powerpc_target_feature)), - ("power9-altivec", Some(sym::powerpc_target_feature)), - ("power8-vector", Some(sym::powerpc_target_feature)), - ("power9-vector", Some(sym::powerpc_target_feature)), - ("vsx", Some(sym::powerpc_target_feature)), -]; - -const MIPS_ALLOWED_FEATURES: &[(&str, Option)] = - &[("fp64", Some(sym::mips_target_feature)), ("msa", Some(sym::mips_target_feature))]; - -const RISCV_ALLOWED_FEATURES: &[(&str, Option)] = &[ - ("m", Some(sym::riscv_target_feature)), - ("a", Some(sym::riscv_target_feature)), - ("c", Some(sym::riscv_target_feature)), - ("f", Some(sym::riscv_target_feature)), - ("d", Some(sym::riscv_target_feature)), - ("e", Some(sym::riscv_target_feature)), - ("v", Some(sym::riscv_target_feature)), - ("zfinx", Some(sym::riscv_target_feature)), - ("zdinx", Some(sym::riscv_target_feature)), - ("zhinx", Some(sym::riscv_target_feature)), - ("zhinxmin", Some(sym::riscv_target_feature)), - ("zfh", Some(sym::riscv_target_feature)), - ("zfhmin", Some(sym::riscv_target_feature)), - ("zbkb", Some(sym::riscv_target_feature)), - ("zbkc", Some(sym::riscv_target_feature)), - ("zbkx", Some(sym::riscv_target_feature)), - ("zknd", Some(sym::riscv_target_feature)), - ("zkne", Some(sym::riscv_target_feature)), - ("zknh", Some(sym::riscv_target_feature)), - ("zksed", Some(sym::riscv_target_feature)), - ("zksh", Some(sym::riscv_target_feature)), - ("zkr", Some(sym::riscv_target_feature)), - ("zkn", Some(sym::riscv_target_feature)), - ("zks", Some(sym::riscv_target_feature)), - ("zk", Some(sym::riscv_target_feature)), - ("zkt", Some(sym::riscv_target_feature)), -]; - -const WASM_ALLOWED_FEATURES: &[(&str, Option)] = &[ - ("simd128", None), - ("atomics", Some(sym::wasm_target_feature)), - ("nontrapping-fptoint", Some(sym::wasm_target_feature)), - ("bulk-memory", Some(sym::wasm_target_feature)), - ("mutable-globals", Some(sym::wasm_target_feature)), - ("reference-types", Some(sym::wasm_target_feature)), - ("sign-ext", Some(sym::wasm_target_feature)), -]; - -const BPF_ALLOWED_FEATURES: &[(&str, Option)] = &[("alu32", Some(sym::bpf_target_feature))]; - -/// When rustdoc is running, provide a list of all known features so that all their respective -/// primitives may be documented. -/// -/// IMPORTANT: If you're adding another feature list above, make sure to add it to this iterator! -pub fn all_known_features() -> impl Iterator)> { - std::iter::empty() - .chain(ARM_ALLOWED_FEATURES.iter()) - .chain(AARCH64_ALLOWED_FEATURES.iter()) - .chain(X86_ALLOWED_FEATURES.iter()) - .chain(HEXAGON_ALLOWED_FEATURES.iter()) - .chain(POWERPC_ALLOWED_FEATURES.iter()) - .chain(MIPS_ALLOWED_FEATURES.iter()) - .chain(RISCV_ALLOWED_FEATURES.iter()) - .chain(WASM_ALLOWED_FEATURES.iter()) - .chain(BPF_ALLOWED_FEATURES.iter()) - .cloned() -} - -pub fn supported_target_features(sess: &Session) -> &'static [(&'static str, Option)] { - match &*sess.target.arch { - "arm" => ARM_ALLOWED_FEATURES, - "aarch64" => AARCH64_ALLOWED_FEATURES, - "x86" | "x86_64" => X86_ALLOWED_FEATURES, - "hexagon" => HEXAGON_ALLOWED_FEATURES, - "mips" | "mips64" => MIPS_ALLOWED_FEATURES, - "powerpc" | "powerpc64" => POWERPC_ALLOWED_FEATURES, - "riscv32" | "riscv64" => RISCV_ALLOWED_FEATURES, - "wasm32" | "wasm64" => WASM_ALLOWED_FEATURES, - "bpf" => BPF_ALLOWED_FEATURES, - _ => &[], - } -} - -pub fn tied_target_features(sess: &Session) -> &'static [&'static [&'static str]] { - match &*sess.target.arch { - "aarch64" => AARCH64_TIED_FEATURES, - _ => &[], - } -} - -pub(crate) fn provide(providers: &mut Providers) { - providers.supported_target_features = |tcx, cnum| { - assert_eq!(cnum, LOCAL_CRATE); - if tcx.sess.opts.actually_rustdoc { - // rustdoc needs to be able to document functions that use all the features, so - // whitelist them all - all_known_features().map(|(a, b)| (a.to_string(), b)).collect() - } else { - supported_target_features(tcx.sess).iter().map(|&(a, b)| (a.to_string(), b)).collect() - } - }; -} diff --git a/compiler/rustc_codegen_ssa/src/traits/abi.rs b/compiler/rustc_codegen_ssa/src/traits/abi.rs deleted file mode 100644 index a00d78daf4d..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/abi.rs +++ /dev/null @@ -1,8 +0,0 @@ -use super::BackendTypes; -use rustc_middle::ty::Ty; -use rustc_target::abi::call::FnAbi; - -pub trait AbiBuilderMethods<'tcx>: BackendTypes { - fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value); - fn get_param(&mut self, index: usize) -> Self::Value; -} diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs deleted file mode 100644 index c2ae74b18d8..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/asm.rs +++ /dev/null @@ -1,66 +0,0 @@ -use super::BackendTypes; -use crate::mir::operand::OperandRef; -use crate::mir::place::PlaceRef; -use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; -use rustc_hir::def_id::DefId; -use rustc_middle::ty::Instance; -use rustc_span::Span; -use rustc_target::asm::InlineAsmRegOrRegClass; - -#[derive(Debug)] -pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> { - In { - reg: InlineAsmRegOrRegClass, - value: OperandRef<'tcx, B::Value>, - }, - Out { - reg: InlineAsmRegOrRegClass, - late: bool, - place: Option>, - }, - InOut { - reg: InlineAsmRegOrRegClass, - late: bool, - in_value: OperandRef<'tcx, B::Value>, - out_place: Option>, - }, - Const { - string: String, - }, - SymFn { - instance: Instance<'tcx>, - }, - SymStatic { - def_id: DefId, - }, -} - -#[derive(Debug)] -pub enum GlobalAsmOperandRef<'tcx> { - Const { string: String }, - SymFn { instance: Instance<'tcx> }, - SymStatic { def_id: DefId }, -} - -pub trait AsmBuilderMethods<'tcx>: BackendTypes { - /// Take an inline assembly expression and splat it out via LLVM - fn codegen_inline_asm( - &mut self, - template: &[InlineAsmTemplatePiece], - operands: &[InlineAsmOperandRef<'tcx, Self>], - options: InlineAsmOptions, - line_spans: &[Span], - instance: Instance<'_>, - dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, - ); -} - -pub trait AsmMethods<'tcx> { - fn codegen_global_asm( - &self, - template: &[InlineAsmTemplatePiece], - operands: &[GlobalAsmOperandRef<'tcx>], - options: InlineAsmOptions, - line_spans: &[Span], - ); -} diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs deleted file mode 100644 index 779bd3ea278..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/backend.rs +++ /dev/null @@ -1,161 +0,0 @@ -use super::write::WriteBackendMethods; -use super::CodegenObject; -use crate::back::write::TargetMachineFactoryFn; -use crate::{CodegenResults, ModuleCodegen}; - -use rustc_ast::expand::allocator::AllocatorKind; -use rustc_data_structures::fx::FxHashMap; -use rustc_errors::ErrorGuaranteed; -use rustc_metadata::EncodedMetadata; -use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf, TyAndLayout}; -use rustc_middle::ty::query::{ExternProviders, Providers}; -use rustc_middle::ty::{Ty, TyCtxt}; -use rustc_session::{ - config::{self, OutputFilenames, PrintRequest}, - cstore::MetadataLoaderDyn, - Session, -}; -use rustc_span::symbol::Symbol; -use rustc_target::abi::call::FnAbi; -use rustc_target::spec::Target; - -pub use rustc_data_structures::sync::MetadataRef; - -use std::any::Any; - -pub trait BackendTypes { - type Value: CodegenObject; - type Function: CodegenObject; - - type BasicBlock: Copy; - type Type: CodegenObject; - type Funclet; - - // FIXME(eddyb) find a common convention for all of the debuginfo-related - // names (choose between `Dbg`, `Debug`, `DebugInfo`, `DI` etc.). - type DIScope: Copy; - type DILocation: Copy; - type DIVariable: Copy; -} - -pub trait Backend<'tcx>: - Sized - + BackendTypes - + HasTyCtxt<'tcx> - + LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>> - + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>> -{ -} - -impl<'tcx, T> Backend<'tcx> for T where - Self: BackendTypes - + HasTyCtxt<'tcx> - + LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>> - + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>> -{ -} - -pub trait CodegenBackend { - fn init(&self, _sess: &Session) {} - fn print(&self, _req: PrintRequest, _sess: &Session) {} - fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec { - vec![] - } - fn print_passes(&self) {} - fn print_version(&self) {} - - /// If this plugin provides additional builtin targets, provide the one enabled by the options here. - /// Be careful: this is called *before* init() is called. - fn target_override(&self, _opts: &config::Options) -> Option { - None - } - - /// The metadata loader used to load rlib and dylib metadata. - /// - /// Alternative codegen backends may want to use different rlib or dylib formats than the - /// default native static archives and dynamic libraries. - fn metadata_loader(&self) -> Box { - Box::new(crate::back::metadata::DefaultMetadataLoader) - } - - fn provide(&self, _providers: &mut Providers) {} - fn provide_extern(&self, _providers: &mut ExternProviders) {} - fn codegen_crate<'tcx>( - &self, - tcx: TyCtxt<'tcx>, - metadata: EncodedMetadata, - need_metadata_module: bool, - ) -> Box; - - /// This is called on the returned `Box` from `codegen_backend` - /// - /// # Panics - /// - /// Panics when the passed `Box` was not returned by `codegen_backend`. - fn join_codegen( - &self, - ongoing_codegen: Box, - sess: &Session, - outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed>; - - /// This is called on the returned `Box` from `join_codegen` - /// - /// # Panics - /// - /// Panics when the passed `Box` was not returned by `join_codegen`. - fn link( - &self, - sess: &Session, - codegen_results: CodegenResults, - outputs: &OutputFilenames, - ) -> Result<(), ErrorGuaranteed>; -} - -pub trait ExtraBackendMethods: CodegenBackend + WriteBackendMethods + Sized + Send + Sync { - fn codegen_allocator<'tcx>( - &self, - tcx: TyCtxt<'tcx>, - module_name: &str, - kind: AllocatorKind, - has_alloc_error_handler: bool, - ) -> Self::Module; - /// This generates the codegen unit and returns it along with - /// a `u64` giving an estimate of the unit's processing cost. - fn compile_codegen_unit( - &self, - tcx: TyCtxt<'_>, - cgu_name: Symbol, - ) -> (ModuleCodegen, u64); - fn target_machine_factory( - &self, - sess: &Session, - opt_level: config::OptLevel, - target_features: &[String], - ) -> TargetMachineFactoryFn; - fn target_cpu<'b>(&self, sess: &'b Session) -> &'b str; - fn tune_cpu<'b>(&self, sess: &'b Session) -> Option<&'b str>; - - fn spawn_thread(_time_trace: bool, f: F) -> std::thread::JoinHandle - where - F: FnOnce() -> T, - F: Send + 'static, - T: Send + 'static, - { - std::thread::spawn(f) - } - - fn spawn_named_thread( - _time_trace: bool, - name: String, - f: F, - ) -> std::io::Result> - where - F: FnOnce() -> T, - F: Send + 'static, - T: Send + 'static, - { - std::thread::Builder::new().name(name).spawn(f) - } -} diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs deleted file mode 100644 index 9f49749bb41..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ /dev/null @@ -1,481 +0,0 @@ -use super::abi::AbiBuilderMethods; -use super::asm::AsmBuilderMethods; -use super::consts::ConstMethods; -use super::coverageinfo::CoverageInfoBuilderMethods; -use super::debuginfo::DebugInfoBuilderMethods; -use super::intrinsic::IntrinsicCallMethods; -use super::misc::MiscMethods; -use super::type_::{ArgAbiMethods, BaseTypeMethods}; -use super::{HasCodegen, StaticBuilderMethods}; - -use crate::common::{ - AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind, -}; -use crate::mir::operand::OperandRef; -use crate::mir::place::PlaceRef; -use crate::MemFlags; - -use rustc_apfloat::{ieee, Float, Round, Status}; -use rustc_middle::ty::layout::{HasParamEnv, TyAndLayout}; -use rustc_middle::ty::Ty; -use rustc_span::Span; -use rustc_target::abi::{Abi, Align, Scalar, Size, WrappingRange}; -use rustc_target::spec::HasTargetSpec; - -#[derive(Copy, Clone)] -pub enum OverflowOp { - Add, - Sub, - Mul, -} - -pub trait BuilderMethods<'a, 'tcx>: - HasCodegen<'tcx> - + CoverageInfoBuilderMethods<'tcx> - + DebugInfoBuilderMethods - + ArgAbiMethods<'tcx> - + AbiBuilderMethods<'tcx> - + IntrinsicCallMethods<'tcx> - + AsmBuilderMethods<'tcx> - + StaticBuilderMethods - + HasParamEnv<'tcx> - + HasTargetSpec -{ - fn build(cx: &'a Self::CodegenCx, llbb: Self::BasicBlock) -> Self; - - fn cx(&self) -> &Self::CodegenCx; - fn llbb(&self) -> Self::BasicBlock; - - fn set_span(&mut self, span: Span); - - // FIXME(eddyb) replace uses of this with `append_sibling_block`. - fn append_block(cx: &'a Self::CodegenCx, llfn: Self::Function, name: &str) -> Self::BasicBlock; - - fn append_sibling_block(&mut self, name: &str) -> Self::BasicBlock; - - fn switch_to_block(&mut self, llbb: Self::BasicBlock); - - fn ret_void(&mut self); - fn ret(&mut self, v: Self::Value); - fn br(&mut self, dest: Self::BasicBlock); - fn cond_br( - &mut self, - cond: Self::Value, - then_llbb: Self::BasicBlock, - else_llbb: Self::BasicBlock, - ); - fn switch( - &mut self, - v: Self::Value, - else_llbb: Self::BasicBlock, - cases: impl ExactSizeIterator, - ); - fn invoke( - &mut self, - llty: Self::Type, - llfn: Self::Value, - args: &[Self::Value], - then: Self::BasicBlock, - catch: Self::BasicBlock, - funclet: Option<&Self::Funclet>, - ) -> Self::Value; - fn unreachable(&mut self); - - fn add(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn fadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn fadd_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn sub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn fsub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn fsub_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn mul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn fmul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn fmul_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn udiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn exactudiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn sdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn exactsdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn fdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn fdiv_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn urem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn srem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn frem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn frem_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn shl(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn lshr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_ssub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_usub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_smul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn and(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn or(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn xor(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn neg(&mut self, v: Self::Value) -> Self::Value; - fn fneg(&mut self, v: Self::Value) -> Self::Value; - fn not(&mut self, v: Self::Value) -> Self::Value; - - fn checked_binop( - &mut self, - oop: OverflowOp, - ty: Ty<'_>, - lhs: Self::Value, - rhs: Self::Value, - ) -> (Self::Value, Self::Value); - - fn from_immediate(&mut self, val: Self::Value) -> Self::Value; - fn to_immediate(&mut self, val: Self::Value, layout: TyAndLayout<'_>) -> Self::Value { - if let Abi::Scalar(scalar) = layout.abi { - self.to_immediate_scalar(val, scalar) - } else { - val - } - } - fn to_immediate_scalar(&mut self, val: Self::Value, scalar: Scalar) -> Self::Value; - - fn alloca(&mut self, ty: Self::Type, align: Align) -> Self::Value; - fn dynamic_alloca(&mut self, ty: Self::Type, align: Align) -> Self::Value; - fn array_alloca(&mut self, ty: Self::Type, len: Self::Value, align: Align) -> Self::Value; - - fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value; - fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value; - fn atomic_load( - &mut self, - ty: Self::Type, - ptr: Self::Value, - order: AtomicOrdering, - size: Size, - ) -> Self::Value; - fn load_operand(&mut self, place: PlaceRef<'tcx, Self::Value>) - -> OperandRef<'tcx, Self::Value>; - - /// Called for Rvalue::Repeat when the elem is neither a ZST nor optimizable using memset. - fn write_operand_repeatedly( - self, - elem: OperandRef<'tcx, Self::Value>, - count: u64, - dest: PlaceRef<'tcx, Self::Value>, - ) -> Self; - - fn range_metadata(&mut self, load: Self::Value, range: WrappingRange); - fn nonnull_metadata(&mut self, load: Self::Value); - - fn store(&mut self, val: Self::Value, ptr: Self::Value, align: Align) -> Self::Value; - fn store_with_flags( - &mut self, - val: Self::Value, - ptr: Self::Value, - align: Align, - flags: MemFlags, - ) -> Self::Value; - fn atomic_store( - &mut self, - val: Self::Value, - ptr: Self::Value, - order: AtomicOrdering, - size: Size, - ); - - fn gep(&mut self, ty: Self::Type, ptr: Self::Value, indices: &[Self::Value]) -> Self::Value; - fn inbounds_gep( - &mut self, - ty: Self::Type, - ptr: Self::Value, - indices: &[Self::Value], - ) -> Self::Value; - fn struct_gep(&mut self, ty: Self::Type, ptr: Self::Value, idx: u64) -> Self::Value; - - fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option; - fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option; - fn fptoui(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn fptosi(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn uitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn sitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn fptrunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn fpext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn ptrtoint(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn inttoptr(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn bitcast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn intcast(&mut self, val: Self::Value, dest_ty: Self::Type, is_signed: bool) -> Self::Value; - fn pointercast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - - fn cast_float_to_int( - &mut self, - signed: bool, - x: Self::Value, - dest_ty: Self::Type, - ) -> Self::Value { - let in_ty = self.cx().val_ty(x); - let (float_ty, int_ty) = if self.cx().type_kind(dest_ty) == TypeKind::Vector - && self.cx().type_kind(in_ty) == TypeKind::Vector - { - (self.cx().element_type(in_ty), self.cx().element_type(dest_ty)) - } else { - (in_ty, dest_ty) - }; - assert!(matches!(self.cx().type_kind(float_ty), TypeKind::Float | TypeKind::Double)); - assert_eq!(self.cx().type_kind(int_ty), TypeKind::Integer); - - if let Some(false) = self.cx().sess().opts.unstable_opts.saturating_float_casts { - return if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) }; - } - - let try_sat_result = - if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) }; - if let Some(try_sat_result) = try_sat_result { - return try_sat_result; - } - - let int_width = self.cx().int_width(int_ty); - let float_width = self.cx().float_width(float_ty); - // LLVM's fpto[su]i returns undef when the input x is infinite, NaN, or does not fit into the - // destination integer type after rounding towards zero. This `undef` value can cause UB in - // safe code (see issue #10184), so we implement a saturating conversion on top of it: - // Semantically, the mathematical value of the input is rounded towards zero to the next - // mathematical integer, and then the result is clamped into the range of the destination - // integer type. Positive and negative infinity are mapped to the maximum and minimum value of - // the destination integer type. NaN is mapped to 0. - // - // Define f_min and f_max as the largest and smallest (finite) floats that are exactly equal to - // a value representable in int_ty. - // They are exactly equal to int_ty::{MIN,MAX} if float_ty has enough significand bits. - // Otherwise, int_ty::MAX must be rounded towards zero, as it is one less than a power of two. - // int_ty::MIN, however, is either zero or a negative power of two and is thus exactly - // representable. Note that this only works if float_ty's exponent range is sufficiently large. - // f16 or 256 bit integers would break this property. Right now the smallest float type is f32 - // with exponents ranging up to 127, which is barely enough for i128::MIN = -2^127. - // On the other hand, f_max works even if int_ty::MAX is greater than float_ty::MAX. Because - // we're rounding towards zero, we just get float_ty::MAX (which is always an integer). - // This already happens today with u128::MAX = 2^128 - 1 > f32::MAX. - let int_max = |signed: bool, int_width: u64| -> u128 { - let shift_amount = 128 - int_width; - if signed { i128::MAX as u128 >> shift_amount } else { u128::MAX >> shift_amount } - }; - let int_min = |signed: bool, int_width: u64| -> i128 { - if signed { i128::MIN >> (128 - int_width) } else { 0 } - }; - - let compute_clamp_bounds_single = |signed: bool, int_width: u64| -> (u128, u128) { - let rounded_min = - ieee::Single::from_i128_r(int_min(signed, int_width), Round::TowardZero); - assert_eq!(rounded_min.status, Status::OK); - let rounded_max = - ieee::Single::from_u128_r(int_max(signed, int_width), Round::TowardZero); - assert!(rounded_max.value.is_finite()); - (rounded_min.value.to_bits(), rounded_max.value.to_bits()) - }; - let compute_clamp_bounds_double = |signed: bool, int_width: u64| -> (u128, u128) { - let rounded_min = - ieee::Double::from_i128_r(int_min(signed, int_width), Round::TowardZero); - assert_eq!(rounded_min.status, Status::OK); - let rounded_max = - ieee::Double::from_u128_r(int_max(signed, int_width), Round::TowardZero); - assert!(rounded_max.value.is_finite()); - (rounded_min.value.to_bits(), rounded_max.value.to_bits()) - }; - // To implement saturation, we perform the following steps: - // - // 1. Cast x to an integer with fpto[su]i. This may result in undef. - // 2. Compare x to f_min and f_max, and use the comparison results to select: - // a) int_ty::MIN if x < f_min or x is NaN - // b) int_ty::MAX if x > f_max - // c) the result of fpto[su]i otherwise - // 3. If x is NaN, return 0.0, otherwise return the result of step 2. - // - // This avoids resulting undef because values in range [f_min, f_max] by definition fit into the - // destination type. It creates an undef temporary, but *producing* undef is not UB. Our use of - // undef does not introduce any non-determinism either. - // More importantly, the above procedure correctly implements saturating conversion. - // Proof (sketch): - // If x is NaN, 0 is returned by definition. - // Otherwise, x is finite or infinite and thus can be compared with f_min and f_max. - // This yields three cases to consider: - // (1) if x in [f_min, f_max], the result of fpto[su]i is returned, which agrees with - // saturating conversion for inputs in that range. - // (2) if x > f_max, then x is larger than int_ty::MAX. This holds even if f_max is rounded - // (i.e., if f_max < int_ty::MAX) because in those cases, nextUp(f_max) is already larger - // than int_ty::MAX. Because x is larger than int_ty::MAX, the return value of int_ty::MAX - // is correct. - // (3) if x < f_min, then x is smaller than int_ty::MIN. As shown earlier, f_min exactly equals - // int_ty::MIN and therefore the return value of int_ty::MIN is correct. - // QED. - - let float_bits_to_llval = |bx: &mut Self, bits| { - let bits_llval = match float_width { - 32 => bx.cx().const_u32(bits as u32), - 64 => bx.cx().const_u64(bits as u64), - n => bug!("unsupported float width {}", n), - }; - bx.bitcast(bits_llval, float_ty) - }; - let (f_min, f_max) = match float_width { - 32 => compute_clamp_bounds_single(signed, int_width), - 64 => compute_clamp_bounds_double(signed, int_width), - n => bug!("unsupported float width {}", n), - }; - let f_min = float_bits_to_llval(self, f_min); - let f_max = float_bits_to_llval(self, f_max); - let int_max = self.cx().const_uint_big(int_ty, int_max(signed, int_width)); - let int_min = self.cx().const_uint_big(int_ty, int_min(signed, int_width) as u128); - let zero = self.cx().const_uint(int_ty, 0); - - // If we're working with vectors, constants must be "splatted": the constant is duplicated - // into each lane of the vector. The algorithm stays the same, we are just using the - // same constant across all lanes. - let maybe_splat = |bx: &mut Self, val| { - if bx.cx().type_kind(dest_ty) == TypeKind::Vector { - bx.vector_splat(bx.vector_length(dest_ty), val) - } else { - val - } - }; - let f_min = maybe_splat(self, f_min); - let f_max = maybe_splat(self, f_max); - let int_max = maybe_splat(self, int_max); - let int_min = maybe_splat(self, int_min); - let zero = maybe_splat(self, zero); - - // Step 1 ... - let fptosui_result = if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) }; - let less_or_nan = self.fcmp(RealPredicate::RealULT, x, f_min); - let greater = self.fcmp(RealPredicate::RealOGT, x, f_max); - - // Step 2: We use two comparisons and two selects, with %s1 being the - // result: - // %less_or_nan = fcmp ult %x, %f_min - // %greater = fcmp olt %x, %f_max - // %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result - // %s1 = select %greater, int_ty::MAX, %s0 - // Note that %less_or_nan uses an *unordered* comparison. This - // comparison is true if the operands are not comparable (i.e., if x is - // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if - // x is NaN. - // - // Performance note: Unordered comparison can be lowered to a "flipped" - // comparison and a negation, and the negation can be merged into the - // select. Therefore, it not necessarily any more expensive than an - // ordered ("normal") comparison. Whether these optimizations will be - // performed is ultimately up to the backend, but at least x86 does - // perform them. - let s0 = self.select(less_or_nan, int_min, fptosui_result); - let s1 = self.select(greater, int_max, s0); - - // Step 3: NaN replacement. - // For unsigned types, the above step already yielded int_ty::MIN == 0 if x is NaN. - // Therefore we only need to execute this step for signed integer types. - if signed { - // LLVM has no isNaN predicate, so we use (x == x) instead - let cmp = self.fcmp(RealPredicate::RealOEQ, x, x); - self.select(cmp, s1, zero) - } else { - s1 - } - } - - fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn fcmp(&mut self, op: RealPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - - fn memcpy( - &mut self, - dst: Self::Value, - dst_align: Align, - src: Self::Value, - src_align: Align, - size: Self::Value, - flags: MemFlags, - ); - fn memmove( - &mut self, - dst: Self::Value, - dst_align: Align, - src: Self::Value, - src_align: Align, - size: Self::Value, - flags: MemFlags, - ); - fn memset( - &mut self, - ptr: Self::Value, - fill_byte: Self::Value, - size: Self::Value, - align: Align, - flags: MemFlags, - ); - - fn select( - &mut self, - cond: Self::Value, - then_val: Self::Value, - else_val: Self::Value, - ) -> Self::Value; - - fn va_arg(&mut self, list: Self::Value, ty: Self::Type) -> Self::Value; - fn extract_element(&mut self, vec: Self::Value, idx: Self::Value) -> Self::Value; - fn vector_splat(&mut self, num_elts: usize, elt: Self::Value) -> Self::Value; - fn extract_value(&mut self, agg_val: Self::Value, idx: u64) -> Self::Value; - fn insert_value(&mut self, agg_val: Self::Value, elt: Self::Value, idx: u64) -> Self::Value; - - fn set_personality_fn(&mut self, personality: Self::Value); - - // These are used by everyone except msvc - fn cleanup_landing_pad(&mut self, ty: Self::Type, pers_fn: Self::Value) -> Self::Value; - fn resume(&mut self, exn: Self::Value); - - // These are used only by msvc - fn cleanup_pad(&mut self, parent: Option, args: &[Self::Value]) -> Self::Funclet; - fn cleanup_ret(&mut self, funclet: &Self::Funclet, unwind: Option); - fn catch_pad(&mut self, parent: Self::Value, args: &[Self::Value]) -> Self::Funclet; - fn catch_switch( - &mut self, - parent: Option, - unwind: Option, - handlers: &[Self::BasicBlock], - ) -> Self::Value; - - fn atomic_cmpxchg( - &mut self, - dst: Self::Value, - cmp: Self::Value, - src: Self::Value, - order: AtomicOrdering, - failure_order: AtomicOrdering, - weak: bool, - ) -> Self::Value; - fn atomic_rmw( - &mut self, - op: AtomicRmwBinOp, - dst: Self::Value, - src: Self::Value, - order: AtomicOrdering, - ) -> Self::Value; - fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope); - fn set_invariant_load(&mut self, load: Self::Value); - - /// Called for `StorageLive` - fn lifetime_start(&mut self, ptr: Self::Value, size: Size); - - /// Called for `StorageDead` - fn lifetime_end(&mut self, ptr: Self::Value, size: Size); - - fn instrprof_increment( - &mut self, - fn_name: Self::Value, - hash: Self::Value, - num_counters: Self::Value, - index: Self::Value, - ); - - fn call( - &mut self, - llty: Self::Type, - llfn: Self::Value, - args: &[Self::Value], - funclet: Option<&Self::Funclet>, - ) -> Self::Value; - fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - - fn do_not_inline(&mut self, llret: Self::Value); -} diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs deleted file mode 100644 index 8a91d4735ba..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/consts.rs +++ /dev/null @@ -1,41 +0,0 @@ -use super::BackendTypes; -use crate::mir::place::PlaceRef; -use rustc_middle::mir::interpret::{ConstAllocation, Scalar}; -use rustc_middle::ty::layout::TyAndLayout; -use rustc_target::abi::{self, Size}; - -pub trait ConstMethods<'tcx>: BackendTypes { - // Constant constructors - fn const_null(&self, t: Self::Type) -> Self::Value; - fn const_undef(&self, t: Self::Type) -> Self::Value; - fn const_int(&self, t: Self::Type, i: i64) -> Self::Value; - fn const_uint(&self, t: Self::Type, i: u64) -> Self::Value; - fn const_uint_big(&self, t: Self::Type, u: u128) -> Self::Value; - fn const_bool(&self, val: bool) -> Self::Value; - fn const_i16(&self, i: i16) -> Self::Value; - fn const_i32(&self, i: i32) -> Self::Value; - fn const_u32(&self, i: u32) -> Self::Value; - fn const_u64(&self, i: u64) -> Self::Value; - fn const_usize(&self, i: u64) -> Self::Value; - fn const_u8(&self, i: u8) -> Self::Value; - fn const_real(&self, t: Self::Type, val: f64) -> Self::Value; - - fn const_str(&self, s: &str) -> (Self::Value, Self::Value); - fn const_struct(&self, elts: &[Self::Value], packed: bool) -> Self::Value; - - fn const_to_opt_uint(&self, v: Self::Value) -> Option; - fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option; - - fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value; - - fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value; - fn zst_to_backend(&self, llty: Self::Type) -> Self::Value; - fn from_const_alloc( - &self, - layout: TyAndLayout<'tcx>, - alloc: ConstAllocation<'tcx>, - offset: Size, - ) -> PlaceRef<'tcx, Self::Value>; - - fn const_ptrcast(&self, val: Self::Value, ty: Self::Type) -> Self::Value; -} diff --git a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs deleted file mode 100644 index e77201cf0c8..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs +++ /dev/null @@ -1,57 +0,0 @@ -use super::BackendTypes; -use rustc_hir::def_id::DefId; -use rustc_middle::mir::coverage::*; -use rustc_middle::ty::Instance; - -pub trait CoverageInfoMethods<'tcx>: BackendTypes { - fn coverageinfo_finalize(&self); - - /// Codegen a small function that will never be called, with one counter - /// that will never be incremented, that gives LLVM coverage tools a - /// function definition it needs in order to resolve coverage map references - /// to unused functions. This is necessary so unused functions will appear - /// as uncovered (coverage execution count `0`) in LLVM coverage reports. - fn define_unused_fn(&self, def_id: DefId); - - /// For LLVM codegen, returns a function-specific `Value` for a global - /// string, to hold the function name passed to LLVM intrinsic - /// `instrprof.increment()`. The `Value` is only created once per instance. - /// Multiple invocations with the same instance return the same `Value`. - fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> Self::Value; -} - -pub trait CoverageInfoBuilderMethods<'tcx>: BackendTypes { - /// Returns true if the function source hash was added to the coverage map (even if it had - /// already been added, for this instance). Returns false *only* if `-C instrument-coverage` is - /// not enabled (a coverage map is not being generated). - fn set_function_source_hash( - &mut self, - instance: Instance<'tcx>, - function_source_hash: u64, - ) -> bool; - - /// Returns true if the counter was added to the coverage map; false if `-C instrument-coverage` - /// is not enabled (a coverage map is not being generated). - fn add_coverage_counter( - &mut self, - instance: Instance<'tcx>, - index: CounterValueReference, - region: CodeRegion, - ) -> bool; - - /// Returns true if the expression was added to the coverage map; false if - /// `-C instrument-coverage` is not enabled (a coverage map is not being generated). - fn add_coverage_counter_expression( - &mut self, - instance: Instance<'tcx>, - id: InjectedExpressionId, - lhs: ExpressionOperandId, - op: Op, - rhs: ExpressionOperandId, - region: Option, - ) -> bool; - - /// Returns true if the region was added to the coverage map; false if `-C instrument-coverage` - /// is not enabled (a coverage map is not being generated). - fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool; -} diff --git a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs deleted file mode 100644 index f310789d144..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs +++ /dev/null @@ -1,79 +0,0 @@ -use super::BackendTypes; -use crate::mir::debuginfo::{FunctionDebugContext, VariableKind}; -use rustc_middle::mir; -use rustc_middle::ty::{Instance, PolyExistentialTraitRef, Ty}; -use rustc_span::{SourceFile, Span, Symbol}; -use rustc_target::abi::call::FnAbi; -use rustc_target::abi::Size; - -pub trait DebugInfoMethods<'tcx>: BackendTypes { - fn create_vtable_debuginfo( - &self, - ty: Ty<'tcx>, - trait_ref: Option>, - vtable: Self::Value, - ); - - /// Creates the function-specific debug context. - /// - /// Returns the FunctionDebugContext for the function which holds state needed - /// for debug info creation, if it is enabled. - fn create_function_debug_context( - &self, - instance: Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - llfn: Self::Function, - mir: &mir::Body<'tcx>, - ) -> Option>; - - // FIXME(eddyb) find a common convention for all of the debuginfo-related - // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). - fn dbg_scope_fn( - &self, - instance: Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - maybe_definition_llfn: Option, - ) -> Self::DIScope; - - fn dbg_loc( - &self, - scope: Self::DIScope, - inlined_at: Option, - span: Span, - ) -> Self::DILocation; - - fn extend_scope_to_file( - &self, - scope_metadata: Self::DIScope, - file: &SourceFile, - ) -> Self::DIScope; - fn debuginfo_finalize(&self); - - // FIXME(eddyb) find a common convention for all of the debuginfo-related - // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). - fn create_dbg_var( - &self, - variable_name: Symbol, - variable_type: Ty<'tcx>, - scope_metadata: Self::DIScope, - variable_kind: VariableKind, - span: Span, - ) -> Self::DIVariable; -} - -pub trait DebugInfoBuilderMethods: BackendTypes { - // FIXME(eddyb) find a common convention for all of the debuginfo-related - // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). - fn dbg_var_addr( - &mut self, - dbg_var: Self::DIVariable, - dbg_loc: Self::DILocation, - variable_alloca: Self::Value, - direct_offset: Size, - // NB: each offset implies a deref (i.e. they're steps in a pointer chain). - indirect_offsets: &[Size], - ); - fn set_dbg_loc(&mut self, dbg_loc: Self::DILocation); - fn insert_reference_to_gdb_debug_scripts_section_global(&mut self); - fn set_var_name(&mut self, value: Self::Value, name: &str); -} diff --git a/compiler/rustc_codegen_ssa/src/traits/declare.rs b/compiler/rustc_codegen_ssa/src/traits/declare.rs deleted file mode 100644 index 655afcd17f0..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/declare.rs +++ /dev/null @@ -1,21 +0,0 @@ -use super::BackendTypes; -use rustc_hir::def_id::DefId; -use rustc_middle::mir::mono::{Linkage, Visibility}; -use rustc_middle::ty::Instance; - -pub trait PreDefineMethods<'tcx>: BackendTypes { - fn predefine_static( - &self, - def_id: DefId, - linkage: Linkage, - visibility: Visibility, - symbol_name: &str, - ); - fn predefine_fn( - &self, - instance: Instance<'tcx>, - linkage: Linkage, - visibility: Visibility, - symbol_name: &str, - ); -} diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs deleted file mode 100644 index 7755e67938c..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs +++ /dev/null @@ -1,39 +0,0 @@ -use super::BackendTypes; -use crate::mir::operand::OperandRef; -use rustc_middle::ty::{self, Ty}; -use rustc_span::Span; -use rustc_target::abi::call::FnAbi; - -pub trait IntrinsicCallMethods<'tcx>: BackendTypes { - /// Remember to add all intrinsics here, in `compiler/rustc_typeck/src/check/mod.rs`, - /// and in `library/core/src/intrinsics.rs`; if you need access to any LLVM intrinsics, - /// add them to `compiler/rustc_codegen_llvm/src/context.rs`. - fn codegen_intrinsic_call( - &mut self, - instance: ty::Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - args: &[OperandRef<'tcx, Self::Value>], - llresult: Self::Value, - span: Span, - ); - - fn abort(&mut self); - fn assume(&mut self, val: Self::Value); - fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value; - /// Trait method used to test whether a given pointer is associated with a type identifier. - fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value; - /// Trait method used to load a function while testing if it is associated with a type - /// identifier. - fn type_checked_load( - &mut self, - llvtable: Self::Value, - vtable_byte_offset: u64, - typeid: Self::Value, - ) -> Self::Value; - /// Trait method used to inject `va_start` on the "spoofed" `VaListImpl` in - /// Rust defined C-variadic functions. - fn va_start(&mut self, val: Self::Value) -> Self::Value; - /// Trait method used to inject `va_end` on the "spoofed" `VaListImpl` before - /// Rust defined C-variadic functions return. - fn va_end(&mut self, val: Self::Value) -> Self::Value; -} diff --git a/compiler/rustc_codegen_ssa/src/traits/misc.rs b/compiler/rustc_codegen_ssa/src/traits/misc.rs deleted file mode 100644 index 4266e42ec2b..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/misc.rs +++ /dev/null @@ -1,26 +0,0 @@ -use super::BackendTypes; -use rustc_data_structures::fx::FxHashMap; -use rustc_middle::mir::mono::CodegenUnit; -use rustc_middle::ty::{self, Instance, Ty}; -use rustc_session::Session; -use std::cell::RefCell; - -pub trait MiscMethods<'tcx>: BackendTypes { - fn vtables( - &self, - ) -> &RefCell, Option>), Self::Value>>; - fn check_overflow(&self) -> bool; - fn get_fn(&self, instance: Instance<'tcx>) -> Self::Function; - fn get_fn_addr(&self, instance: Instance<'tcx>) -> Self::Value; - fn eh_personality(&self) -> Self::Value; - fn sess(&self) -> &Session; - fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx>; - fn used_statics(&self) -> &RefCell>; - fn compiler_used_statics(&self) -> &RefCell>; - fn set_frame_pointer_type(&self, llfn: Self::Function); - fn apply_target_cpu_attr(&self, llfn: Self::Function); - fn create_used_variable(&self); - fn create_compiler_used_variable(&self); - /// Declares the extern "C" main function for the entry point. Returns None if the symbol already exists. - fn declare_c_main(&self, fn_type: Self::Type) -> Option; -} diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs deleted file mode 100644 index 782fdadbfb8..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/mod.rs +++ /dev/null @@ -1,102 +0,0 @@ -//! Interface of a Rust codegen backend -//! -//! This crate defines all the traits that have to be implemented by a codegen backend in order to -//! use the backend-agnostic codegen code in `rustc_codegen_ssa`. -//! -//! The interface is designed around two backend-specific data structures, the codegen context and -//! the builder. The codegen context is supposed to be read-only after its creation and during the -//! actual codegen, while the builder stores the information about the function during codegen and -//! is used to produce the instructions of the backend IR. -//! -//! Finally, a third `Backend` structure has to implement methods related to how codegen information -//! is passed to the backend, especially for asynchronous compilation. -//! -//! The traits contain associated types that are backend-specific, such as the backend's value or -//! basic blocks. - -mod abi; -mod asm; -mod backend; -mod builder; -mod consts; -mod coverageinfo; -mod debuginfo; -mod declare; -mod intrinsic; -mod misc; -mod statics; -mod type_; -mod write; - -pub use self::abi::AbiBuilderMethods; -pub use self::asm::{AsmBuilderMethods, AsmMethods, GlobalAsmOperandRef, InlineAsmOperandRef}; -pub use self::backend::{Backend, BackendTypes, CodegenBackend, ExtraBackendMethods}; -pub use self::builder::{BuilderMethods, OverflowOp}; -pub use self::consts::ConstMethods; -pub use self::coverageinfo::{CoverageInfoBuilderMethods, CoverageInfoMethods}; -pub use self::debuginfo::{DebugInfoBuilderMethods, DebugInfoMethods}; -pub use self::declare::PreDefineMethods; -pub use self::intrinsic::IntrinsicCallMethods; -pub use self::misc::MiscMethods; -pub use self::statics::{StaticBuilderMethods, StaticMethods}; -pub use self::type_::{ - ArgAbiMethods, BaseTypeMethods, DerivedTypeMethods, LayoutTypeMethods, TypeMembershipMethods, - TypeMethods, -}; -pub use self::write::{ModuleBufferMethods, ThinBufferMethods, WriteBackendMethods}; - -use rustc_middle::ty::layout::{HasParamEnv, HasTyCtxt}; -use rustc_target::spec::HasTargetSpec; - -use std::fmt; - -pub trait CodegenObject: Copy + PartialEq + fmt::Debug {} -impl CodegenObject for T {} - -pub trait CodegenMethods<'tcx>: - Backend<'tcx> - + TypeMethods<'tcx> - + MiscMethods<'tcx> - + ConstMethods<'tcx> - + StaticMethods - + CoverageInfoMethods<'tcx> - + DebugInfoMethods<'tcx> - + AsmMethods<'tcx> - + PreDefineMethods<'tcx> - + HasParamEnv<'tcx> - + HasTyCtxt<'tcx> - + HasTargetSpec -{ -} - -impl<'tcx, T> CodegenMethods<'tcx> for T where - Self: Backend<'tcx> - + TypeMethods<'tcx> - + MiscMethods<'tcx> - + ConstMethods<'tcx> - + StaticMethods - + CoverageInfoMethods<'tcx> - + DebugInfoMethods<'tcx> - + AsmMethods<'tcx> - + PreDefineMethods<'tcx> - + HasParamEnv<'tcx> - + HasTyCtxt<'tcx> - + HasTargetSpec -{ -} - -pub trait HasCodegen<'tcx>: - Backend<'tcx> + std::ops::Deref>::CodegenCx> -{ - type CodegenCx: CodegenMethods<'tcx> - + BackendTypes< - Value = Self::Value, - Function = Self::Function, - BasicBlock = Self::BasicBlock, - Type = Self::Type, - Funclet = Self::Funclet, - DIScope = Self::DIScope, - DILocation = Self::DILocation, - DIVariable = Self::DIVariable, - >; -} diff --git a/compiler/rustc_codegen_ssa/src/traits/statics.rs b/compiler/rustc_codegen_ssa/src/traits/statics.rs deleted file mode 100644 index 413d31bb942..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/statics.rs +++ /dev/null @@ -1,24 +0,0 @@ -use super::BackendTypes; -use rustc_hir::def_id::DefId; -use rustc_target::abi::Align; - -pub trait StaticMethods: BackendTypes { - fn static_addr_of(&self, cv: Self::Value, align: Align, kind: Option<&str>) -> Self::Value; - fn codegen_static(&self, def_id: DefId, is_mutable: bool); - - /// Mark the given global value as "used", to prevent the compiler and linker from potentially - /// removing a static variable that may otherwise appear unused. - fn add_used_global(&self, global: Self::Value); - - /// Same as add_used_global(), but only prevent the compiler from potentially removing an - /// otherwise unused symbol. The linker is still permitted to drop it. - /// - /// This corresponds to the documented semantics of the `#[used]` attribute, although - /// on some targets (non-ELF), we may use `add_used_global` for `#[used]` statics - /// instead. - fn add_compiler_used_global(&self, global: Self::Value); -} - -pub trait StaticBuilderMethods: BackendTypes { - fn get_static(&mut self, def_id: DefId) -> Self::Value; -} diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs deleted file mode 100644 index 8158e8dd011..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ /dev/null @@ -1,151 +0,0 @@ -use super::misc::MiscMethods; -use super::Backend; -use super::HasCodegen; -use crate::common::TypeKind; -use crate::mir::place::PlaceRef; -use rustc_middle::ty::layout::TyAndLayout; -use rustc_middle::ty::{self, Ty}; -use rustc_span::DUMMY_SP; -use rustc_target::abi::call::{ArgAbi, CastTarget, FnAbi, Reg}; -use rustc_target::abi::{AddressSpace, Integer}; - -// This depends on `Backend` and not `BackendTypes`, because consumers will probably want to use -// `LayoutOf` or `HasTyCtxt`. This way, they don't have to add a constraint on it themselves. -pub trait BaseTypeMethods<'tcx>: Backend<'tcx> { - fn type_i1(&self) -> Self::Type; - fn type_i8(&self) -> Self::Type; - fn type_i16(&self) -> Self::Type; - fn type_i32(&self) -> Self::Type; - fn type_i64(&self) -> Self::Type; - fn type_i128(&self) -> Self::Type; - fn type_isize(&self) -> Self::Type; - - fn type_f32(&self) -> Self::Type; - fn type_f64(&self) -> Self::Type; - - fn type_func(&self, args: &[Self::Type], ret: Self::Type) -> Self::Type; - fn type_struct(&self, els: &[Self::Type], packed: bool) -> Self::Type; - fn type_kind(&self, ty: Self::Type) -> TypeKind; - fn type_ptr_to(&self, ty: Self::Type) -> Self::Type; - fn type_ptr_to_ext(&self, ty: Self::Type, address_space: AddressSpace) -> Self::Type; - fn element_type(&self, ty: Self::Type) -> Self::Type; - - /// Returns the number of elements in `self` if it is a LLVM vector type. - fn vector_length(&self, ty: Self::Type) -> usize; - - fn float_width(&self, ty: Self::Type) -> usize; - - /// Retrieves the bit width of the integer type `self`. - fn int_width(&self, ty: Self::Type) -> u64; - - fn val_ty(&self, v: Self::Value) -> Self::Type; -} - -pub trait DerivedTypeMethods<'tcx>: BaseTypeMethods<'tcx> + MiscMethods<'tcx> { - fn type_i8p(&self) -> Self::Type { - self.type_i8p_ext(AddressSpace::DATA) - } - - fn type_i8p_ext(&self, address_space: AddressSpace) -> Self::Type { - self.type_ptr_to_ext(self.type_i8(), address_space) - } - - fn type_int(&self) -> Self::Type { - match &self.sess().target.c_int_width[..] { - "16" => self.type_i16(), - "32" => self.type_i32(), - "64" => self.type_i64(), - width => bug!("Unsupported c_int_width: {}", width), - } - } - - fn type_from_integer(&self, i: Integer) -> Self::Type { - use Integer::*; - match i { - I8 => self.type_i8(), - I16 => self.type_i16(), - I32 => self.type_i32(), - I64 => self.type_i64(), - I128 => self.type_i128(), - } - } - - fn type_needs_drop(&self, ty: Ty<'tcx>) -> bool { - ty.needs_drop(self.tcx(), ty::ParamEnv::reveal_all()) - } - - fn type_is_sized(&self, ty: Ty<'tcx>) -> bool { - ty.is_sized(self.tcx().at(DUMMY_SP), ty::ParamEnv::reveal_all()) - } - - fn type_is_freeze(&self, ty: Ty<'tcx>) -> bool { - ty.is_freeze(self.tcx().at(DUMMY_SP), ty::ParamEnv::reveal_all()) - } - - fn type_has_metadata(&self, ty: Ty<'tcx>) -> bool { - let param_env = ty::ParamEnv::reveal_all(); - if ty.is_sized(self.tcx().at(DUMMY_SP), param_env) { - return false; - } - - let tail = self.tcx().struct_tail_erasing_lifetimes(ty, param_env); - match tail.kind() { - ty::Foreign(..) => false, - ty::Str | ty::Slice(..) | ty::Dynamic(..) => true, - _ => bug!("unexpected unsized tail: {:?}", tail), - } - } -} - -impl<'tcx, T> DerivedTypeMethods<'tcx> for T where Self: BaseTypeMethods<'tcx> + MiscMethods<'tcx> {} - -pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> { - fn backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type; - fn cast_backend_type(&self, ty: &CastTarget) -> Self::Type; - fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; - fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; - fn reg_backend_type(&self, ty: &Reg) -> Self::Type; - fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type; - fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool; - fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool; - fn backend_field_index(&self, layout: TyAndLayout<'tcx>, index: usize) -> u64; - fn scalar_pair_element_backend_type( - &self, - layout: TyAndLayout<'tcx>, - index: usize, - immediate: bool, - ) -> Self::Type; -} - -// For backends that support CFI using type membership (i.e., testing whether a given pointer is -// associated with a type identifier). -pub trait TypeMembershipMethods<'tcx>: Backend<'tcx> { - fn set_type_metadata(&self, function: Self::Function, typeid: String); - fn typeid_metadata(&self, typeid: String) -> Self::Value; -} - -pub trait ArgAbiMethods<'tcx>: HasCodegen<'tcx> { - fn store_fn_arg( - &mut self, - arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, - idx: &mut usize, - dst: PlaceRef<'tcx, Self::Value>, - ); - fn store_arg( - &mut self, - arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, - val: Self::Value, - dst: PlaceRef<'tcx, Self::Value>, - ); - fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> Self::Type; -} - -pub trait TypeMethods<'tcx>: - DerivedTypeMethods<'tcx> + LayoutTypeMethods<'tcx> + TypeMembershipMethods<'tcx> -{ -} - -impl<'tcx, T> TypeMethods<'tcx> for T where - Self: DerivedTypeMethods<'tcx> + LayoutTypeMethods<'tcx> + TypeMembershipMethods<'tcx> -{ -} diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs deleted file mode 100644 index e54ec34f1ce..00000000000 --- a/compiler/rustc_codegen_ssa/src/traits/write.rs +++ /dev/null @@ -1,68 +0,0 @@ -use crate::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; -use crate::back::write::{CodegenContext, FatLTOInput, ModuleConfig}; -use crate::{CompiledModule, ModuleCodegen}; - -use rustc_errors::{FatalError, Handler}; -use rustc_middle::dep_graph::WorkProduct; - -pub trait WriteBackendMethods: 'static + Sized + Clone { - type Module: Send + Sync; - type TargetMachine; - type ModuleBuffer: ModuleBufferMethods; - type Context: ?Sized; - type ThinData: Send + Sync; - type ThinBuffer: ThinBufferMethods; - - /// Merge all modules into main_module and returning it - fn run_link( - cgcx: &CodegenContext, - diag_handler: &Handler, - modules: Vec>, - ) -> Result, FatalError>; - /// Performs fat LTO by merging all modules into a single one and returning it - /// for further optimization. - fn run_fat_lto( - cgcx: &CodegenContext, - modules: Vec>, - cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result, FatalError>; - /// Performs thin LTO by performing necessary global analysis and returning two - /// lists, one of the modules that need optimization and another for modules that - /// can simply be copied over from the incr. comp. cache. - fn run_thin_lto( - cgcx: &CodegenContext, - modules: Vec<(String, Self::ThinBuffer)>, - cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result<(Vec>, Vec), FatalError>; - fn print_pass_timings(&self); - unsafe fn optimize( - cgcx: &CodegenContext, - diag_handler: &Handler, - module: &ModuleCodegen, - config: &ModuleConfig, - ) -> Result<(), FatalError>; - fn optimize_fat( - cgcx: &CodegenContext, - llmod: &mut ModuleCodegen, - ) -> Result<(), FatalError>; - unsafe fn optimize_thin( - cgcx: &CodegenContext, - thin: ThinModule, - ) -> Result, FatalError>; - unsafe fn codegen( - cgcx: &CodegenContext, - diag_handler: &Handler, - module: ModuleCodegen, - config: &ModuleConfig, - ) -> Result; - fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer); - fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer); -} - -pub trait ThinBufferMethods: Send + Sync { - fn data(&self) -> &[u8]; -} - -pub trait ModuleBufferMethods: Send + Sync { - fn data(&self) -> &[u8]; -} From b991d8b86bb3bf92301023bb03981034e286ac63 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 6 Dec 2023 22:23:15 +0000 Subject: [PATCH 11/70] disable unused --- compiler/rustc_llvm/build.rs | 40 +- .../llvm-wrapper/ArchiveWrapper.cpp | 444 +- compiler/rustc_llvm/llvm-wrapper/Linker.cpp | 96 +- .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 3526 +++++++-------- .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 3944 ++++++++--------- compiler/rustc_llvm/src/lib.rs | 308 +- 6 files changed, 4176 insertions(+), 4182 deletions(-) diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index 62ef5804dce..cde9f912418 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -46,12 +46,12 @@ fn detect_llvm_link() -> (&'static str, &'static str) { // perfect -- we might actually want to see something from Cargo's added library paths -- but // for now it works. fn restore_library_path() { - let key = tracked_env_var_os("REAL_LIBRARY_PATH_VAR").expect("REAL_LIBRARY_PATH_VAR"); - if let Some(env) = tracked_env_var_os("REAL_LIBRARY_PATH") { - env::set_var(&key, &env); - } else { - env::remove_var(&key); - } + // let key = tracked_env_var_os("REAL_LIBRARY_PATH_VAR").expect("REAL_LIBRARY_PATH_VAR"); + // if let Some(env) = tracked_env_var_os("REAL_LIBRARY_PATH") { + // env::set_var(&key, &env); + // } else { + // env::remove_var(&key); + // } } /// Reads an environment variable and adds it to dependencies. @@ -63,12 +63,8 @@ fn tracked_env_var_os + Display>(key: K) -> Option { } fn rerun_if_changed_anything_in_dir(dir: &Path) { - let mut stack = dir - .read_dir() - .unwrap() - .map(|e| e.unwrap()) - .filter(|e| &*e.file_name() != ".git") - .collect::>(); + let mut stack = + dir.read_dir().unwrap().map(|e| e.unwrap()).filter(|e| &*e.file_name() != ".git").collect::>(); while let Some(entry) = stack.pop() { let path = entry.path(); if entry.file_type().unwrap().is_dir() { @@ -114,13 +110,8 @@ fn main() { let llvm_config = tracked_env_var_os("LLVM_CONFIG").map(|x| Some(PathBuf::from(x))).unwrap_or_else(|| { if let Some(dir) = tracked_env_var_os("CARGO_TARGET_DIR").map(PathBuf::from) { - let to_test = dir - .parent() - .unwrap() - .parent() - .unwrap() - .join(&target) - .join("llvm/bin/llvm-config"); + let to_test = + dir.parent().unwrap().parent().unwrap().join(&target).join("llvm/bin/llvm-config"); if Command::new(&to_test).output().is_ok() { return Some(to_test); } @@ -319,11 +310,12 @@ fn main() { let llvm_use_libcxx = tracked_env_var_os("LLVM_USE_LIBCXX"); let stdcppname = if target.contains("openbsd") { - if target.contains("sparc64") { "estdc++" } else { "c++" } - } else if target.contains("darwin") - || target.contains("freebsd") - || target.contains("windows-gnullvm") - { + if target.contains("sparc64") { + "estdc++" + } else { + "c++" + } + } else if target.contains("darwin") || target.contains("freebsd") || target.contains("windows-gnullvm") { "c++" } else if target.contains("netbsd") && llvm_static_stdcpp.is_some() { // NetBSD uses a separate library when relocation is required diff --git a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp index 97541e615da..5b6ba85bee9 100644 --- a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp @@ -1,222 +1,222 @@ -#include "LLVMWrapper.h" - -#include "llvm/Object/Archive.h" -#include "llvm/Object/ArchiveWriter.h" -#include "llvm/Support/Path.h" - -using namespace llvm; -using namespace llvm::object; - -struct RustArchiveMember { - const char *Filename; - const char *Name; - Archive::Child Child; - - RustArchiveMember() - : Filename(nullptr), Name(nullptr), - Child(nullptr, nullptr, nullptr) - { - } - ~RustArchiveMember() {} -}; - -struct RustArchiveIterator { - bool First; - Archive::child_iterator Cur; - Archive::child_iterator End; - std::unique_ptr Err; - - RustArchiveIterator(Archive::child_iterator Cur, Archive::child_iterator End, - std::unique_ptr Err) - : First(true), - Cur(Cur), - End(End), - Err(std::move(Err)) {} -}; - -enum class LLVMRustArchiveKind { - GNU, - BSD, - DARWIN, - COFF, -}; - -static Archive::Kind fromRust(LLVMRustArchiveKind Kind) { - switch (Kind) { - case LLVMRustArchiveKind::GNU: - return Archive::K_GNU; - case LLVMRustArchiveKind::BSD: - return Archive::K_BSD; - case LLVMRustArchiveKind::DARWIN: - return Archive::K_DARWIN; - case LLVMRustArchiveKind::COFF: - return Archive::K_COFF; - default: - report_fatal_error("Bad ArchiveKind."); - } -} - -typedef OwningBinary *LLVMRustArchiveRef; -typedef RustArchiveMember *LLVMRustArchiveMemberRef; -typedef Archive::Child *LLVMRustArchiveChildRef; -typedef Archive::Child const *LLVMRustArchiveChildConstRef; -typedef RustArchiveIterator *LLVMRustArchiveIteratorRef; - -extern "C" LLVMRustArchiveRef LLVMRustOpenArchive(char *Path) { - ErrorOr> BufOr = - MemoryBuffer::getFile(Path, -1, false); - if (!BufOr) { - LLVMRustSetLastError(BufOr.getError().message().c_str()); - return nullptr; - } - - Expected> ArchiveOr = - Archive::create(BufOr.get()->getMemBufferRef()); - - if (!ArchiveOr) { - LLVMRustSetLastError(toString(ArchiveOr.takeError()).c_str()); - return nullptr; - } - - OwningBinary *Ret = new OwningBinary( - std::move(ArchiveOr.get()), std::move(BufOr.get())); - - return Ret; -} - -extern "C" void LLVMRustDestroyArchive(LLVMRustArchiveRef RustArchive) { - delete RustArchive; -} - -extern "C" LLVMRustArchiveIteratorRef -LLVMRustArchiveIteratorNew(LLVMRustArchiveRef RustArchive) { - Archive *Archive = RustArchive->getBinary(); - std::unique_ptr Err = std::make_unique(Error::success()); - auto Cur = Archive->child_begin(*Err); - if (*Err) { - LLVMRustSetLastError(toString(std::move(*Err)).c_str()); - return nullptr; - } - auto End = Archive->child_end(); - return new RustArchiveIterator(Cur, End, std::move(Err)); -} - -extern "C" LLVMRustArchiveChildConstRef -LLVMRustArchiveIteratorNext(LLVMRustArchiveIteratorRef RAI) { - if (RAI->Cur == RAI->End) - return nullptr; - - // Advancing the iterator validates the next child, and this can - // uncover an error. LLVM requires that we check all Errors, - // so we only advance the iterator if we actually need to fetch - // the next child. - // This means we must not advance the iterator in the *first* call, - // but instead advance it *before* fetching the child in all later calls. - if (!RAI->First) { - ++RAI->Cur; - if (*RAI->Err) { - LLVMRustSetLastError(toString(std::move(*RAI->Err)).c_str()); - return nullptr; - } - } else { - RAI->First = false; - } - - if (RAI->Cur == RAI->End) - return nullptr; - - const Archive::Child &Child = *RAI->Cur.operator->(); - Archive::Child *Ret = new Archive::Child(Child); - - return Ret; -} - -extern "C" void LLVMRustArchiveChildFree(LLVMRustArchiveChildRef Child) { - delete Child; -} - -extern "C" void LLVMRustArchiveIteratorFree(LLVMRustArchiveIteratorRef RAI) { - delete RAI; -} - -extern "C" const char * -LLVMRustArchiveChildName(LLVMRustArchiveChildConstRef Child, size_t *Size) { - Expected NameOrErr = Child->getName(); - if (!NameOrErr) { - // rustc_codegen_llvm currently doesn't use this error string, but it might be - // useful in the future, and in the mean time this tells LLVM that the - // error was not ignored and that it shouldn't abort the process. - LLVMRustSetLastError(toString(NameOrErr.takeError()).c_str()); - return nullptr; - } - StringRef Name = NameOrErr.get(); - *Size = Name.size(); - return Name.data(); -} - -extern "C" const char *LLVMRustArchiveChildData(LLVMRustArchiveChildRef Child, - size_t *Size) { - StringRef Buf; - Expected BufOrErr = Child->getBuffer(); - if (!BufOrErr) { - LLVMRustSetLastError(toString(BufOrErr.takeError()).c_str()); - return nullptr; - } - Buf = BufOrErr.get(); - *Size = Buf.size(); - return Buf.data(); -} - -extern "C" LLVMRustArchiveMemberRef -LLVMRustArchiveMemberNew(char *Filename, char *Name, - LLVMRustArchiveChildRef Child) { - RustArchiveMember *Member = new RustArchiveMember; - Member->Filename = Filename; - Member->Name = Name; - if (Child) - Member->Child = *Child; - return Member; -} - -extern "C" void LLVMRustArchiveMemberFree(LLVMRustArchiveMemberRef Member) { - delete Member; -} - -extern "C" LLVMRustResult -LLVMRustWriteArchive(char *Dst, size_t NumMembers, - const LLVMRustArchiveMemberRef *NewMembers, - bool WriteSymbtab, LLVMRustArchiveKind RustKind) { - - std::vector Members; - auto Kind = fromRust(RustKind); - - for (size_t I = 0; I < NumMembers; I++) { - auto Member = NewMembers[I]; - assert(Member->Name); - if (Member->Filename) { - Expected MOrErr = - NewArchiveMember::getFile(Member->Filename, true); - if (!MOrErr) { - LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); - return LLVMRustResult::Failure; - } - MOrErr->MemberName = sys::path::filename(MOrErr->MemberName); - Members.push_back(std::move(*MOrErr)); - } else { - Expected MOrErr = - NewArchiveMember::getOldMember(Member->Child, true); - if (!MOrErr) { - LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); - return LLVMRustResult::Failure; - } - Members.push_back(std::move(*MOrErr)); - } - } - - auto Result = writeArchive(Dst, Members, WriteSymbtab, Kind, true, false); - if (!Result) - return LLVMRustResult::Success; - LLVMRustSetLastError(toString(std::move(Result)).c_str()); - - return LLVMRustResult::Failure; -} +// #include "LLVMWrapper.h" + +// #include "llvm/Object/Archive.h" +// #include "llvm/Object/ArchiveWriter.h" +// #include "llvm/Support/Path.h" + +// using namespace llvm; +// using namespace llvm::object; + +// struct RustArchiveMember { +// const char *Filename; +// const char *Name; +// Archive::Child Child; + +// RustArchiveMember() +// : Filename(nullptr), Name(nullptr), +// Child(nullptr, nullptr, nullptr) +// { +// } +// ~RustArchiveMember() {} +// }; + +// struct RustArchiveIterator { +// bool First; +// Archive::child_iterator Cur; +// Archive::child_iterator End; +// std::unique_ptr Err; + +// RustArchiveIterator(Archive::child_iterator Cur, Archive::child_iterator End, +// std::unique_ptr Err) +// : First(true), +// Cur(Cur), +// End(End), +// Err(std::move(Err)) {} +// }; + +// enum class LLVMRustArchiveKind { +// GNU, +// BSD, +// DARWIN, +// COFF, +// }; + +// static Archive::Kind fromRust(LLVMRustArchiveKind Kind) { +// switch (Kind) { +// case LLVMRustArchiveKind::GNU: +// return Archive::K_GNU; +// case LLVMRustArchiveKind::BSD: +// return Archive::K_BSD; +// case LLVMRustArchiveKind::DARWIN: +// return Archive::K_DARWIN; +// case LLVMRustArchiveKind::COFF: +// return Archive::K_COFF; +// default: +// report_fatal_error("Bad ArchiveKind."); +// } +// } + +// typedef OwningBinary *LLVMRustArchiveRef; +// typedef RustArchiveMember *LLVMRustArchiveMemberRef; +// typedef Archive::Child *LLVMRustArchiveChildRef; +// typedef Archive::Child const *LLVMRustArchiveChildConstRef; +// typedef RustArchiveIterator *LLVMRustArchiveIteratorRef; + +// extern "C" LLVMRustArchiveRef LLVMRustOpenArchive(char *Path) { +// ErrorOr> BufOr = +// MemoryBuffer::getFile(Path, -1, false); +// if (!BufOr) { +// LLVMRustSetLastError(BufOr.getError().message().c_str()); +// return nullptr; +// } + +// Expected> ArchiveOr = +// Archive::create(BufOr.get()->getMemBufferRef()); + +// if (!ArchiveOr) { +// LLVMRustSetLastError(toString(ArchiveOr.takeError()).c_str()); +// return nullptr; +// } + +// OwningBinary *Ret = new OwningBinary( +// std::move(ArchiveOr.get()), std::move(BufOr.get())); + +// return Ret; +// } + +// extern "C" void LLVMRustDestroyArchive(LLVMRustArchiveRef RustArchive) { +// delete RustArchive; +// } + +// extern "C" LLVMRustArchiveIteratorRef +// LLVMRustArchiveIteratorNew(LLVMRustArchiveRef RustArchive) { +// Archive *Archive = RustArchive->getBinary(); +// std::unique_ptr Err = std::make_unique(Error::success()); +// auto Cur = Archive->child_begin(*Err); +// if (*Err) { +// LLVMRustSetLastError(toString(std::move(*Err)).c_str()); +// return nullptr; +// } +// auto End = Archive->child_end(); +// return new RustArchiveIterator(Cur, End, std::move(Err)); +// } + +// extern "C" LLVMRustArchiveChildConstRef +// LLVMRustArchiveIteratorNext(LLVMRustArchiveIteratorRef RAI) { +// if (RAI->Cur == RAI->End) +// return nullptr; + +// // Advancing the iterator validates the next child, and this can +// // uncover an error. LLVM requires that we check all Errors, +// // so we only advance the iterator if we actually need to fetch +// // the next child. +// // This means we must not advance the iterator in the *first* call, +// // but instead advance it *before* fetching the child in all later calls. +// if (!RAI->First) { +// ++RAI->Cur; +// if (*RAI->Err) { +// LLVMRustSetLastError(toString(std::move(*RAI->Err)).c_str()); +// return nullptr; +// } +// } else { +// RAI->First = false; +// } + +// if (RAI->Cur == RAI->End) +// return nullptr; + +// const Archive::Child &Child = *RAI->Cur.operator->(); +// Archive::Child *Ret = new Archive::Child(Child); + +// return Ret; +// } + +// extern "C" void LLVMRustArchiveChildFree(LLVMRustArchiveChildRef Child) { +// delete Child; +// } + +// extern "C" void LLVMRustArchiveIteratorFree(LLVMRustArchiveIteratorRef RAI) { +// delete RAI; +// } + +// extern "C" const char * +// LLVMRustArchiveChildName(LLVMRustArchiveChildConstRef Child, size_t *Size) { +// Expected NameOrErr = Child->getName(); +// if (!NameOrErr) { +// // rustc_codegen_llvm currently doesn't use this error string, but it might be +// // useful in the future, and in the mean time this tells LLVM that the +// // error was not ignored and that it shouldn't abort the process. +// LLVMRustSetLastError(toString(NameOrErr.takeError()).c_str()); +// return nullptr; +// } +// StringRef Name = NameOrErr.get(); +// *Size = Name.size(); +// return Name.data(); +// } + +// extern "C" const char *LLVMRustArchiveChildData(LLVMRustArchiveChildRef Child, +// size_t *Size) { +// StringRef Buf; +// Expected BufOrErr = Child->getBuffer(); +// if (!BufOrErr) { +// LLVMRustSetLastError(toString(BufOrErr.takeError()).c_str()); +// return nullptr; +// } +// Buf = BufOrErr.get(); +// *Size = Buf.size(); +// return Buf.data(); +// } + +// extern "C" LLVMRustArchiveMemberRef +// LLVMRustArchiveMemberNew(char *Filename, char *Name, +// LLVMRustArchiveChildRef Child) { +// RustArchiveMember *Member = new RustArchiveMember; +// Member->Filename = Filename; +// Member->Name = Name; +// if (Child) +// Member->Child = *Child; +// return Member; +// } + +// extern "C" void LLVMRustArchiveMemberFree(LLVMRustArchiveMemberRef Member) { +// delete Member; +// } + +// extern "C" LLVMRustResult +// LLVMRustWriteArchive(char *Dst, size_t NumMembers, +// const LLVMRustArchiveMemberRef *NewMembers, +// bool WriteSymbtab, LLVMRustArchiveKind RustKind) { + +// std::vector Members; +// auto Kind = fromRust(RustKind); + +// for (size_t I = 0; I < NumMembers; I++) { +// auto Member = NewMembers[I]; +// assert(Member->Name); +// if (Member->Filename) { +// Expected MOrErr = +// NewArchiveMember::getFile(Member->Filename, true); +// if (!MOrErr) { +// LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); +// return LLVMRustResult::Failure; +// } +// MOrErr->MemberName = sys::path::filename(MOrErr->MemberName); +// Members.push_back(std::move(*MOrErr)); +// } else { +// Expected MOrErr = +// NewArchiveMember::getOldMember(Member->Child, true); +// if (!MOrErr) { +// LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); +// return LLVMRustResult::Failure; +// } +// Members.push_back(std::move(*MOrErr)); +// } +// } + +// auto Result = writeArchive(Dst, Members, WriteSymbtab, Kind, true, false); +// if (!Result) +// return LLVMRustResult::Success; +// LLVMRustSetLastError(toString(std::move(Result)).c_str()); + +// return LLVMRustResult::Failure; +// } diff --git a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp index 8766e96f086..aacb97f0696 100644 --- a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp @@ -1,48 +1,48 @@ -#include "llvm/Linker/Linker.h" - -#include "LLVMWrapper.h" - -using namespace llvm; - -struct RustLinker { - Linker L; - LLVMContext &Ctx; - - RustLinker(Module &M) : - L(M), - Ctx(M.getContext()) - {} -}; - -extern "C" RustLinker* -LLVMRustLinkerNew(LLVMModuleRef DstRef) { - Module *Dst = unwrap(DstRef); - - return new RustLinker(*Dst); -} - -extern "C" void -LLVMRustLinkerFree(RustLinker *L) { - delete L; -} - -extern "C" bool -LLVMRustLinkerAdd(RustLinker *L, char *BC, size_t Len) { - std::unique_ptr Buf = - MemoryBuffer::getMemBufferCopy(StringRef(BC, Len)); - - Expected> SrcOrError = - llvm::getLazyBitcodeModule(Buf->getMemBufferRef(), L->Ctx); - if (!SrcOrError) { - LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); - return false; - } - - auto Src = std::move(*SrcOrError); - - if (L->L.linkInModule(std::move(Src))) { - LLVMRustSetLastError(""); - return false; - } - return true; -} +// #include "llvm/Linker/Linker.h" + +// #include "LLVMWrapper.h" + +// using namespace llvm; + +// struct RustLinker { +// Linker L; +// LLVMContext &Ctx; + +// RustLinker(Module &M) : +// L(M), +// Ctx(M.getContext()) +// {} +// }; + +// extern "C" RustLinker* +// LLVMRustLinkerNew(LLVMModuleRef DstRef) { +// Module *Dst = unwrap(DstRef); + +// return new RustLinker(*Dst); +// } + +// extern "C" void +// LLVMRustLinkerFree(RustLinker *L) { +// delete L; +// } + +// extern "C" bool +// LLVMRustLinkerAdd(RustLinker *L, char *BC, size_t Len) { +// std::unique_ptr Buf = +// MemoryBuffer::getMemBufferCopy(StringRef(BC, Len)); + +// Expected> SrcOrError = +// llvm::getLazyBitcodeModule(Buf->getMemBufferRef(), L->Ctx); +// if (!SrcOrError) { +// LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); +// return false; +// } + +// auto Src = std::move(*SrcOrError); + +// if (L->L.linkInModule(std::move(Src))) { +// LLVMRustSetLastError(""); +// return false; +// } +// return true; +// } diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 0a6bd49992d..c7634175414 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -1,1763 +1,1763 @@ -#include - -#include -#include - -#include "LLVMWrapper.h" - -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/InitializePasses.h" -#include "llvm/IR/AutoUpgrade.h" -#include "llvm/IR/AssemblyAnnotationWriter.h" -#include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Verifier.h" -#include "llvm/Object/ObjectFile.h" -#include "llvm/Object/IRObjectFile.h" -#include "llvm/Passes/PassBuilder.h" -#include "llvm/Passes/PassPlugin.h" -#include "llvm/Passes/StandardInstrumentations.h" -#include "llvm/Support/CBindingWrapping.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#if LLVM_VERSION_LT(14, 0) -#include "llvm/Support/TargetRegistry.h" -#else -#include "llvm/MC/TargetRegistry.h" -#endif -#include "llvm/Target/TargetMachine.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" -#include "llvm/Transforms/IPO/AlwaysInliner.h" -#include "llvm/Transforms/IPO/FunctionImport.h" -#include "llvm/Transforms/Utils/AddDiscriminators.h" -#include "llvm/Transforms/Utils/FunctionImportUtils.h" -#include "llvm/LTO/LTO.h" -#include "llvm/Bitcode/BitcodeWriterPass.h" -#include "llvm-c/Transforms/PassManagerBuilder.h" - -#include "llvm/Transforms/Instrumentation.h" -#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" -#include "llvm/Support/TimeProfiler.h" -#include "llvm/Transforms/Instrumentation/GCOVProfiler.h" -#include "llvm/Transforms/Instrumentation/InstrProfiling.h" -#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" -#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" -#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" -#include "llvm/Transforms/Utils/CanonicalizeAliases.h" -#include "llvm/Transforms/Utils/NameAnonGlobals.h" -#include "llvm/Transforms/Utils.h" - -using namespace llvm; - -typedef struct LLVMOpaquePass *LLVMPassRef; -typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; - -DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) -DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) - -extern "C" void LLVMInitializePasses() { - PassRegistry &Registry = *PassRegistry::getPassRegistry(); - initializeCore(Registry); - initializeCodeGen(Registry); - initializeScalarOpts(Registry); - initializeVectorization(Registry); - initializeIPO(Registry); - initializeAnalysis(Registry); - initializeTransformUtils(Registry); - initializeInstCombine(Registry); - initializeInstrumentation(Registry); - initializeTarget(Registry); -} - -extern "C" void LLVMTimeTraceProfilerInitialize() { - timeTraceProfilerInitialize( - /* TimeTraceGranularity */ 0, - /* ProcName */ "rustc"); -} - -extern "C" void LLVMTimeTraceProfilerFinishThread() { - timeTraceProfilerFinishThread(); -} - -extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { - StringRef FN(FileName); - std::error_code EC; - raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); - - timeTraceProfilerWrite(OS); - timeTraceProfilerCleanup(); -} - -extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { -#if LLVM_VERSION_LT(15, 0) - StringRef SR(PassName); - PassRegistry *PR = PassRegistry::getPassRegistry(); - - const PassInfo *PI = PR->getPassInfo(SR); - if (PI) { - return wrap(PI->createPass()); - } - return nullptr; -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) { -#if LLVM_VERSION_LT(15, 0) - const bool CompileKernel = false; - const bool UseAfterScope = true; - - return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope)); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) { -#if LLVM_VERSION_LT(15, 0) - const bool CompileKernel = false; - - return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover)); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) { -#if LLVM_VERSION_LT(15, 0) - const bool CompileKernel = false; - - return wrap(createMemorySanitizerLegacyPassPass( - MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel})); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() { -#if LLVM_VERSION_LT(15, 0) - return wrap(createThreadSanitizerLegacyPassPass()); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) { -#if LLVM_VERSION_LT(15, 0) - const bool CompileKernel = false; - - return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover)); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { -#if LLVM_VERSION_LT(15, 0) - assert(RustPass); - Pass *Pass = unwrap(RustPass); - PassManagerBase *PMB = unwrap(PMR); - PMB->add(Pass); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() { -#if LLVM_VERSION_LT(15, 0) - return LLVMPassManagerBuilderCreate(); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) { -#if LLVM_VERSION_LT(15, 0) - LLVMPassManagerBuilderDispose(PMB); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager( - LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { -#if LLVM_VERSION_LT(15, 0) - LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager( - LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { -#if LLVM_VERSION_LT(15, 0) - LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager( - LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) { -#if LLVM_VERSION_LT(15, 0) - LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" -void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( - LLVMPassManagerBuilderRef PMBR, - LLVMPassManagerRef PMR -) { -#if LLVM_VERSION_LT(15, 0) - unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold( - LLVMPassManagerBuilderRef PMB, unsigned Threshold) { -#if LLVM_VERSION_LT(15, 0) - LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -extern "C" -void LLVMRustAddLastExtensionPasses( - LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { -#if LLVM_VERSION_LT(15, 0) - auto AddExtensionPasses = [Passes, NumPasses]( - const PassManagerBuilder &Builder, PassManagerBase &PM) { - for (size_t I = 0; I < NumPasses; I++) { - PM.add(unwrap(Passes[I])); - } - }; - // Add the passes to both of the pre-finalization extension points, - // so they are run for optimized and non-optimized builds. - unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, - AddExtensionPasses); - unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, - AddExtensionPasses); -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -#ifdef LLVM_COMPONENT_X86 -#define SUBTARGET_X86 SUBTARGET(X86) -#else -#define SUBTARGET_X86 -#endif - -#ifdef LLVM_COMPONENT_ARM -#define SUBTARGET_ARM SUBTARGET(ARM) -#else -#define SUBTARGET_ARM -#endif - -#ifdef LLVM_COMPONENT_AARCH64 -#define SUBTARGET_AARCH64 SUBTARGET(AArch64) -#else -#define SUBTARGET_AARCH64 -#endif - -#ifdef LLVM_COMPONENT_AVR -#define SUBTARGET_AVR SUBTARGET(AVR) -#else -#define SUBTARGET_AVR -#endif - -#ifdef LLVM_COMPONENT_M68k -#define SUBTARGET_M68K SUBTARGET(M68k) -#else -#define SUBTARGET_M68K -#endif - -#ifdef LLVM_COMPONENT_MIPS -#define SUBTARGET_MIPS SUBTARGET(Mips) -#else -#define SUBTARGET_MIPS -#endif - -#ifdef LLVM_COMPONENT_POWERPC -#define SUBTARGET_PPC SUBTARGET(PPC) -#else -#define SUBTARGET_PPC -#endif - -#ifdef LLVM_COMPONENT_SYSTEMZ -#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) -#else -#define SUBTARGET_SYSTEMZ -#endif - -#ifdef LLVM_COMPONENT_MSP430 -#define SUBTARGET_MSP430 SUBTARGET(MSP430) -#else -#define SUBTARGET_MSP430 -#endif - -#ifdef LLVM_COMPONENT_RISCV -#define SUBTARGET_RISCV SUBTARGET(RISCV) -#else -#define SUBTARGET_RISCV -#endif - -#ifdef LLVM_COMPONENT_SPARC -#define SUBTARGET_SPARC SUBTARGET(Sparc) -#else -#define SUBTARGET_SPARC -#endif - -#ifdef LLVM_COMPONENT_HEXAGON -#define SUBTARGET_HEXAGON SUBTARGET(Hexagon) -#else -#define SUBTARGET_HEXAGON -#endif - -#define GEN_SUBTARGETS \ - SUBTARGET_X86 \ - SUBTARGET_ARM \ - SUBTARGET_AARCH64 \ - SUBTARGET_AVR \ - SUBTARGET_M68K \ - SUBTARGET_MIPS \ - SUBTARGET_PPC \ - SUBTARGET_SYSTEMZ \ - SUBTARGET_MSP430 \ - SUBTARGET_SPARC \ - SUBTARGET_HEXAGON \ - SUBTARGET_RISCV \ - -#define SUBTARGET(x) \ - namespace llvm { \ - extern const SubtargetFeatureKV x##FeatureKV[]; \ - extern const SubtargetFeatureKV x##SubTypeKV[]; \ - } - -GEN_SUBTARGETS -#undef SUBTARGET - -extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, - const char *Feature) { - TargetMachine *Target = unwrap(TM); - const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); - return MCInfo->checkFeatures(std::string("+") + Feature); -} - -enum class LLVMRustCodeModel { - Tiny, - Small, - Kernel, - Medium, - Large, - None, -}; - -static Optional fromRust(LLVMRustCodeModel Model) { - switch (Model) { - case LLVMRustCodeModel::Tiny: - return CodeModel::Tiny; - case LLVMRustCodeModel::Small: - return CodeModel::Small; - case LLVMRustCodeModel::Kernel: - return CodeModel::Kernel; - case LLVMRustCodeModel::Medium: - return CodeModel::Medium; - case LLVMRustCodeModel::Large: - return CodeModel::Large; - case LLVMRustCodeModel::None: - return None; - default: - report_fatal_error("Bad CodeModel."); - } -} - -enum class LLVMRustCodeGenOptLevel { - None, - Less, - Default, - Aggressive, -}; - -static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { - switch (Level) { - case LLVMRustCodeGenOptLevel::None: - return CodeGenOpt::None; - case LLVMRustCodeGenOptLevel::Less: - return CodeGenOpt::Less; - case LLVMRustCodeGenOptLevel::Default: - return CodeGenOpt::Default; - case LLVMRustCodeGenOptLevel::Aggressive: - return CodeGenOpt::Aggressive; - default: - report_fatal_error("Bad CodeGenOptLevel."); - } -} - -enum class LLVMRustPassBuilderOptLevel { - O0, - O1, - O2, - O3, - Os, - Oz, -}; - -#if LLVM_VERSION_LT(14,0) -using OptimizationLevel = PassBuilder::OptimizationLevel; -#endif - -static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { - switch (Level) { - case LLVMRustPassBuilderOptLevel::O0: - return OptimizationLevel::O0; - case LLVMRustPassBuilderOptLevel::O1: - return OptimizationLevel::O1; - case LLVMRustPassBuilderOptLevel::O2: - return OptimizationLevel::O2; - case LLVMRustPassBuilderOptLevel::O3: - return OptimizationLevel::O3; - case LLVMRustPassBuilderOptLevel::Os: - return OptimizationLevel::Os; - case LLVMRustPassBuilderOptLevel::Oz: - return OptimizationLevel::Oz; - default: - report_fatal_error("Bad PassBuilderOptLevel."); - } -} - -enum class LLVMRustRelocModel { - Static, - PIC, - DynamicNoPic, - ROPI, - RWPI, - ROPIRWPI, -}; - -static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { - switch (RustReloc) { - case LLVMRustRelocModel::Static: - return Reloc::Static; - case LLVMRustRelocModel::PIC: - return Reloc::PIC_; - case LLVMRustRelocModel::DynamicNoPic: - return Reloc::DynamicNoPIC; - case LLVMRustRelocModel::ROPI: - return Reloc::ROPI; - case LLVMRustRelocModel::RWPI: - return Reloc::RWPI; - case LLVMRustRelocModel::ROPIRWPI: - return Reloc::ROPI_RWPI; - } - report_fatal_error("Bad RelocModel."); -} - -#ifdef LLVM_RUSTLLVM -/// getLongestEntryLength - Return the length of the longest entry in the table. -template -static size_t getLongestEntryLength(ArrayRef Table) { - size_t MaxLen = 0; - for (auto &I : Table) - MaxLen = std::max(MaxLen, std::strlen(I.Key)); - return MaxLen; -} - -extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { - const TargetMachine *Target = unwrap(TM); - const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); - const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); - const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); - const ArrayRef CPUTable = MCInfo->getCPUTable(); - unsigned MaxCPULen = getLongestEntryLength(CPUTable); - - printf("Available CPUs for this target:\n"); - if (HostArch == TargetArch) { - const StringRef HostCPU = sys::getHostCPUName(); - printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", - MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); - } - for (auto &CPU : CPUTable) - printf(" %-*s\n", MaxCPULen, CPU.Key); - printf("\n"); -} - -extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { - const TargetMachine *Target = unwrap(TM); - const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); - const ArrayRef FeatTable = MCInfo->getFeatureTable(); - return FeatTable.size(); -} - -extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, - const char** Feature, const char** Desc) { - const TargetMachine *Target = unwrap(TM); - const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); - const ArrayRef FeatTable = MCInfo->getFeatureTable(); - const SubtargetFeatureKV Feat = FeatTable[Index]; - *Feature = Feat.Key; - *Desc = Feat.Desc; -} - -#else - -extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { - printf("Target CPU help is not supported by this LLVM version.\n\n"); -} - -extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) { - return 0; -} - -extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {} -#endif - -extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { - StringRef Name = sys::getHostCPUName(); - *len = Name.size(); - return Name.data(); -} - -extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( - const char *TripleStr, const char *CPU, const char *Feature, - const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, - LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, - bool FunctionSections, - bool DataSections, - bool UniqueSectionNames, - bool TrapUnreachable, - bool Singlethread, - bool AsmComments, - bool EmitStackSizeSection, - bool RelaxELFRelocations, - bool UseInitArray, - const char *SplitDwarfFile) { - - auto OptLevel = fromRust(RustOptLevel); - auto RM = fromRust(RustReloc); - auto CM = fromRust(RustCM); - - std::string Error; - Triple Trip(Triple::normalize(TripleStr)); - const llvm::Target *TheTarget = - TargetRegistry::lookupTarget(Trip.getTriple(), Error); - if (TheTarget == nullptr) { - LLVMRustSetLastError(Error.c_str()); - return nullptr; - } - - TargetOptions Options; - - Options.FloatABIType = FloatABI::Default; - if (UseSoftFloat) { - Options.FloatABIType = FloatABI::Soft; - } - Options.DataSections = DataSections; - Options.FunctionSections = FunctionSections; - Options.UniqueSectionNames = UniqueSectionNames; - Options.MCOptions.AsmVerbose = AsmComments; - Options.MCOptions.PreserveAsmComments = AsmComments; - Options.MCOptions.ABIName = ABIStr; - if (SplitDwarfFile) { - Options.MCOptions.SplitDwarfFile = SplitDwarfFile; - } - Options.RelaxELFRelocations = RelaxELFRelocations; - Options.UseInitArray = UseInitArray; - - if (TrapUnreachable) { - // Tell LLVM to codegen `unreachable` into an explicit trap instruction. - // This limits the extent of possible undefined behavior in some cases, as - // it prevents control flow from "falling through" into whatever code - // happens to be laid out next in memory. - Options.TrapUnreachable = true; - } - - if (Singlethread) { - Options.ThreadModel = ThreadModel::Single; - } - - Options.EmitStackSizeSection = EmitStackSizeSection; - - TargetMachine *TM = TheTarget->createTargetMachine( - Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); - return wrap(TM); -} - -extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { - delete unwrap(TM); -} - -extern "C" void LLVMRustConfigurePassManagerBuilder( - LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, - bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, - const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath, - int SizeLevel) { -#if LLVM_VERSION_LT(15, 0) - unwrap(PMBR)->MergeFunctions = MergeFunctions; - unwrap(PMBR)->SLPVectorize = SLPVectorize; - unwrap(PMBR)->OptLevel = fromRust(OptLevel); - unwrap(PMBR)->LoopVectorize = LoopVectorize; - unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; - unwrap(PMBR)->SizeLevel = SizeLevel; - unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0; - - if (PGOGenPath) { - assert(!PGOUsePath && !PGOSampleUsePath); - unwrap(PMBR)->EnablePGOInstrGen = true; - unwrap(PMBR)->PGOInstrGen = PGOGenPath; - } else if (PGOUsePath) { - assert(!PGOSampleUsePath); - unwrap(PMBR)->PGOInstrUse = PGOUsePath; - } else if (PGOSampleUsePath) { - unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath; - } -#else - report_fatal_error("Legacy PM not supported with LLVM 15"); -#endif -} - -// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` -// field of a PassManagerBuilder, we expose our own method of doing so. -extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, - LLVMModuleRef M, - bool DisableSimplifyLibCalls) { - Triple TargetTriple(unwrap(M)->getTargetTriple()); - TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); - if (DisableSimplifyLibCalls) - TLI->disableAllFunctions(); - unwrap(PMBR)->LibraryInfo = TLI; -} - -// Unfortunately, the LLVM C API doesn't provide a way to create the -// TargetLibraryInfo pass, so we use this method to do so. -extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, - bool DisableSimplifyLibCalls) { - Triple TargetTriple(unwrap(M)->getTargetTriple()); - TargetLibraryInfoImpl TLII(TargetTriple); - if (DisableSimplifyLibCalls) - TLII.disableAllFunctions(); - unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); -} - -// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over -// all the functions in a module, so we do that manually here. You'll find -// similar code in clang's BackendUtil.cpp file. -extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, - LLVMModuleRef M) { - llvm::legacy::FunctionPassManager *P = - unwrap(PMR); - P->doInitialization(); - - // Upgrade all calls to old intrinsics first. - for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) - UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove - - for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; - ++I) - if (!I->isDeclaration()) - P->run(*I); - - P->doFinalization(); -} - -extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { - // Initializing the command-line options more than once is not allowed. So, - // check if they've already been initialized. (This could happen if we're - // being called from rustpkg, for example). If the arguments change, then - // that's just kinda unfortunate. - static bool Initialized = false; - if (Initialized) - return; - Initialized = true; - cl::ParseCommandLineOptions(Argc, Argv); -} - -enum class LLVMRustFileType { - AssemblyFile, - ObjectFile, -}; - -static CodeGenFileType fromRust(LLVMRustFileType Type) { - switch (Type) { - case LLVMRustFileType::AssemblyFile: - return CGFT_AssemblyFile; - case LLVMRustFileType::ObjectFile: - return CGFT_ObjectFile; - default: - report_fatal_error("Bad FileType."); - } -} - -extern "C" LLVMRustResult -LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, - LLVMModuleRef M, const char *Path, const char *DwoPath, - LLVMRustFileType RustFileType) { - llvm::legacy::PassManager *PM = unwrap(PMR); - auto FileType = fromRust(RustFileType); - - std::string ErrorInfo; - std::error_code EC; - raw_fd_ostream OS(Path, EC, sys::fs::OF_None); - if (EC) - ErrorInfo = EC.message(); - if (ErrorInfo != "") { - LLVMRustSetLastError(ErrorInfo.c_str()); - return LLVMRustResult::Failure; - } - - buffer_ostream BOS(OS); - if (DwoPath) { - raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None); - EC.clear(); - if (EC) - ErrorInfo = EC.message(); - if (ErrorInfo != "") { - LLVMRustSetLastError(ErrorInfo.c_str()); - return LLVMRustResult::Failure; - } - buffer_ostream DBOS(DOS); - unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); - PM->run(*unwrap(M)); - } else { - unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); - PM->run(*unwrap(M)); - } - - // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output - // stream (OS), so the only real safe place to delete this is here? Don't we - // wish this was written in Rust? - LLVMDisposePassManager(PMR); - return LLVMRustResult::Success; -} - -extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler - const char*, // pass name - const char*); // IR name -extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler - -std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { - if (any_isa(WrappedIr)) - return any_cast(WrappedIr)->getName().str(); - if (any_isa(WrappedIr)) - return any_cast(WrappedIr)->getName().str(); - if (any_isa(WrappedIr)) - return any_cast(WrappedIr)->getName().str(); - if (any_isa(WrappedIr)) - return any_cast(WrappedIr)->getName(); - return ""; -} - - -void LLVMSelfProfileInitializeCallbacks( - PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, - LLVMRustSelfProfileBeforePassCallback BeforePassCallback, - LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { - PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( - StringRef Pass, llvm::Any Ir) { - std::string PassName = Pass.str(); - std::string IrName = LLVMRustwrappedIrGetName(Ir); - BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); - }); - - PIC.registerAfterPassCallback( - [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, - const PreservedAnalyses &Preserved) { - AfterPassCallback(LlvmSelfProfiler); - }); - - PIC.registerAfterPassInvalidatedCallback( - [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { - AfterPassCallback(LlvmSelfProfiler); - }); - - PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( - StringRef Pass, llvm::Any Ir) { - std::string PassName = Pass.str(); - std::string IrName = LLVMRustwrappedIrGetName(Ir); - BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); - }); - - PIC.registerAfterAnalysisCallback( - [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { - AfterPassCallback(LlvmSelfProfiler); - }); -} - -enum class LLVMRustOptStage { - PreLinkNoLTO, - PreLinkThinLTO, - PreLinkFatLTO, - ThinLTO, - FatLTO, -}; - -struct LLVMRustSanitizerOptions { - bool SanitizeAddress; - bool SanitizeAddressRecover; - bool SanitizeMemory; - bool SanitizeMemoryRecover; - int SanitizeMemoryTrackOrigins; - bool SanitizeThread; - bool SanitizeHWAddress; - bool SanitizeHWAddressRecover; -}; - -extern "C" LLVMRustResult -LLVMRustOptimizeWithNewPassManager( - LLVMModuleRef ModuleRef, - LLVMTargetMachineRef TMRef, - LLVMRustPassBuilderOptLevel OptLevelRust, - LLVMRustOptStage OptStage, - bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, - bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, - bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, - LLVMRustSanitizerOptions *SanitizerOptions, - const char *PGOGenPath, const char *PGOUsePath, - bool InstrumentCoverage, bool InstrumentGCOV, - const char *PGOSampleUsePath, bool DebugInfoForProfiling, - void* LlvmSelfProfiler, - LLVMRustSelfProfileBeforePassCallback BeforePassCallback, - LLVMRustSelfProfileAfterPassCallback AfterPassCallback, - const char *ExtraPasses, size_t ExtraPassesLen, - const char *LLVMPlugins, size_t LLVMPluginsLen) { - Module *TheModule = unwrap(ModuleRef); - TargetMachine *TM = unwrap(TMRef); - OptimizationLevel OptLevel = fromRust(OptLevelRust); - - - PipelineTuningOptions PTO; - PTO.LoopUnrolling = UnrollLoops; - PTO.LoopInterleaving = UnrollLoops; - PTO.LoopVectorization = LoopVectorize; - PTO.SLPVectorization = SLPVectorize; - PTO.MergeFunctions = MergeFunctions; - - // FIXME: We may want to expose this as an option. - bool DebugPassManager = false; - - PassInstrumentationCallbacks PIC; - StandardInstrumentations SI(DebugPassManager); - SI.registerCallbacks(PIC); - - if (LlvmSelfProfiler){ - LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); - } - - Optional PGOOpt; - if (PGOGenPath) { - assert(!PGOUsePath && !PGOSampleUsePath); - PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr, - PGOOptions::NoCSAction, DebugInfoForProfiling); - } else if (PGOUsePath) { - assert(!PGOSampleUsePath); - PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse, - PGOOptions::NoCSAction, DebugInfoForProfiling); - } else if (PGOSampleUsePath) { - PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse, - PGOOptions::NoCSAction, DebugInfoForProfiling); - } else if (DebugInfoForProfiling) { - PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction, - PGOOptions::NoCSAction, DebugInfoForProfiling); - } - -#if LLVM_VERSION_GE(13, 0) - PassBuilder PB(TM, PTO, PGOOpt, &PIC); - LoopAnalysisManager LAM; - FunctionAnalysisManager FAM; - CGSCCAnalysisManager CGAM; - ModuleAnalysisManager MAM; -#else - PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC); - LoopAnalysisManager LAM(DebugPassManager); - FunctionAnalysisManager FAM(DebugPassManager); - CGSCCAnalysisManager CGAM(DebugPassManager); - ModuleAnalysisManager MAM(DebugPassManager); -#endif - - FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); - - Triple TargetTriple(TheModule->getTargetTriple()); - std::unique_ptr TLII(new TargetLibraryInfoImpl(TargetTriple)); - if (DisableSimplifyLibCalls) - TLII->disableAllFunctions(); - FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); - - PB.registerModuleAnalyses(MAM); - PB.registerCGSCCAnalyses(CGAM); - PB.registerFunctionAnalyses(FAM); - PB.registerLoopAnalyses(LAM); - PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); - - // We manually collect pipeline callbacks so we can apply them at O0, where the - // PassBuilder does not create a pipeline. - std::vector> - PipelineStartEPCallbacks; - std::vector> - OptimizerLastEPCallbacks; - - if (VerifyIR) { - PipelineStartEPCallbacks.push_back( - [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(VerifierPass()); - } - ); - } - - if (InstrumentGCOV) { - PipelineStartEPCallbacks.push_back( - [](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); - } - ); - } - - if (InstrumentCoverage) { - PipelineStartEPCallbacks.push_back( - [](ModulePassManager &MPM, OptimizationLevel Level) { - InstrProfOptions Options; - MPM.addPass(InstrProfiling(Options, false)); - } - ); - } - - if (SanitizerOptions) { - if (SanitizerOptions->SanitizeMemory) { - MemorySanitizerOptions Options( - SanitizerOptions->SanitizeMemoryTrackOrigins, - SanitizerOptions->SanitizeMemoryRecover, - /*CompileKernel=*/false); - OptimizerLastEPCallbacks.push_back( - [Options](ModulePassManager &MPM, OptimizationLevel Level) { -#if LLVM_VERSION_GE(14, 0) - MPM.addPass(ModuleMemorySanitizerPass(Options)); -#else - MPM.addPass(MemorySanitizerPass(Options)); -#endif - MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); - } - ); - } - - if (SanitizerOptions->SanitizeThread) { - OptimizerLastEPCallbacks.push_back( - [](ModulePassManager &MPM, OptimizationLevel Level) { -#if LLVM_VERSION_GE(14, 0) - MPM.addPass(ModuleThreadSanitizerPass()); -#else - MPM.addPass(ThreadSanitizerPass()); -#endif - MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); - } - ); - } - - if (SanitizerOptions->SanitizeAddress) { - OptimizerLastEPCallbacks.push_back( - [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { -#if LLVM_VERSION_LT(15, 0) - MPM.addPass(RequireAnalysisPass()); -#endif -#if LLVM_VERSION_GE(14, 0) - AddressSanitizerOptions opts = AddressSanitizerOptions{ - /*CompileKernel=*/false, - SanitizerOptions->SanitizeAddressRecover, - /*UseAfterScope=*/true, - AsanDetectStackUseAfterReturnMode::Runtime, - }; - MPM.addPass(ModuleAddressSanitizerPass(opts)); -#else - MPM.addPass(ModuleAddressSanitizerPass( - /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover)); - MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( - /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover, - /*UseAfterScope=*/true))); -#endif - } - ); - } - if (SanitizerOptions->SanitizeHWAddress) { - OptimizerLastEPCallbacks.push_back( - [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { -#if LLVM_VERSION_GE(14, 0) - HWAddressSanitizerOptions opts( - /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, - /*DisableOptimization=*/false); - MPM.addPass(HWAddressSanitizerPass(opts)); -#else - MPM.addPass(HWAddressSanitizerPass( - /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover)); -#endif - } - ); - } - } - - if (LLVMPluginsLen) { - auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); - SmallVector Plugins; - PluginsStr.split(Plugins, ',', -1, false); - for (auto PluginPath: Plugins) { - auto Plugin = PassPlugin::Load(PluginPath.str()); - if (!Plugin) { - LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); - continue; - } - Plugin->registerPassBuilderCallbacks(PB); - } - } - -#if LLVM_VERSION_GE(13, 0) - ModulePassManager MPM; -#else - ModulePassManager MPM(DebugPassManager); -#endif - bool NeedThinLTOBufferPasses = UseThinLTOBuffers; - if (!NoPrepopulatePasses) { - // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead. - // At the same time, the LTO pipelines do support O0 and using them is required. - bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; - if (OptLevel == OptimizationLevel::O0 && !IsLTO) { - for (const auto &C : PipelineStartEPCallbacks) - PB.registerPipelineStartEPCallback(C); - for (const auto &C : OptimizerLastEPCallbacks) - PB.registerOptimizerLastEPCallback(C); - - // Pass false as we manually schedule ThinLTOBufferPasses below. - MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); - } else { - for (const auto &C : PipelineStartEPCallbacks) - PB.registerPipelineStartEPCallback(C); - if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { - for (const auto &C : OptimizerLastEPCallbacks) - PB.registerOptimizerLastEPCallback(C); - } - - switch (OptStage) { - case LLVMRustOptStage::PreLinkNoLTO: - MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); - break; - case LLVMRustOptStage::PreLinkThinLTO: - MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); - // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback - // passes may still run afterwards. This means we need to run the buffer passes again. - // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks - // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks. - if (OptimizerLastEPCallbacks.empty()) - NeedThinLTOBufferPasses = false; - for (const auto &C : OptimizerLastEPCallbacks) - C(MPM, OptLevel); - break; - case LLVMRustOptStage::PreLinkFatLTO: - MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); - NeedThinLTOBufferPasses = false; - break; - case LLVMRustOptStage::ThinLTO: - // FIXME: Does it make sense to pass the ModuleSummaryIndex? - // It only seems to be needed for C++ specific optimizations. - MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); - break; - case LLVMRustOptStage::FatLTO: - MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); - break; - } - } - } else { - // We're not building any of the default pipelines but we still want to - // add the verifier, instrumentation, etc passes if they were requested - for (const auto &C : PipelineStartEPCallbacks) - C(MPM, OptLevel); - for (const auto &C : OptimizerLastEPCallbacks) - C(MPM, OptLevel); - } - - if (ExtraPassesLen) { - if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { - std::string ErrMsg = toString(std::move(Err)); - LLVMRustSetLastError(ErrMsg.c_str()); - return LLVMRustResult::Failure; - } - } - - if (NeedThinLTOBufferPasses) { - MPM.addPass(CanonicalizeAliasesPass()); - MPM.addPass(NameAnonGlobalPass()); - } - - // Upgrade all calls to old intrinsics first. - for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) - UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove - - MPM.run(*TheModule, MAM); - return LLVMRustResult::Success; -} - -// Callback to demangle function name -// Parameters: -// * name to be demangled -// * name len -// * output buffer -// * output buffer len -// Returns len of demangled string, or 0 if demangle failed. -typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); - - -namespace { - -class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { - DemangleFn Demangle; - std::vector Buf; - -public: - RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} - - // Return empty string if demangle failed - // or if name does not need to be demangled - StringRef CallDemangle(StringRef name) { - if (!Demangle) { - return StringRef(); - } - - if (Buf.size() < name.size() * 2) { - // Semangled name usually shorter than mangled, - // but allocate twice as much memory just in case - Buf.resize(name.size() * 2); - } - - auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); - if (!R) { - // Demangle failed. - return StringRef(); - } - - auto Demangled = StringRef(Buf.data(), R); - if (Demangled == name) { - // Do not print anything if demangled name is equal to mangled. - return StringRef(); - } - - return Demangled; - } - - void emitFunctionAnnot(const Function *F, - formatted_raw_ostream &OS) override { - StringRef Demangled = CallDemangle(F->getName()); - if (Demangled.empty()) { - return; - } - - OS << "; " << Demangled << "\n"; - } - - void emitInstructionAnnot(const Instruction *I, - formatted_raw_ostream &OS) override { - const char *Name; - const Value *Value; - if (const CallInst *CI = dyn_cast(I)) { - Name = "call"; - Value = CI->getCalledOperand(); - } else if (const InvokeInst* II = dyn_cast(I)) { - Name = "invoke"; - Value = II->getCalledOperand(); - } else { - // Could demangle more operations, e. g. - // `store %place, @function`. - return; - } - - if (!Value->hasName()) { - return; - } - - StringRef Demangled = CallDemangle(Value->getName()); - if (Demangled.empty()) { - return; - } - - OS << "; " << Name << " " << Demangled << "\n"; - } -}; - -} // namespace - -extern "C" LLVMRustResult -LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { - std::string ErrorInfo; - std::error_code EC; - raw_fd_ostream OS(Path, EC, sys::fs::OF_None); - if (EC) - ErrorInfo = EC.message(); - if (ErrorInfo != "") { - LLVMRustSetLastError(ErrorInfo.c_str()); - return LLVMRustResult::Failure; - } - - RustAssemblyAnnotationWriter AAW(Demangle); - formatted_raw_ostream FOS(OS); - unwrap(M)->print(FOS, &AAW); - - return LLVMRustResult::Success; -} - -extern "C" void LLVMRustPrintPasses() { - LLVMInitializePasses(); - struct MyListener : PassRegistrationListener { - void passEnumerate(const PassInfo *Info) { - StringRef PassArg = Info->getPassArgument(); - StringRef PassName = Info->getPassName(); - if (!PassArg.empty()) { - // These unsigned->signed casts could theoretically overflow, but - // realistically never will (and even if, the result is implementation - // defined rather plain UB). - printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), - (int)PassName.size(), PassName.data()); - } - } - } Listener; - - PassRegistry *PR = PassRegistry::getPassRegistry(); - PR->enumerateWith(&Listener); -} - -extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, - bool AddLifetimes) { - unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); -} - -extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, - size_t Len) { - llvm::legacy::PassManager passes; - - auto PreserveFunctions = [=](const GlobalValue &GV) { - for (size_t I = 0; I < Len; I++) { - if (GV.getName() == Symbols[I]) { - return true; - } - } - return false; - }; - - passes.add(llvm::createInternalizePass(PreserveFunctions)); - - passes.run(*unwrap(M)); -} - -extern "C" void -LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, - LLVMTargetMachineRef TMR) { - TargetMachine *Target = unwrap(TMR); - unwrap(Module)->setDataLayout(Target->createDataLayout()); -} - -extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { - unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); -} - -extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { - unwrap(M)->setPIELevel(PIELevel::Level::Large); -} - -extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, - LLVMRustCodeModel Model) { - auto CM = fromRust(Model); - if (!CM.hasValue()) - return; - unwrap(M)->setCodeModel(*CM); -} - -// Here you'll find an implementation of ThinLTO as used by the Rust compiler -// right now. This ThinLTO support is only enabled on "recent ish" versions of -// LLVM, and otherwise it's just blanket rejected from other compilers. -// -// Most of this implementation is straight copied from LLVM. At the time of -// this writing it wasn't *quite* suitable to reuse more code from upstream -// for our purposes, but we should strive to upstream this support once it's -// ready to go! I figure we may want a bit of testing locally first before -// sending this upstream to LLVM. I hear though they're quite eager to receive -// feedback like this! -// -// If you're reading this code and wondering "what in the world" or you're -// working "good lord by LLVM upgrade is *still* failing due to these bindings" -// then fear not! (ok maybe fear a little). All code here is mostly based -// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. -// -// You'll find that the general layout here roughly corresponds to the `run` -// method in that file as well as `ProcessThinLTOModule`. Functions are -// specifically commented below as well, but if you're updating this code -// or otherwise trying to understand it, the LLVM source will be useful in -// interpreting the mysteries within. -// -// Otherwise I'll apologize in advance, it probably requires a relatively -// significant investment on your part to "truly understand" what's going on -// here. Not saying I do myself, but it took me awhile staring at LLVM's source -// and various online resources about ThinLTO to make heads or tails of all -// this. - -// This is a shared data structure which *must* be threadsafe to share -// read-only amongst threads. This also corresponds basically to the arguments -// of the `ProcessThinLTOModule` function in the LLVM source. -struct LLVMRustThinLTOData { - // The combined index that is the global analysis over all modules we're - // performing ThinLTO for. This is mostly managed by LLVM. - ModuleSummaryIndex Index; - - // All modules we may look at, stored as in-memory serialized versions. This - // is later used when inlining to ensure we can extract any module to inline - // from. - StringMap ModuleMap; - - // A set that we manage of everything we *don't* want internalized. Note that - // this includes all transitive references right now as well, but it may not - // always! - DenseSet GUIDPreservedSymbols; - - // Not 100% sure what these are, but they impact what's internalized and - // what's inlined across modules, I believe. - StringMap ImportLists; - StringMap ExportLists; - StringMap ModuleToDefinedGVSummaries; - StringMap> ResolvedODR; - - LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} -}; - -// Just an argument to the `LLVMRustCreateThinLTOData` function below. -struct LLVMRustThinLTOModule { - const char *identifier; - const char *data; - size_t len; -}; - -// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it -// does. -static const GlobalValueSummary * -getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { - auto StrongDefForLinker = llvm::find_if( - GVSummaryList, [](const std::unique_ptr &Summary) { - auto Linkage = Summary->linkage(); - return !GlobalValue::isAvailableExternallyLinkage(Linkage) && - !GlobalValue::isWeakForLinker(Linkage); - }); - if (StrongDefForLinker != GVSummaryList.end()) - return StrongDefForLinker->get(); - - auto FirstDefForLinker = llvm::find_if( - GVSummaryList, [](const std::unique_ptr &Summary) { - auto Linkage = Summary->linkage(); - return !GlobalValue::isAvailableExternallyLinkage(Linkage); - }); - if (FirstDefForLinker == GVSummaryList.end()) - return nullptr; - return FirstDefForLinker->get(); -} - -// The main entry point for creating the global ThinLTO analysis. The structure -// here is basically the same as before threads are spawned in the `run` -// function of `lib/LTO/ThinLTOCodeGenerator.cpp`. -extern "C" LLVMRustThinLTOData* -LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, - int num_modules, - const char **preserved_symbols, - int num_symbols) { - auto Ret = std::make_unique(); - - // Load each module's summary and merge it into one combined index - for (int i = 0; i < num_modules; i++) { - auto module = &modules[i]; - StringRef buffer(module->data, module->len); - MemoryBufferRef mem_buffer(buffer, module->identifier); - - Ret->ModuleMap[module->identifier] = mem_buffer; - - if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { - LLVMRustSetLastError(toString(std::move(Err)).c_str()); - return nullptr; - } - } - - // Collect for each module the list of function it defines (GUID -> Summary) - Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); - - // Convert the preserved symbols set from string to GUID, this is then needed - // for internalization. - for (int i = 0; i < num_symbols; i++) { - auto GUID = GlobalValue::getGUID(preserved_symbols[i]); - Ret->GUIDPreservedSymbols.insert(GUID); - } - - // Collect the import/export lists for all modules from the call-graph in the - // combined index - // - // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` - auto deadIsPrevailing = [&](GlobalValue::GUID G) { - return PrevailingType::Unknown; - }; - // We don't have a complete picture in our use of ThinLTO, just our immediate - // crate, so we need `ImportEnabled = false` to limit internalization. - // Otherwise, we sometimes lose `static` values -- see #60184. - computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, - deadIsPrevailing, /* ImportEnabled = */ false); - ComputeCrossModuleImport( - Ret->Index, - Ret->ModuleToDefinedGVSummaries, - Ret->ImportLists, - Ret->ExportLists - ); - - // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it - // impacts the caching. - // - // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this - // being lifted from `lib/LTO/LTO.cpp` as well - DenseMap PrevailingCopy; - for (auto &I : Ret->Index) { - if (I.second.SummaryList.size() > 1) - PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); - } - auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { - const auto &Prevailing = PrevailingCopy.find(GUID); - if (Prevailing == PrevailingCopy.end()) - return true; - return Prevailing->second == S; - }; - auto recordNewLinkage = [&](StringRef ModuleIdentifier, - GlobalValue::GUID GUID, - GlobalValue::LinkageTypes NewLinkage) { - Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; - }; - -#if LLVM_VERSION_GE(13,0) - // Uses FromPrevailing visibility scheme which works for many binary - // formats. We probably could and should use ELF visibility scheme for many of - // our targets, however. - lto::Config conf; - thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, - Ret->GUIDPreservedSymbols); -#else - thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, - Ret->GUIDPreservedSymbols); -#endif - // Here we calculate an `ExportedGUIDs` set for use in the `isExported` - // callback below. This callback below will dictate the linkage for all - // summaries in the index, and we basically just only want to ensure that dead - // symbols are internalized. Otherwise everything that's already external - // linkage will stay as external, and internal will stay as internal. - std::set ExportedGUIDs; - for (auto &List : Ret->Index) { - for (auto &GVS: List.second.SummaryList) { - if (GlobalValue::isLocalLinkage(GVS->linkage())) - continue; - auto GUID = GVS->getOriginalName(); - if (GVS->flags().Live) - ExportedGUIDs.insert(GUID); - } - } - auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { - const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); - return (ExportList != Ret->ExportLists.end() && - ExportList->second.count(VI)) || - ExportedGUIDs.count(VI.getGUID()); - }; - thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); - - return Ret.release(); -} - -extern "C" void -LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { - delete Data; -} - -// Below are the various passes that happen *per module* when doing ThinLTO. -// -// In other words, these are the functions that are all run concurrently -// with one another, one per module. The passes here correspond to the analysis -// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the -// `ProcessThinLTOModule` function. Here they're split up into separate steps -// so rustc can save off the intermediate bytecode between each step. - -static bool -clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { - // When linking an ELF shared object, dso_local should be dropped. We - // conservatively do this for -fpic. - bool ClearDSOLocalOnDeclarations = - TM.getTargetTriple().isOSBinFormatELF() && - TM.getRelocationModel() != Reloc::Static && - Mod.getPIELevel() == PIELevel::Default; - return ClearDSOLocalOnDeclarations; -} - -extern "C" bool -LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, - LLVMTargetMachineRef TM) { - Module &Mod = *unwrap(M); - TargetMachine &Target = *unwrap(TM); - - bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); - bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); - - if (error) { - LLVMRustSetLastError("renameModuleForThinLTO failed"); - return false; - } - return true; -} - -extern "C" bool -LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { - Module &Mod = *unwrap(M); - const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); -#if LLVM_VERSION_GE(14, 0) - thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); -#else - thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); -#endif - return true; -} - -extern "C" bool -LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { - Module &Mod = *unwrap(M); - const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); - thinLTOInternalizeModule(Mod, DefinedGlobals); - return true; -} - -extern "C" bool -LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, - LLVMTargetMachineRef TM) { - Module &Mod = *unwrap(M); - TargetMachine &Target = *unwrap(TM); - - const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); - auto Loader = [&](StringRef Identifier) { - const auto &Memory = Data->ModuleMap.lookup(Identifier); - auto &Context = Mod.getContext(); - auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); - - if (!MOrErr) - return MOrErr; - - // The rest of this closure is a workaround for - // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports - // we accidentally import wasm custom sections into different modules, - // duplicating them by in the final output artifact. - // - // The issue is worked around here by manually removing the - // `wasm.custom_sections` named metadata node from any imported module. This - // we know isn't used by any optimization pass so there's no need for it to - // be imported. - // - // Note that the metadata is currently lazily loaded, so we materialize it - // here before looking up if there's metadata inside. The `FunctionImporter` - // will immediately materialize metadata anyway after an import, so this - // shouldn't be a perf hit. - if (Error Err = (*MOrErr)->materializeMetadata()) { - Expected> Ret(std::move(Err)); - return Ret; - } - - auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); - if (WasmCustomSections) - WasmCustomSections->eraseFromParent(); - - return MOrErr; - }; - bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); - FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal); - Expected Result = Importer.importFunctions(Mod, ImportList); - if (!Result) { - LLVMRustSetLastError(toString(Result.takeError()).c_str()); - return false; - } - return true; -} - -// This struct and various functions are sort of a hack right now, but the -// problem is that we've got in-memory LLVM modules after we generate and -// optimize all codegen-units for one compilation in rustc. To be compatible -// with the LTO support above we need to serialize the modules plus their -// ThinLTO summary into memory. -// -// This structure is basically an owned version of a serialize module, with -// a ThinLTO summary attached. -struct LLVMRustThinLTOBuffer { - std::string data; -}; - -extern "C" LLVMRustThinLTOBuffer* -LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { - auto Ret = std::make_unique(); - { - raw_string_ostream OS(Ret->data); - { - legacy::PassManager PM; - if (is_thin) { - PM.add(createWriteThinLTOBitcodePass(OS)); - } else { - PM.add(createBitcodeWriterPass(OS)); - } - PM.run(*unwrap(M)); - } - } - return Ret.release(); -} - -extern "C" void -LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { - delete Buffer; -} - -extern "C" const void* -LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { - return Buffer->data.data(); -} - -extern "C" size_t -LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { - return Buffer->data.length(); -} - -// This is what we used to parse upstream bitcode for actual ThinLTO -// processing. We'll call this once per module optimized through ThinLTO, and -// it'll be called concurrently on many threads. -extern "C" LLVMModuleRef -LLVMRustParseBitcodeForLTO(LLVMContextRef Context, - const char *data, - size_t len, - const char *identifier) { - StringRef Data(data, len); - MemoryBufferRef Buffer(Data, identifier); - unwrap(Context)->enableDebugTypeODRUniquing(); - Expected> SrcOrError = - parseBitcodeFile(Buffer, *unwrap(Context)); - if (!SrcOrError) { - LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); - return nullptr; - } - return wrap(std::move(*SrcOrError).release()); -} - -// Find the bitcode section in the object file data and return it as a slice. -// Fail if the bitcode section is present but empty. -// -// On success, the return value is the pointer to the start of the slice and -// `out_len` is filled with the (non-zero) length. On failure, the return value -// is `nullptr` and `out_len` is set to zero. -extern "C" const char* -LLVMRustGetBitcodeSliceFromObjectData(const char *data, - size_t len, - size_t *out_len) { - *out_len = 0; - - StringRef Data(data, len); - MemoryBufferRef Buffer(Data, ""); // The id is unused. - - Expected BitcodeOrError = - object::IRObjectFile::findBitcodeInMemBuffer(Buffer); - if (!BitcodeOrError) { - LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str()); - return nullptr; - } - - *out_len = BitcodeOrError->getBufferSize(); - return BitcodeOrError->getBufferStart(); -} - -// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See -// the comment in `back/lto.rs` for why this exists. -extern "C" void -LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, - DICompileUnit **A, - DICompileUnit **B) { - Module *M = unwrap(Mod); - DICompileUnit **Cur = A; - DICompileUnit **Next = B; - for (DICompileUnit *CU : M->debug_compile_units()) { - *Cur = CU; - Cur = Next; - Next = nullptr; - if (Cur == nullptr) - break; - } -} - -// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See -// the comment in `back/lto.rs` for why this exists. -extern "C" void -LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { - Module *M = unwrap(Mod); - - // If the original source module didn't have a `DICompileUnit` then try to - // merge all the existing compile units. If there aren't actually any though - // then there's not much for us to do so return. - if (Unit == nullptr) { - for (DICompileUnit *CU : M->debug_compile_units()) { - Unit = CU; - break; - } - if (Unit == nullptr) - return; - } - - // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and - // process it recursively. Note that we used to specifically iterate over - // instructions to ensure we feed everything into it, but `processModule` - // started doing this the same way in LLVM 7 (commit d769eb36ab2b8). - DebugInfoFinder Finder; - Finder.processModule(*M); - - // After we've found all our debuginfo, rewrite all subprograms to point to - // the same `DICompileUnit`. - for (auto &F : Finder.subprograms()) { - F->replaceUnit(Unit); - } - - // Erase any other references to other `DICompileUnit` instances, the verifier - // will later ensure that we don't actually have any other stale references to - // worry about. - auto *MD = M->getNamedMetadata("llvm.dbg.cu"); - MD->clearOperands(); - MD->addOperand(Unit); -} - -// Computes the LTO cache key for the provided 'ModId' in the given 'Data', -// storing the result in 'KeyOut'. -// Currently, this cache key is a SHA-1 hash of anything that could affect -// the result of optimizing this module (e.g. module imports, exports, liveness -// of access globals, etc). -// The precise details are determined by LLVM in `computeLTOCacheKey`, which is -// used during the normal linker-plugin incremental thin-LTO process. -extern "C" void -LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) { - SmallString<40> Key; - llvm::lto::Config conf; - const auto &ImportList = Data->ImportLists.lookup(ModId); - const auto &ExportList = Data->ExportLists.lookup(ModId); - const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); - const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); - std::set CfiFunctionDefs; - std::set CfiFunctionDecls; - - // Based on the 'InProcessThinBackend' constructor in LLVM - for (auto &Name : Data->Index.cfiFunctionDefs()) - CfiFunctionDefs.insert( - GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); - for (auto &Name : Data->Index.cfiFunctionDecls()) - CfiFunctionDecls.insert( - GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); - - llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, - ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls - ); - - LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); -} +// #include + +// #include +// #include + +// #include "LLVMWrapper.h" + +// #include "llvm/Analysis/AliasAnalysis.h" +// #include "llvm/Analysis/TargetLibraryInfo.h" +// #include "llvm/Analysis/TargetTransformInfo.h" +// #include "llvm/CodeGen/TargetSubtargetInfo.h" +// #include "llvm/InitializePasses.h" +// #include "llvm/IR/AutoUpgrade.h" +// #include "llvm/IR/AssemblyAnnotationWriter.h" +// #include "llvm/IR/IntrinsicInst.h" +// #include "llvm/IR/Verifier.h" +// #include "llvm/Object/ObjectFile.h" +// #include "llvm/Object/IRObjectFile.h" +// #include "llvm/Passes/PassBuilder.h" +// #include "llvm/Passes/PassPlugin.h" +// #include "llvm/Passes/StandardInstrumentations.h" +// #include "llvm/Support/CBindingWrapping.h" +// #include "llvm/Support/FileSystem.h" +// #include "llvm/Support/Host.h" +// #if LLVM_VERSION_LT(14, 0) +// #include "llvm/Support/TargetRegistry.h" +// #else +// #include "llvm/MC/TargetRegistry.h" +// #endif +// #include "llvm/Target/TargetMachine.h" +// #include "llvm/Transforms/IPO/PassManagerBuilder.h" +// #include "llvm/Transforms/IPO/AlwaysInliner.h" +// #include "llvm/Transforms/IPO/FunctionImport.h" +// #include "llvm/Transforms/Utils/AddDiscriminators.h" +// #include "llvm/Transforms/Utils/FunctionImportUtils.h" +// #include "llvm/LTO/LTO.h" +// #include "llvm/Bitcode/BitcodeWriterPass.h" +// #include "llvm-c/Transforms/PassManagerBuilder.h" + +// #include "llvm/Transforms/Instrumentation.h" +// #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" +// #include "llvm/Support/TimeProfiler.h" +// #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" +// #include "llvm/Transforms/Instrumentation/InstrProfiling.h" +// #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" +// #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +// #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" +// #include "llvm/Transforms/Utils/CanonicalizeAliases.h" +// #include "llvm/Transforms/Utils/NameAnonGlobals.h" +// #include "llvm/Transforms/Utils.h" + +// using namespace llvm; + +// typedef struct LLVMOpaquePass *LLVMPassRef; +// typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; + +// DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) +// DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + +// extern "C" void LLVMInitializePasses() { +// PassRegistry &Registry = *PassRegistry::getPassRegistry(); +// initializeCore(Registry); +// initializeCodeGen(Registry); +// initializeScalarOpts(Registry); +// initializeVectorization(Registry); +// initializeIPO(Registry); +// initializeAnalysis(Registry); +// initializeTransformUtils(Registry); +// initializeInstCombine(Registry); +// initializeInstrumentation(Registry); +// initializeTarget(Registry); +// } + +// extern "C" void LLVMTimeTraceProfilerInitialize() { +// timeTraceProfilerInitialize( +// /* TimeTraceGranularity */ 0, +// /* ProcName */ "rustc"); +// } + +// extern "C" void LLVMTimeTraceProfilerFinishThread() { +// timeTraceProfilerFinishThread(); +// } + +// extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { +// StringRef FN(FileName); +// std::error_code EC; +// raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); + +// timeTraceProfilerWrite(OS); +// timeTraceProfilerCleanup(); +// } + +// extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { +// #if LLVM_VERSION_LT(15, 0) +// StringRef SR(PassName); +// PassRegistry *PR = PassRegistry::getPassRegistry(); + +// const PassInfo *PI = PR->getPassInfo(SR); +// if (PI) { +// return wrap(PI->createPass()); +// } +// return nullptr; +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) { +// #if LLVM_VERSION_LT(15, 0) +// const bool CompileKernel = false; +// const bool UseAfterScope = true; + +// return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope)); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) { +// #if LLVM_VERSION_LT(15, 0) +// const bool CompileKernel = false; + +// return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover)); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) { +// #if LLVM_VERSION_LT(15, 0) +// const bool CompileKernel = false; + +// return wrap(createMemorySanitizerLegacyPassPass( +// MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel})); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() { +// #if LLVM_VERSION_LT(15, 0) +// return wrap(createThreadSanitizerLegacyPassPass()); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) { +// #if LLVM_VERSION_LT(15, 0) +// const bool CompileKernel = false; + +// return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover)); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { +// #if LLVM_VERSION_LT(15, 0) +// assert(RustPass); +// Pass *Pass = unwrap(RustPass); +// PassManagerBase *PMB = unwrap(PMR); +// PMB->add(Pass); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() { +// #if LLVM_VERSION_LT(15, 0) +// return LLVMPassManagerBuilderCreate(); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) { +// #if LLVM_VERSION_LT(15, 0) +// LLVMPassManagerBuilderDispose(PMB); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager( +// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { +// #if LLVM_VERSION_LT(15, 0) +// LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager( +// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { +// #if LLVM_VERSION_LT(15, 0) +// LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager( +// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) { +// #if LLVM_VERSION_LT(15, 0) +// LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" +// void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( +// LLVMPassManagerBuilderRef PMBR, +// LLVMPassManagerRef PMR +// ) { +// #if LLVM_VERSION_LT(15, 0) +// unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold( +// LLVMPassManagerBuilderRef PMB, unsigned Threshold) { +// #if LLVM_VERSION_LT(15, 0) +// LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" +// void LLVMRustAddLastExtensionPasses( +// LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { +// #if LLVM_VERSION_LT(15, 0) +// auto AddExtensionPasses = [Passes, NumPasses]( +// const PassManagerBuilder &Builder, PassManagerBase &PM) { +// for (size_t I = 0; I < NumPasses; I++) { +// PM.add(unwrap(Passes[I])); +// } +// }; +// // Add the passes to both of the pre-finalization extension points, +// // so they are run for optimized and non-optimized builds. +// unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, +// AddExtensionPasses); +// unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, +// AddExtensionPasses); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// #ifdef LLVM_COMPONENT_X86 +// #define SUBTARGET_X86 SUBTARGET(X86) +// #else +// #define SUBTARGET_X86 +// #endif + +// #ifdef LLVM_COMPONENT_ARM +// #define SUBTARGET_ARM SUBTARGET(ARM) +// #else +// #define SUBTARGET_ARM +// #endif + +// #ifdef LLVM_COMPONENT_AARCH64 +// #define SUBTARGET_AARCH64 SUBTARGET(AArch64) +// #else +// #define SUBTARGET_AARCH64 +// #endif + +// #ifdef LLVM_COMPONENT_AVR +// #define SUBTARGET_AVR SUBTARGET(AVR) +// #else +// #define SUBTARGET_AVR +// #endif + +// #ifdef LLVM_COMPONENT_M68k +// #define SUBTARGET_M68K SUBTARGET(M68k) +// #else +// #define SUBTARGET_M68K +// #endif + +// #ifdef LLVM_COMPONENT_MIPS +// #define SUBTARGET_MIPS SUBTARGET(Mips) +// #else +// #define SUBTARGET_MIPS +// #endif + +// #ifdef LLVM_COMPONENT_POWERPC +// #define SUBTARGET_PPC SUBTARGET(PPC) +// #else +// #define SUBTARGET_PPC +// #endif + +// #ifdef LLVM_COMPONENT_SYSTEMZ +// #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) +// #else +// #define SUBTARGET_SYSTEMZ +// #endif + +// #ifdef LLVM_COMPONENT_MSP430 +// #define SUBTARGET_MSP430 SUBTARGET(MSP430) +// #else +// #define SUBTARGET_MSP430 +// #endif + +// #ifdef LLVM_COMPONENT_RISCV +// #define SUBTARGET_RISCV SUBTARGET(RISCV) +// #else +// #define SUBTARGET_RISCV +// #endif + +// #ifdef LLVM_COMPONENT_SPARC +// #define SUBTARGET_SPARC SUBTARGET(Sparc) +// #else +// #define SUBTARGET_SPARC +// #endif + +// #ifdef LLVM_COMPONENT_HEXAGON +// #define SUBTARGET_HEXAGON SUBTARGET(Hexagon) +// #else +// #define SUBTARGET_HEXAGON +// #endif + +// #define GEN_SUBTARGETS \ +// SUBTARGET_X86 \ +// SUBTARGET_ARM \ +// SUBTARGET_AARCH64 \ +// SUBTARGET_AVR \ +// SUBTARGET_M68K \ +// SUBTARGET_MIPS \ +// SUBTARGET_PPC \ +// SUBTARGET_SYSTEMZ \ +// SUBTARGET_MSP430 \ +// SUBTARGET_SPARC \ +// SUBTARGET_HEXAGON \ +// SUBTARGET_RISCV \ + +// #define SUBTARGET(x) \ +// namespace llvm { \ +// extern const SubtargetFeatureKV x##FeatureKV[]; \ +// extern const SubtargetFeatureKV x##SubTypeKV[]; \ +// } + +// GEN_SUBTARGETS +// #undef SUBTARGET + +// extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, +// const char *Feature) { +// TargetMachine *Target = unwrap(TM); +// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); +// return MCInfo->checkFeatures(std::string("+") + Feature); +// } + +// enum class LLVMRustCodeModel { +// Tiny, +// Small, +// Kernel, +// Medium, +// Large, +// None, +// }; + +// static Optional fromRust(LLVMRustCodeModel Model) { +// switch (Model) { +// case LLVMRustCodeModel::Tiny: +// return CodeModel::Tiny; +// case LLVMRustCodeModel::Small: +// return CodeModel::Small; +// case LLVMRustCodeModel::Kernel: +// return CodeModel::Kernel; +// case LLVMRustCodeModel::Medium: +// return CodeModel::Medium; +// case LLVMRustCodeModel::Large: +// return CodeModel::Large; +// case LLVMRustCodeModel::None: +// return None; +// default: +// report_fatal_error("Bad CodeModel."); +// } +// } + +// enum class LLVMRustCodeGenOptLevel { +// None, +// Less, +// Default, +// Aggressive, +// }; + +// static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { +// switch (Level) { +// case LLVMRustCodeGenOptLevel::None: +// return CodeGenOpt::None; +// case LLVMRustCodeGenOptLevel::Less: +// return CodeGenOpt::Less; +// case LLVMRustCodeGenOptLevel::Default: +// return CodeGenOpt::Default; +// case LLVMRustCodeGenOptLevel::Aggressive: +// return CodeGenOpt::Aggressive; +// default: +// report_fatal_error("Bad CodeGenOptLevel."); +// } +// } + +// enum class LLVMRustPassBuilderOptLevel { +// O0, +// O1, +// O2, +// O3, +// Os, +// Oz, +// }; + +// #if LLVM_VERSION_LT(14,0) +// using OptimizationLevel = PassBuilder::OptimizationLevel; +// #endif + +// static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { +// switch (Level) { +// case LLVMRustPassBuilderOptLevel::O0: +// return OptimizationLevel::O0; +// case LLVMRustPassBuilderOptLevel::O1: +// return OptimizationLevel::O1; +// case LLVMRustPassBuilderOptLevel::O2: +// return OptimizationLevel::O2; +// case LLVMRustPassBuilderOptLevel::O3: +// return OptimizationLevel::O3; +// case LLVMRustPassBuilderOptLevel::Os: +// return OptimizationLevel::Os; +// case LLVMRustPassBuilderOptLevel::Oz: +// return OptimizationLevel::Oz; +// default: +// report_fatal_error("Bad PassBuilderOptLevel."); +// } +// } + +// enum class LLVMRustRelocModel { +// Static, +// PIC, +// DynamicNoPic, +// ROPI, +// RWPI, +// ROPIRWPI, +// }; + +// static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { +// switch (RustReloc) { +// case LLVMRustRelocModel::Static: +// return Reloc::Static; +// case LLVMRustRelocModel::PIC: +// return Reloc::PIC_; +// case LLVMRustRelocModel::DynamicNoPic: +// return Reloc::DynamicNoPIC; +// case LLVMRustRelocModel::ROPI: +// return Reloc::ROPI; +// case LLVMRustRelocModel::RWPI: +// return Reloc::RWPI; +// case LLVMRustRelocModel::ROPIRWPI: +// return Reloc::ROPI_RWPI; +// } +// report_fatal_error("Bad RelocModel."); +// } + +// #ifdef LLVM_RUSTLLVM +// /// getLongestEntryLength - Return the length of the longest entry in the table. +// template +// static size_t getLongestEntryLength(ArrayRef Table) { +// size_t MaxLen = 0; +// for (auto &I : Table) +// MaxLen = std::max(MaxLen, std::strlen(I.Key)); +// return MaxLen; +// } + +// extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { +// const TargetMachine *Target = unwrap(TM); +// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); +// const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); +// const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); +// const ArrayRef CPUTable = MCInfo->getCPUTable(); +// unsigned MaxCPULen = getLongestEntryLength(CPUTable); + +// printf("Available CPUs for this target:\n"); +// if (HostArch == TargetArch) { +// const StringRef HostCPU = sys::getHostCPUName(); +// printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", +// MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); +// } +// for (auto &CPU : CPUTable) +// printf(" %-*s\n", MaxCPULen, CPU.Key); +// printf("\n"); +// } + +// extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { +// const TargetMachine *Target = unwrap(TM); +// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); +// const ArrayRef FeatTable = MCInfo->getFeatureTable(); +// return FeatTable.size(); +// } + +// extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, +// const char** Feature, const char** Desc) { +// const TargetMachine *Target = unwrap(TM); +// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); +// const ArrayRef FeatTable = MCInfo->getFeatureTable(); +// const SubtargetFeatureKV Feat = FeatTable[Index]; +// *Feature = Feat.Key; +// *Desc = Feat.Desc; +// } + +// #else + +// extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { +// printf("Target CPU help is not supported by this LLVM version.\n\n"); +// } + +// extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) { +// return 0; +// } + +// extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {} +// #endif + +// extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { +// StringRef Name = sys::getHostCPUName(); +// *len = Name.size(); +// return Name.data(); +// } + +// extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( +// const char *TripleStr, const char *CPU, const char *Feature, +// const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, +// LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, +// bool FunctionSections, +// bool DataSections, +// bool UniqueSectionNames, +// bool TrapUnreachable, +// bool Singlethread, +// bool AsmComments, +// bool EmitStackSizeSection, +// bool RelaxELFRelocations, +// bool UseInitArray, +// const char *SplitDwarfFile) { + +// auto OptLevel = fromRust(RustOptLevel); +// auto RM = fromRust(RustReloc); +// auto CM = fromRust(RustCM); + +// std::string Error; +// Triple Trip(Triple::normalize(TripleStr)); +// const llvm::Target *TheTarget = +// TargetRegistry::lookupTarget(Trip.getTriple(), Error); +// if (TheTarget == nullptr) { +// LLVMRustSetLastError(Error.c_str()); +// return nullptr; +// } + +// TargetOptions Options; + +// Options.FloatABIType = FloatABI::Default; +// if (UseSoftFloat) { +// Options.FloatABIType = FloatABI::Soft; +// } +// Options.DataSections = DataSections; +// Options.FunctionSections = FunctionSections; +// Options.UniqueSectionNames = UniqueSectionNames; +// Options.MCOptions.AsmVerbose = AsmComments; +// Options.MCOptions.PreserveAsmComments = AsmComments; +// Options.MCOptions.ABIName = ABIStr; +// if (SplitDwarfFile) { +// Options.MCOptions.SplitDwarfFile = SplitDwarfFile; +// } +// Options.RelaxELFRelocations = RelaxELFRelocations; +// Options.UseInitArray = UseInitArray; + +// if (TrapUnreachable) { +// // Tell LLVM to codegen `unreachable` into an explicit trap instruction. +// // This limits the extent of possible undefined behavior in some cases, as +// // it prevents control flow from "falling through" into whatever code +// // happens to be laid out next in memory. +// Options.TrapUnreachable = true; +// } + +// if (Singlethread) { +// Options.ThreadModel = ThreadModel::Single; +// } + +// Options.EmitStackSizeSection = EmitStackSizeSection; + +// TargetMachine *TM = TheTarget->createTargetMachine( +// Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); +// return wrap(TM); +// } + +// extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { +// delete unwrap(TM); +// } + +// extern "C" void LLVMRustConfigurePassManagerBuilder( +// LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, +// bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, +// const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath, +// int SizeLevel) { +// #if LLVM_VERSION_LT(15, 0) +// unwrap(PMBR)->MergeFunctions = MergeFunctions; +// unwrap(PMBR)->SLPVectorize = SLPVectorize; +// unwrap(PMBR)->OptLevel = fromRust(OptLevel); +// unwrap(PMBR)->LoopVectorize = LoopVectorize; +// unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; +// unwrap(PMBR)->SizeLevel = SizeLevel; +// unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0; + +// if (PGOGenPath) { +// assert(!PGOUsePath && !PGOSampleUsePath); +// unwrap(PMBR)->EnablePGOInstrGen = true; +// unwrap(PMBR)->PGOInstrGen = PGOGenPath; +// } else if (PGOUsePath) { +// assert(!PGOSampleUsePath); +// unwrap(PMBR)->PGOInstrUse = PGOUsePath; +// } else if (PGOSampleUsePath) { +// unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath; +// } +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` +// // field of a PassManagerBuilder, we expose our own method of doing so. +// extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, +// LLVMModuleRef M, +// bool DisableSimplifyLibCalls) { +// Triple TargetTriple(unwrap(M)->getTargetTriple()); +// TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); +// if (DisableSimplifyLibCalls) +// TLI->disableAllFunctions(); +// unwrap(PMBR)->LibraryInfo = TLI; +// } + +// // Unfortunately, the LLVM C API doesn't provide a way to create the +// // TargetLibraryInfo pass, so we use this method to do so. +// extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, +// bool DisableSimplifyLibCalls) { +// Triple TargetTriple(unwrap(M)->getTargetTriple()); +// TargetLibraryInfoImpl TLII(TargetTriple); +// if (DisableSimplifyLibCalls) +// TLII.disableAllFunctions(); +// unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); +// } + +// // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over +// // all the functions in a module, so we do that manually here. You'll find +// // similar code in clang's BackendUtil.cpp file. +// extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, +// LLVMModuleRef M) { +// llvm::legacy::FunctionPassManager *P = +// unwrap(PMR); +// P->doInitialization(); + +// // Upgrade all calls to old intrinsics first. +// for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) +// UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove + +// for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; +// ++I) +// if (!I->isDeclaration()) +// P->run(*I); + +// P->doFinalization(); +// } + +// extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { +// // Initializing the command-line options more than once is not allowed. So, +// // check if they've already been initialized. (This could happen if we're +// // being called from rustpkg, for example). If the arguments change, then +// // that's just kinda unfortunate. +// static bool Initialized = false; +// if (Initialized) +// return; +// Initialized = true; +// cl::ParseCommandLineOptions(Argc, Argv); +// } + +// enum class LLVMRustFileType { +// AssemblyFile, +// ObjectFile, +// }; + +// static CodeGenFileType fromRust(LLVMRustFileType Type) { +// switch (Type) { +// case LLVMRustFileType::AssemblyFile: +// return CGFT_AssemblyFile; +// case LLVMRustFileType::ObjectFile: +// return CGFT_ObjectFile; +// default: +// report_fatal_error("Bad FileType."); +// } +// } + +// extern "C" LLVMRustResult +// LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, +// LLVMModuleRef M, const char *Path, const char *DwoPath, +// LLVMRustFileType RustFileType) { +// llvm::legacy::PassManager *PM = unwrap(PMR); +// auto FileType = fromRust(RustFileType); + +// std::string ErrorInfo; +// std::error_code EC; +// raw_fd_ostream OS(Path, EC, sys::fs::OF_None); +// if (EC) +// ErrorInfo = EC.message(); +// if (ErrorInfo != "") { +// LLVMRustSetLastError(ErrorInfo.c_str()); +// return LLVMRustResult::Failure; +// } + +// buffer_ostream BOS(OS); +// if (DwoPath) { +// raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None); +// EC.clear(); +// if (EC) +// ErrorInfo = EC.message(); +// if (ErrorInfo != "") { +// LLVMRustSetLastError(ErrorInfo.c_str()); +// return LLVMRustResult::Failure; +// } +// buffer_ostream DBOS(DOS); +// unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); +// PM->run(*unwrap(M)); +// } else { +// unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); +// PM->run(*unwrap(M)); +// } + +// // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output +// // stream (OS), so the only real safe place to delete this is here? Don't we +// // wish this was written in Rust? +// LLVMDisposePassManager(PMR); +// return LLVMRustResult::Success; +// } + +// extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler +// const char*, // pass name +// const char*); // IR name +// extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler + +// std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { +// if (any_isa(WrappedIr)) +// return any_cast(WrappedIr)->getName().str(); +// if (any_isa(WrappedIr)) +// return any_cast(WrappedIr)->getName().str(); +// if (any_isa(WrappedIr)) +// return any_cast(WrappedIr)->getName().str(); +// if (any_isa(WrappedIr)) +// return any_cast(WrappedIr)->getName(); +// return ""; +// } + + +// void LLVMSelfProfileInitializeCallbacks( +// PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, +// LLVMRustSelfProfileBeforePassCallback BeforePassCallback, +// LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { +// PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( +// StringRef Pass, llvm::Any Ir) { +// std::string PassName = Pass.str(); +// std::string IrName = LLVMRustwrappedIrGetName(Ir); +// BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); +// }); + +// PIC.registerAfterPassCallback( +// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, +// const PreservedAnalyses &Preserved) { +// AfterPassCallback(LlvmSelfProfiler); +// }); + +// PIC.registerAfterPassInvalidatedCallback( +// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { +// AfterPassCallback(LlvmSelfProfiler); +// }); + +// PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( +// StringRef Pass, llvm::Any Ir) { +// std::string PassName = Pass.str(); +// std::string IrName = LLVMRustwrappedIrGetName(Ir); +// BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); +// }); + +// PIC.registerAfterAnalysisCallback( +// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { +// AfterPassCallback(LlvmSelfProfiler); +// }); +// } + +// enum class LLVMRustOptStage { +// PreLinkNoLTO, +// PreLinkThinLTO, +// PreLinkFatLTO, +// ThinLTO, +// FatLTO, +// }; + +// struct LLVMRustSanitizerOptions { +// bool SanitizeAddress; +// bool SanitizeAddressRecover; +// bool SanitizeMemory; +// bool SanitizeMemoryRecover; +// int SanitizeMemoryTrackOrigins; +// bool SanitizeThread; +// bool SanitizeHWAddress; +// bool SanitizeHWAddressRecover; +// }; + +// extern "C" LLVMRustResult +// LLVMRustOptimizeWithNewPassManager( +// LLVMModuleRef ModuleRef, +// LLVMTargetMachineRef TMRef, +// LLVMRustPassBuilderOptLevel OptLevelRust, +// LLVMRustOptStage OptStage, +// bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, +// bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, +// bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, +// LLVMRustSanitizerOptions *SanitizerOptions, +// const char *PGOGenPath, const char *PGOUsePath, +// bool InstrumentCoverage, bool InstrumentGCOV, +// const char *PGOSampleUsePath, bool DebugInfoForProfiling, +// void* LlvmSelfProfiler, +// LLVMRustSelfProfileBeforePassCallback BeforePassCallback, +// LLVMRustSelfProfileAfterPassCallback AfterPassCallback, +// const char *ExtraPasses, size_t ExtraPassesLen, +// const char *LLVMPlugins, size_t LLVMPluginsLen) { +// Module *TheModule = unwrap(ModuleRef); +// TargetMachine *TM = unwrap(TMRef); +// OptimizationLevel OptLevel = fromRust(OptLevelRust); + + +// PipelineTuningOptions PTO; +// PTO.LoopUnrolling = UnrollLoops; +// PTO.LoopInterleaving = UnrollLoops; +// PTO.LoopVectorization = LoopVectorize; +// PTO.SLPVectorization = SLPVectorize; +// PTO.MergeFunctions = MergeFunctions; + +// // FIXME: We may want to expose this as an option. +// bool DebugPassManager = false; + +// PassInstrumentationCallbacks PIC; +// StandardInstrumentations SI(DebugPassManager); +// SI.registerCallbacks(PIC); + +// if (LlvmSelfProfiler){ +// LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); +// } + +// Optional PGOOpt; +// if (PGOGenPath) { +// assert(!PGOUsePath && !PGOSampleUsePath); +// PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr, +// PGOOptions::NoCSAction, DebugInfoForProfiling); +// } else if (PGOUsePath) { +// assert(!PGOSampleUsePath); +// PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse, +// PGOOptions::NoCSAction, DebugInfoForProfiling); +// } else if (PGOSampleUsePath) { +// PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse, +// PGOOptions::NoCSAction, DebugInfoForProfiling); +// } else if (DebugInfoForProfiling) { +// PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction, +// PGOOptions::NoCSAction, DebugInfoForProfiling); +// } + +// #if LLVM_VERSION_GE(13, 0) +// PassBuilder PB(TM, PTO, PGOOpt, &PIC); +// LoopAnalysisManager LAM; +// FunctionAnalysisManager FAM; +// CGSCCAnalysisManager CGAM; +// ModuleAnalysisManager MAM; +// #else +// PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC); +// LoopAnalysisManager LAM(DebugPassManager); +// FunctionAnalysisManager FAM(DebugPassManager); +// CGSCCAnalysisManager CGAM(DebugPassManager); +// ModuleAnalysisManager MAM(DebugPassManager); +// #endif + +// FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); + +// Triple TargetTriple(TheModule->getTargetTriple()); +// std::unique_ptr TLII(new TargetLibraryInfoImpl(TargetTriple)); +// if (DisableSimplifyLibCalls) +// TLII->disableAllFunctions(); +// FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); + +// PB.registerModuleAnalyses(MAM); +// PB.registerCGSCCAnalyses(CGAM); +// PB.registerFunctionAnalyses(FAM); +// PB.registerLoopAnalyses(LAM); +// PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + +// // We manually collect pipeline callbacks so we can apply them at O0, where the +// // PassBuilder does not create a pipeline. +// std::vector> +// PipelineStartEPCallbacks; +// std::vector> +// OptimizerLastEPCallbacks; + +// if (VerifyIR) { +// PipelineStartEPCallbacks.push_back( +// [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { +// MPM.addPass(VerifierPass()); +// } +// ); +// } + +// if (InstrumentGCOV) { +// PipelineStartEPCallbacks.push_back( +// [](ModulePassManager &MPM, OptimizationLevel Level) { +// MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); +// } +// ); +// } + +// if (InstrumentCoverage) { +// PipelineStartEPCallbacks.push_back( +// [](ModulePassManager &MPM, OptimizationLevel Level) { +// InstrProfOptions Options; +// MPM.addPass(InstrProfiling(Options, false)); +// } +// ); +// } + +// if (SanitizerOptions) { +// if (SanitizerOptions->SanitizeMemory) { +// MemorySanitizerOptions Options( +// SanitizerOptions->SanitizeMemoryTrackOrigins, +// SanitizerOptions->SanitizeMemoryRecover, +// /*CompileKernel=*/false); +// OptimizerLastEPCallbacks.push_back( +// [Options](ModulePassManager &MPM, OptimizationLevel Level) { +// #if LLVM_VERSION_GE(14, 0) +// MPM.addPass(ModuleMemorySanitizerPass(Options)); +// #else +// MPM.addPass(MemorySanitizerPass(Options)); +// #endif +// MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); +// } +// ); +// } + +// if (SanitizerOptions->SanitizeThread) { +// OptimizerLastEPCallbacks.push_back( +// [](ModulePassManager &MPM, OptimizationLevel Level) { +// #if LLVM_VERSION_GE(14, 0) +// MPM.addPass(ModuleThreadSanitizerPass()); +// #else +// MPM.addPass(ThreadSanitizerPass()); +// #endif +// MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); +// } +// ); +// } + +// if (SanitizerOptions->SanitizeAddress) { +// OptimizerLastEPCallbacks.push_back( +// [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +// #if LLVM_VERSION_LT(15, 0) +// MPM.addPass(RequireAnalysisPass()); +// #endif +// #if LLVM_VERSION_GE(14, 0) +// AddressSanitizerOptions opts = AddressSanitizerOptions{ +// /*CompileKernel=*/false, +// SanitizerOptions->SanitizeAddressRecover, +// /*UseAfterScope=*/true, +// AsanDetectStackUseAfterReturnMode::Runtime, +// }; +// MPM.addPass(ModuleAddressSanitizerPass(opts)); +// #else +// MPM.addPass(ModuleAddressSanitizerPass( +// /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover)); +// MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( +// /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover, +// /*UseAfterScope=*/true))); +// #endif +// } +// ); +// } +// if (SanitizerOptions->SanitizeHWAddress) { +// OptimizerLastEPCallbacks.push_back( +// [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +// #if LLVM_VERSION_GE(14, 0) +// HWAddressSanitizerOptions opts( +// /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, +// /*DisableOptimization=*/false); +// MPM.addPass(HWAddressSanitizerPass(opts)); +// #else +// MPM.addPass(HWAddressSanitizerPass( +// /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover)); +// #endif +// } +// ); +// } +// } + +// if (LLVMPluginsLen) { +// auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); +// SmallVector Plugins; +// PluginsStr.split(Plugins, ',', -1, false); +// for (auto PluginPath: Plugins) { +// auto Plugin = PassPlugin::Load(PluginPath.str()); +// if (!Plugin) { +// LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); +// continue; +// } +// Plugin->registerPassBuilderCallbacks(PB); +// } +// } + +// #if LLVM_VERSION_GE(13, 0) +// ModulePassManager MPM; +// #else +// ModulePassManager MPM(DebugPassManager); +// #endif +// bool NeedThinLTOBufferPasses = UseThinLTOBuffers; +// if (!NoPrepopulatePasses) { +// // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead. +// // At the same time, the LTO pipelines do support O0 and using them is required. +// bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; +// if (OptLevel == OptimizationLevel::O0 && !IsLTO) { +// for (const auto &C : PipelineStartEPCallbacks) +// PB.registerPipelineStartEPCallback(C); +// for (const auto &C : OptimizerLastEPCallbacks) +// PB.registerOptimizerLastEPCallback(C); + +// // Pass false as we manually schedule ThinLTOBufferPasses below. +// MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); +// } else { +// for (const auto &C : PipelineStartEPCallbacks) +// PB.registerPipelineStartEPCallback(C); +// if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { +// for (const auto &C : OptimizerLastEPCallbacks) +// PB.registerOptimizerLastEPCallback(C); +// } + +// switch (OptStage) { +// case LLVMRustOptStage::PreLinkNoLTO: +// MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); +// break; +// case LLVMRustOptStage::PreLinkThinLTO: +// MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); +// // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback +// // passes may still run afterwards. This means we need to run the buffer passes again. +// // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks +// // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks. +// if (OptimizerLastEPCallbacks.empty()) +// NeedThinLTOBufferPasses = false; +// for (const auto &C : OptimizerLastEPCallbacks) +// C(MPM, OptLevel); +// break; +// case LLVMRustOptStage::PreLinkFatLTO: +// MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); +// NeedThinLTOBufferPasses = false; +// break; +// case LLVMRustOptStage::ThinLTO: +// // FIXME: Does it make sense to pass the ModuleSummaryIndex? +// // It only seems to be needed for C++ specific optimizations. +// MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); +// break; +// case LLVMRustOptStage::FatLTO: +// MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); +// break; +// } +// } +// } else { +// // We're not building any of the default pipelines but we still want to +// // add the verifier, instrumentation, etc passes if they were requested +// for (const auto &C : PipelineStartEPCallbacks) +// C(MPM, OptLevel); +// for (const auto &C : OptimizerLastEPCallbacks) +// C(MPM, OptLevel); +// } + +// if (ExtraPassesLen) { +// if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { +// std::string ErrMsg = toString(std::move(Err)); +// LLVMRustSetLastError(ErrMsg.c_str()); +// return LLVMRustResult::Failure; +// } +// } + +// if (NeedThinLTOBufferPasses) { +// MPM.addPass(CanonicalizeAliasesPass()); +// MPM.addPass(NameAnonGlobalPass()); +// } + +// // Upgrade all calls to old intrinsics first. +// for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) +// UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove + +// MPM.run(*TheModule, MAM); +// return LLVMRustResult::Success; +// } + +// // Callback to demangle function name +// // Parameters: +// // * name to be demangled +// // * name len +// // * output buffer +// // * output buffer len +// // Returns len of demangled string, or 0 if demangle failed. +// typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); + + +// namespace { + +// class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { +// DemangleFn Demangle; +// std::vector Buf; + +// public: +// RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} + +// // Return empty string if demangle failed +// // or if name does not need to be demangled +// StringRef CallDemangle(StringRef name) { +// if (!Demangle) { +// return StringRef(); +// } + +// if (Buf.size() < name.size() * 2) { +// // Semangled name usually shorter than mangled, +// // but allocate twice as much memory just in case +// Buf.resize(name.size() * 2); +// } + +// auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); +// if (!R) { +// // Demangle failed. +// return StringRef(); +// } + +// auto Demangled = StringRef(Buf.data(), R); +// if (Demangled == name) { +// // Do not print anything if demangled name is equal to mangled. +// return StringRef(); +// } + +// return Demangled; +// } + +// void emitFunctionAnnot(const Function *F, +// formatted_raw_ostream &OS) override { +// StringRef Demangled = CallDemangle(F->getName()); +// if (Demangled.empty()) { +// return; +// } + +// OS << "; " << Demangled << "\n"; +// } + +// void emitInstructionAnnot(const Instruction *I, +// formatted_raw_ostream &OS) override { +// const char *Name; +// const Value *Value; +// if (const CallInst *CI = dyn_cast(I)) { +// Name = "call"; +// Value = CI->getCalledOperand(); +// } else if (const InvokeInst* II = dyn_cast(I)) { +// Name = "invoke"; +// Value = II->getCalledOperand(); +// } else { +// // Could demangle more operations, e. g. +// // `store %place, @function`. +// return; +// } + +// if (!Value->hasName()) { +// return; +// } + +// StringRef Demangled = CallDemangle(Value->getName()); +// if (Demangled.empty()) { +// return; +// } + +// OS << "; " << Name << " " << Demangled << "\n"; +// } +// }; + +// } // namespace + +// extern "C" LLVMRustResult +// LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { +// std::string ErrorInfo; +// std::error_code EC; +// raw_fd_ostream OS(Path, EC, sys::fs::OF_None); +// if (EC) +// ErrorInfo = EC.message(); +// if (ErrorInfo != "") { +// LLVMRustSetLastError(ErrorInfo.c_str()); +// return LLVMRustResult::Failure; +// } + +// RustAssemblyAnnotationWriter AAW(Demangle); +// formatted_raw_ostream FOS(OS); +// unwrap(M)->print(FOS, &AAW); + +// return LLVMRustResult::Success; +// } + +// extern "C" void LLVMRustPrintPasses() { +// LLVMInitializePasses(); +// struct MyListener : PassRegistrationListener { +// void passEnumerate(const PassInfo *Info) { +// StringRef PassArg = Info->getPassArgument(); +// StringRef PassName = Info->getPassName(); +// if (!PassArg.empty()) { +// // These unsigned->signed casts could theoretically overflow, but +// // realistically never will (and even if, the result is implementation +// // defined rather plain UB). +// printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), +// (int)PassName.size(), PassName.data()); +// } +// } +// } Listener; + +// PassRegistry *PR = PassRegistry::getPassRegistry(); +// PR->enumerateWith(&Listener); +// } + +// extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, +// bool AddLifetimes) { +// unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); +// } + +// extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, +// size_t Len) { +// llvm::legacy::PassManager passes; + +// auto PreserveFunctions = [=](const GlobalValue &GV) { +// for (size_t I = 0; I < Len; I++) { +// if (GV.getName() == Symbols[I]) { +// return true; +// } +// } +// return false; +// }; + +// passes.add(llvm::createInternalizePass(PreserveFunctions)); + +// passes.run(*unwrap(M)); +// } + +// extern "C" void +// LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, +// LLVMTargetMachineRef TMR) { +// TargetMachine *Target = unwrap(TMR); +// unwrap(Module)->setDataLayout(Target->createDataLayout()); +// } + +// extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { +// unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); +// } + +// extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { +// unwrap(M)->setPIELevel(PIELevel::Level::Large); +// } + +// extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, +// LLVMRustCodeModel Model) { +// auto CM = fromRust(Model); +// if (!CM.hasValue()) +// return; +// unwrap(M)->setCodeModel(*CM); +// } + +// // Here you'll find an implementation of ThinLTO as used by the Rust compiler +// // right now. This ThinLTO support is only enabled on "recent ish" versions of +// // LLVM, and otherwise it's just blanket rejected from other compilers. +// // +// // Most of this implementation is straight copied from LLVM. At the time of +// // this writing it wasn't *quite* suitable to reuse more code from upstream +// // for our purposes, but we should strive to upstream this support once it's +// // ready to go! I figure we may want a bit of testing locally first before +// // sending this upstream to LLVM. I hear though they're quite eager to receive +// // feedback like this! +// // +// // If you're reading this code and wondering "what in the world" or you're +// // working "good lord by LLVM upgrade is *still* failing due to these bindings" +// // then fear not! (ok maybe fear a little). All code here is mostly based +// // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. +// // +// // You'll find that the general layout here roughly corresponds to the `run` +// // method in that file as well as `ProcessThinLTOModule`. Functions are +// // specifically commented below as well, but if you're updating this code +// // or otherwise trying to understand it, the LLVM source will be useful in +// // interpreting the mysteries within. +// // +// // Otherwise I'll apologize in advance, it probably requires a relatively +// // significant investment on your part to "truly understand" what's going on +// // here. Not saying I do myself, but it took me awhile staring at LLVM's source +// // and various online resources about ThinLTO to make heads or tails of all +// // this. + +// // This is a shared data structure which *must* be threadsafe to share +// // read-only amongst threads. This also corresponds basically to the arguments +// // of the `ProcessThinLTOModule` function in the LLVM source. +// struct LLVMRustThinLTOData { +// // The combined index that is the global analysis over all modules we're +// // performing ThinLTO for. This is mostly managed by LLVM. +// ModuleSummaryIndex Index; + +// // All modules we may look at, stored as in-memory serialized versions. This +// // is later used when inlining to ensure we can extract any module to inline +// // from. +// StringMap ModuleMap; + +// // A set that we manage of everything we *don't* want internalized. Note that +// // this includes all transitive references right now as well, but it may not +// // always! +// DenseSet GUIDPreservedSymbols; + +// // Not 100% sure what these are, but they impact what's internalized and +// // what's inlined across modules, I believe. +// StringMap ImportLists; +// StringMap ExportLists; +// StringMap ModuleToDefinedGVSummaries; +// StringMap> ResolvedODR; + +// LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} +// }; + +// // Just an argument to the `LLVMRustCreateThinLTOData` function below. +// struct LLVMRustThinLTOModule { +// const char *identifier; +// const char *data; +// size_t len; +// }; + +// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it +// // does. +// static const GlobalValueSummary * +// getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { +// auto StrongDefForLinker = llvm::find_if( +// GVSummaryList, [](const std::unique_ptr &Summary) { +// auto Linkage = Summary->linkage(); +// return !GlobalValue::isAvailableExternallyLinkage(Linkage) && +// !GlobalValue::isWeakForLinker(Linkage); +// }); +// if (StrongDefForLinker != GVSummaryList.end()) +// return StrongDefForLinker->get(); + +// auto FirstDefForLinker = llvm::find_if( +// GVSummaryList, [](const std::unique_ptr &Summary) { +// auto Linkage = Summary->linkage(); +// return !GlobalValue::isAvailableExternallyLinkage(Linkage); +// }); +// if (FirstDefForLinker == GVSummaryList.end()) +// return nullptr; +// return FirstDefForLinker->get(); +// } + +// // The main entry point for creating the global ThinLTO analysis. The structure +// // here is basically the same as before threads are spawned in the `run` +// // function of `lib/LTO/ThinLTOCodeGenerator.cpp`. +// extern "C" LLVMRustThinLTOData* +// LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, +// int num_modules, +// const char **preserved_symbols, +// int num_symbols) { +// auto Ret = std::make_unique(); + +// // Load each module's summary and merge it into one combined index +// for (int i = 0; i < num_modules; i++) { +// auto module = &modules[i]; +// StringRef buffer(module->data, module->len); +// MemoryBufferRef mem_buffer(buffer, module->identifier); + +// Ret->ModuleMap[module->identifier] = mem_buffer; + +// if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { +// LLVMRustSetLastError(toString(std::move(Err)).c_str()); +// return nullptr; +// } +// } + +// // Collect for each module the list of function it defines (GUID -> Summary) +// Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); + +// // Convert the preserved symbols set from string to GUID, this is then needed +// // for internalization. +// for (int i = 0; i < num_symbols; i++) { +// auto GUID = GlobalValue::getGUID(preserved_symbols[i]); +// Ret->GUIDPreservedSymbols.insert(GUID); +// } + +// // Collect the import/export lists for all modules from the call-graph in the +// // combined index +// // +// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` +// auto deadIsPrevailing = [&](GlobalValue::GUID G) { +// return PrevailingType::Unknown; +// }; +// // We don't have a complete picture in our use of ThinLTO, just our immediate +// // crate, so we need `ImportEnabled = false` to limit internalization. +// // Otherwise, we sometimes lose `static` values -- see #60184. +// computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, +// deadIsPrevailing, /* ImportEnabled = */ false); +// ComputeCrossModuleImport( +// Ret->Index, +// Ret->ModuleToDefinedGVSummaries, +// Ret->ImportLists, +// Ret->ExportLists +// ); + +// // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it +// // impacts the caching. +// // +// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this +// // being lifted from `lib/LTO/LTO.cpp` as well +// DenseMap PrevailingCopy; +// for (auto &I : Ret->Index) { +// if (I.second.SummaryList.size() > 1) +// PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); +// } +// auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { +// const auto &Prevailing = PrevailingCopy.find(GUID); +// if (Prevailing == PrevailingCopy.end()) +// return true; +// return Prevailing->second == S; +// }; +// auto recordNewLinkage = [&](StringRef ModuleIdentifier, +// GlobalValue::GUID GUID, +// GlobalValue::LinkageTypes NewLinkage) { +// Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; +// }; + +// #if LLVM_VERSION_GE(13,0) +// // Uses FromPrevailing visibility scheme which works for many binary +// // formats. We probably could and should use ELF visibility scheme for many of +// // our targets, however. +// lto::Config conf; +// thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, +// Ret->GUIDPreservedSymbols); +// #else +// thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, +// Ret->GUIDPreservedSymbols); +// #endif +// // Here we calculate an `ExportedGUIDs` set for use in the `isExported` +// // callback below. This callback below will dictate the linkage for all +// // summaries in the index, and we basically just only want to ensure that dead +// // symbols are internalized. Otherwise everything that's already external +// // linkage will stay as external, and internal will stay as internal. +// std::set ExportedGUIDs; +// for (auto &List : Ret->Index) { +// for (auto &GVS: List.second.SummaryList) { +// if (GlobalValue::isLocalLinkage(GVS->linkage())) +// continue; +// auto GUID = GVS->getOriginalName(); +// if (GVS->flags().Live) +// ExportedGUIDs.insert(GUID); +// } +// } +// auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { +// const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); +// return (ExportList != Ret->ExportLists.end() && +// ExportList->second.count(VI)) || +// ExportedGUIDs.count(VI.getGUID()); +// }; +// thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); + +// return Ret.release(); +// } + +// extern "C" void +// LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { +// delete Data; +// } + +// // Below are the various passes that happen *per module* when doing ThinLTO. +// // +// // In other words, these are the functions that are all run concurrently +// // with one another, one per module. The passes here correspond to the analysis +// // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the +// // `ProcessThinLTOModule` function. Here they're split up into separate steps +// // so rustc can save off the intermediate bytecode between each step. + +// static bool +// clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { +// // When linking an ELF shared object, dso_local should be dropped. We +// // conservatively do this for -fpic. +// bool ClearDSOLocalOnDeclarations = +// TM.getTargetTriple().isOSBinFormatELF() && +// TM.getRelocationModel() != Reloc::Static && +// Mod.getPIELevel() == PIELevel::Default; +// return ClearDSOLocalOnDeclarations; +// } + +// extern "C" bool +// LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, +// LLVMTargetMachineRef TM) { +// Module &Mod = *unwrap(M); +// TargetMachine &Target = *unwrap(TM); + +// bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); +// bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); + +// if (error) { +// LLVMRustSetLastError("renameModuleForThinLTO failed"); +// return false; +// } +// return true; +// } + +// extern "C" bool +// LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { +// Module &Mod = *unwrap(M); +// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); +// #if LLVM_VERSION_GE(14, 0) +// thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); +// #else +// thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); +// #endif +// return true; +// } + +// extern "C" bool +// LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { +// Module &Mod = *unwrap(M); +// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); +// thinLTOInternalizeModule(Mod, DefinedGlobals); +// return true; +// } + +// extern "C" bool +// LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, +// LLVMTargetMachineRef TM) { +// Module &Mod = *unwrap(M); +// TargetMachine &Target = *unwrap(TM); + +// const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); +// auto Loader = [&](StringRef Identifier) { +// const auto &Memory = Data->ModuleMap.lookup(Identifier); +// auto &Context = Mod.getContext(); +// auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); + +// if (!MOrErr) +// return MOrErr; + +// // The rest of this closure is a workaround for +// // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports +// // we accidentally import wasm custom sections into different modules, +// // duplicating them by in the final output artifact. +// // +// // The issue is worked around here by manually removing the +// // `wasm.custom_sections` named metadata node from any imported module. This +// // we know isn't used by any optimization pass so there's no need for it to +// // be imported. +// // +// // Note that the metadata is currently lazily loaded, so we materialize it +// // here before looking up if there's metadata inside. The `FunctionImporter` +// // will immediately materialize metadata anyway after an import, so this +// // shouldn't be a perf hit. +// if (Error Err = (*MOrErr)->materializeMetadata()) { +// Expected> Ret(std::move(Err)); +// return Ret; +// } + +// auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); +// if (WasmCustomSections) +// WasmCustomSections->eraseFromParent(); + +// return MOrErr; +// }; +// bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); +// FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal); +// Expected Result = Importer.importFunctions(Mod, ImportList); +// if (!Result) { +// LLVMRustSetLastError(toString(Result.takeError()).c_str()); +// return false; +// } +// return true; +// } + +// // This struct and various functions are sort of a hack right now, but the +// // problem is that we've got in-memory LLVM modules after we generate and +// // optimize all codegen-units for one compilation in rustc. To be compatible +// // with the LTO support above we need to serialize the modules plus their +// // ThinLTO summary into memory. +// // +// // This structure is basically an owned version of a serialize module, with +// // a ThinLTO summary attached. +// struct LLVMRustThinLTOBuffer { +// std::string data; +// }; + +// extern "C" LLVMRustThinLTOBuffer* +// LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { +// auto Ret = std::make_unique(); +// { +// raw_string_ostream OS(Ret->data); +// { +// legacy::PassManager PM; +// if (is_thin) { +// PM.add(createWriteThinLTOBitcodePass(OS)); +// } else { +// PM.add(createBitcodeWriterPass(OS)); +// } +// PM.run(*unwrap(M)); +// } +// } +// return Ret.release(); +// } + +// extern "C" void +// LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { +// delete Buffer; +// } + +// extern "C" const void* +// LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { +// return Buffer->data.data(); +// } + +// extern "C" size_t +// LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { +// return Buffer->data.length(); +// } + +// // This is what we used to parse upstream bitcode for actual ThinLTO +// // processing. We'll call this once per module optimized through ThinLTO, and +// // it'll be called concurrently on many threads. +// extern "C" LLVMModuleRef +// LLVMRustParseBitcodeForLTO(LLVMContextRef Context, +// const char *data, +// size_t len, +// const char *identifier) { +// StringRef Data(data, len); +// MemoryBufferRef Buffer(Data, identifier); +// unwrap(Context)->enableDebugTypeODRUniquing(); +// Expected> SrcOrError = +// parseBitcodeFile(Buffer, *unwrap(Context)); +// if (!SrcOrError) { +// LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); +// return nullptr; +// } +// return wrap(std::move(*SrcOrError).release()); +// } + +// // Find the bitcode section in the object file data and return it as a slice. +// // Fail if the bitcode section is present but empty. +// // +// // On success, the return value is the pointer to the start of the slice and +// // `out_len` is filled with the (non-zero) length. On failure, the return value +// // is `nullptr` and `out_len` is set to zero. +// extern "C" const char* +// LLVMRustGetBitcodeSliceFromObjectData(const char *data, +// size_t len, +// size_t *out_len) { +// *out_len = 0; + +// StringRef Data(data, len); +// MemoryBufferRef Buffer(Data, ""); // The id is unused. + +// Expected BitcodeOrError = +// object::IRObjectFile::findBitcodeInMemBuffer(Buffer); +// if (!BitcodeOrError) { +// LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str()); +// return nullptr; +// } + +// *out_len = BitcodeOrError->getBufferSize(); +// return BitcodeOrError->getBufferStart(); +// } + +// // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See +// // the comment in `back/lto.rs` for why this exists. +// extern "C" void +// LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, +// DICompileUnit **A, +// DICompileUnit **B) { +// Module *M = unwrap(Mod); +// DICompileUnit **Cur = A; +// DICompileUnit **Next = B; +// for (DICompileUnit *CU : M->debug_compile_units()) { +// *Cur = CU; +// Cur = Next; +// Next = nullptr; +// if (Cur == nullptr) +// break; +// } +// } + +// // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See +// // the comment in `back/lto.rs` for why this exists. +// extern "C" void +// LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { +// Module *M = unwrap(Mod); + +// // If the original source module didn't have a `DICompileUnit` then try to +// // merge all the existing compile units. If there aren't actually any though +// // then there's not much for us to do so return. +// if (Unit == nullptr) { +// for (DICompileUnit *CU : M->debug_compile_units()) { +// Unit = CU; +// break; +// } +// if (Unit == nullptr) +// return; +// } + +// // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and +// // process it recursively. Note that we used to specifically iterate over +// // instructions to ensure we feed everything into it, but `processModule` +// // started doing this the same way in LLVM 7 (commit d769eb36ab2b8). +// DebugInfoFinder Finder; +// Finder.processModule(*M); + +// // After we've found all our debuginfo, rewrite all subprograms to point to +// // the same `DICompileUnit`. +// for (auto &F : Finder.subprograms()) { +// F->replaceUnit(Unit); +// } + +// // Erase any other references to other `DICompileUnit` instances, the verifier +// // will later ensure that we don't actually have any other stale references to +// // worry about. +// auto *MD = M->getNamedMetadata("llvm.dbg.cu"); +// MD->clearOperands(); +// MD->addOperand(Unit); +// } + +// // Computes the LTO cache key for the provided 'ModId' in the given 'Data', +// // storing the result in 'KeyOut'. +// // Currently, this cache key is a SHA-1 hash of anything that could affect +// // the result of optimizing this module (e.g. module imports, exports, liveness +// // of access globals, etc). +// // The precise details are determined by LLVM in `computeLTOCacheKey`, which is +// // used during the normal linker-plugin incremental thin-LTO process. +// extern "C" void +// LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) { +// SmallString<40> Key; +// llvm::lto::Config conf; +// const auto &ImportList = Data->ImportLists.lookup(ModId); +// const auto &ExportList = Data->ExportLists.lookup(ModId); +// const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); +// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); +// std::set CfiFunctionDefs; +// std::set CfiFunctionDecls; + +// // Based on the 'InProcessThinBackend' constructor in LLVM +// for (auto &Name : Data->Index.cfiFunctionDefs()) +// CfiFunctionDefs.insert( +// GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); +// for (auto &Name : Data->Index.cfiFunctionDecls()) +// CfiFunctionDecls.insert( +// GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); + +// llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, +// ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls +// ); + +// LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); +// } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 5f5b5de790e..56886275a7b 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1,1972 +1,1972 @@ -#include "LLVMWrapper.h" -#include "llvm/IR/DebugInfoMetadata.h" -#include "llvm/IR/DiagnosticHandler.h" -#include "llvm/IR/DiagnosticInfo.h" -#include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Intrinsics.h" -#include "llvm/IR/IntrinsicsARM.h" -#include "llvm/IR/Mangler.h" -#include "llvm/Object/Archive.h" -#include "llvm/Object/COFFImportFile.h" -#include "llvm/Object/ObjectFile.h" -#include "llvm/Pass.h" -#include "llvm/Bitcode/BitcodeWriterPass.h" -#include "llvm/Support/Signals.h" -#include "llvm/ADT/Optional.h" - -#include - -//===----------------------------------------------------------------------=== -// -// This file defines alternate interfaces to core functions that are more -// readily callable by Rust's FFI. -// -//===----------------------------------------------------------------------=== - -using namespace llvm; -using namespace llvm::sys; -using namespace llvm::object; - -// LLVMAtomicOrdering is already an enum - don't create another -// one. -static AtomicOrdering fromRust(LLVMAtomicOrdering Ordering) { - switch (Ordering) { - case LLVMAtomicOrderingNotAtomic: - return AtomicOrdering::NotAtomic; - case LLVMAtomicOrderingUnordered: - return AtomicOrdering::Unordered; - case LLVMAtomicOrderingMonotonic: - return AtomicOrdering::Monotonic; - case LLVMAtomicOrderingAcquire: - return AtomicOrdering::Acquire; - case LLVMAtomicOrderingRelease: - return AtomicOrdering::Release; - case LLVMAtomicOrderingAcquireRelease: - return AtomicOrdering::AcquireRelease; - case LLVMAtomicOrderingSequentiallyConsistent: - return AtomicOrdering::SequentiallyConsistent; - } - - report_fatal_error("Invalid LLVMAtomicOrdering value!"); -} - -static LLVM_THREAD_LOCAL char *LastError; - -// Custom error handler for fatal LLVM errors. -// -// Notably it exits the process with code 101, unlike LLVM's default of 1. -static void FatalErrorHandler(void *UserData, -#if LLVM_VERSION_LT(14, 0) - const std::string& Reason, -#else - const char* Reason, -#endif - bool GenCrashDiag) { - // Do the same thing that the default error handler does. - std::cerr << "LLVM ERROR: " << Reason << std::endl; - - // Since this error handler exits the process, we have to run any cleanup that - // LLVM would run after handling the error. This might change with an LLVM - // upgrade. - sys::RunInterruptHandlers(); - - exit(101); -} - -extern "C" void LLVMRustInstallFatalErrorHandler() { - install_fatal_error_handler(FatalErrorHandler); -} - -extern "C" void LLVMRustDisableSystemDialogsOnCrash() { - sys::DisableSystemDialogsOnCrash(); -} - -extern "C" char *LLVMRustGetLastError(void) { - char *Ret = LastError; - LastError = nullptr; - return Ret; -} - -extern "C" void LLVMRustSetLastError(const char *Err) { - free((void *)LastError); - LastError = strdup(Err); -} - -extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames) { - auto ctx = new LLVMContext(); - ctx->setDiscardValueNames(shouldDiscardNames); - return wrap(ctx); -} - -extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, - const char *Triple) { - unwrap(M)->setTargetTriple(Triple::normalize(Triple)); -} - -extern "C" void LLVMRustPrintPassTimings() { - raw_fd_ostream OS(2, false); // stderr. - TimerGroup::printAll(OS); -} - -extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, - size_t NameLen) { - return wrap(unwrap(M)->getNamedValue(StringRef(Name, NameLen))); -} - -extern "C" LLVMValueRef LLVMRustGetOrInsertFunction(LLVMModuleRef M, - const char *Name, - size_t NameLen, - LLVMTypeRef FunctionTy) { - return wrap(unwrap(M) - ->getOrInsertFunction(StringRef(Name, NameLen), - unwrap(FunctionTy)) - .getCallee() - ); -} - -extern "C" LLVMValueRef -LLVMRustGetOrInsertGlobal(LLVMModuleRef M, const char *Name, size_t NameLen, LLVMTypeRef Ty) { - Module *Mod = unwrap(M); - StringRef NameRef(Name, NameLen); - - // We don't use Module::getOrInsertGlobal because that returns a Constant*, - // which may either be the real GlobalVariable*, or a constant bitcast of it - // if our type doesn't match the original declaration. We always want the - // GlobalVariable* so we can access linkage, visibility, etc. - GlobalVariable *GV = Mod->getGlobalVariable(NameRef, true); - if (!GV) - GV = new GlobalVariable(*Mod, unwrap(Ty), false, - GlobalValue::ExternalLinkage, nullptr, NameRef); - return wrap(GV); -} - -extern "C" LLVMValueRef -LLVMRustInsertPrivateGlobal(LLVMModuleRef M, LLVMTypeRef Ty) { - return wrap(new GlobalVariable(*unwrap(M), - unwrap(Ty), - false, - GlobalValue::PrivateLinkage, - nullptr)); -} - -extern "C" LLVMTypeRef LLVMRustMetadataTypeInContext(LLVMContextRef C) { - return wrap(Type::getMetadataTy(*unwrap(C))); -} - -static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { - switch (Kind) { - case AlwaysInline: - return Attribute::AlwaysInline; - case ByVal: - return Attribute::ByVal; - case Cold: - return Attribute::Cold; - case InlineHint: - return Attribute::InlineHint; - case MinSize: - return Attribute::MinSize; - case Naked: - return Attribute::Naked; - case NoAlias: - return Attribute::NoAlias; - case NoCapture: - return Attribute::NoCapture; - case NoCfCheck: - return Attribute::NoCfCheck; - case NoInline: - return Attribute::NoInline; - case NonNull: - return Attribute::NonNull; - case NoRedZone: - return Attribute::NoRedZone; - case NoReturn: - return Attribute::NoReturn; - case NoUnwind: - return Attribute::NoUnwind; - case OptimizeForSize: - return Attribute::OptimizeForSize; - case ReadOnly: - return Attribute::ReadOnly; - case SExt: - return Attribute::SExt; - case StructRet: - return Attribute::StructRet; - case UWTable: - return Attribute::UWTable; - case ZExt: - return Attribute::ZExt; - case InReg: - return Attribute::InReg; - case SanitizeThread: - return Attribute::SanitizeThread; - case SanitizeAddress: - return Attribute::SanitizeAddress; - case SanitizeMemory: - return Attribute::SanitizeMemory; - case NonLazyBind: - return Attribute::NonLazyBind; - case OptimizeNone: - return Attribute::OptimizeNone; - case ReturnsTwice: - return Attribute::ReturnsTwice; - case ReadNone: - return Attribute::ReadNone; - case InaccessibleMemOnly: - return Attribute::InaccessibleMemOnly; - case SanitizeHWAddress: - return Attribute::SanitizeHWAddress; - case WillReturn: - return Attribute::WillReturn; - case StackProtectReq: - return Attribute::StackProtectReq; - case StackProtectStrong: - return Attribute::StackProtectStrong; - case StackProtect: - return Attribute::StackProtect; - case NoUndef: - return Attribute::NoUndef; - case SanitizeMemTag: - return Attribute::SanitizeMemTag; - case ShadowCallStack: - return Attribute::ShadowCallStack; - case AllocSize: - return Attribute::AllocSize; -#if LLVM_VERSION_GE(15, 0) - case AllocatedPointer: - return Attribute::AllocatedPointer; - case AllocAlign: - return Attribute::AllocAlign; -#endif - } - report_fatal_error("bad AttributeKind"); -} - -template static inline void AddAttributes(T *t, unsigned Index, - LLVMAttributeRef *Attrs, size_t AttrsLen) { - AttributeList PAL = t->getAttributes(); - AttributeList PALNew; -#if LLVM_VERSION_LT(14, 0) - AttrBuilder B; - for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen)) - B.addAttribute(unwrap(Attr)); - PALNew = PAL.addAttributes(t->getContext(), Index, B); -#else - AttrBuilder B(t->getContext()); - for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen)) - B.addAttribute(unwrap(Attr)); - PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B); -#endif - t->setAttributes(PALNew); -} - -extern "C" void LLVMRustAddFunctionAttributes(LLVMValueRef Fn, unsigned Index, - LLVMAttributeRef *Attrs, size_t AttrsLen) { - Function *F = unwrap(Fn); - AddAttributes(F, Index, Attrs, AttrsLen); -} - -extern "C" void LLVMRustAddCallSiteAttributes(LLVMValueRef Instr, unsigned Index, - LLVMAttributeRef *Attrs, size_t AttrsLen) { - CallBase *Call = unwrap(Instr); - AddAttributes(Call, Index, Attrs, AttrsLen); -} - -extern "C" LLVMAttributeRef LLVMRustCreateAttrNoValue(LLVMContextRef C, - LLVMRustAttribute RustAttr) { - return wrap(Attribute::get(*unwrap(C), fromRust(RustAttr))); -} - -extern "C" LLVMAttributeRef LLVMRustCreateAlignmentAttr(LLVMContextRef C, - uint64_t Bytes) { - return wrap(Attribute::getWithAlignment(*unwrap(C), llvm::Align(Bytes))); -} - -extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableAttr(LLVMContextRef C, - uint64_t Bytes) { - return wrap(Attribute::getWithDereferenceableBytes(*unwrap(C), Bytes)); -} - -extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableOrNullAttr(LLVMContextRef C, - uint64_t Bytes) { - return wrap(Attribute::getWithDereferenceableOrNullBytes(*unwrap(C), Bytes)); -} - -extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C, LLVMTypeRef Ty) { - return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty))); -} - -extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C, LLVMTypeRef Ty) { - return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty))); -} - -extern "C" LLVMAttributeRef LLVMRustCreateElementTypeAttr(LLVMContextRef C, LLVMTypeRef Ty) { -#if LLVM_VERSION_GE(15, 0) - return wrap(Attribute::get(*unwrap(C), Attribute::ElementType, unwrap(Ty))); -#else - report_fatal_error("Should not be needed on LLVM < 15"); -#endif -} - -extern "C" LLVMAttributeRef LLVMRustCreateUWTableAttr(LLVMContextRef C, bool Async) { -#if LLVM_VERSION_LT(15, 0) - return wrap(Attribute::get(*unwrap(C), Attribute::UWTable)); -#else - return wrap(Attribute::getWithUWTableKind( - *unwrap(C), Async ? UWTableKind::Async : UWTableKind::Sync)); -#endif -} - -extern "C" LLVMAttributeRef LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) { - return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, None)); -} - -#if LLVM_VERSION_GE(15, 0) - -// These values **must** match ffi::AllocKindFlags. -// It _happens_ to match the LLVM values of llvm::AllocFnKind, -// but that's happenstance and we do explicit conversions before -// passing them to LLVM. -enum class LLVMRustAllocKindFlags : uint64_t { - Unknown = 0, - Alloc = 1, - Realloc = 1 << 1, - Free = 1 << 2, - Uninitialized = 1 << 3, - Zeroed = 1 << 4, - Aligned = 1 << 5, -}; - -static LLVMRustAllocKindFlags operator&(LLVMRustAllocKindFlags A, LLVMRustAllocKindFlags B) { - return static_cast(static_cast(A) & - static_cast(B)); -} - -static bool isSet(LLVMRustAllocKindFlags F) { return F != LLVMRustAllocKindFlags::Unknown; } - -static llvm::AllocFnKind allocKindFromRust(LLVMRustAllocKindFlags F) { - llvm::AllocFnKind AFK = llvm::AllocFnKind::Unknown; - if (isSet(F & LLVMRustAllocKindFlags::Alloc)) { - AFK |= llvm::AllocFnKind::Alloc; - } - if (isSet(F & LLVMRustAllocKindFlags::Realloc)) { - AFK |= llvm::AllocFnKind::Realloc; - } - if (isSet(F & LLVMRustAllocKindFlags::Free)) { - AFK |= llvm::AllocFnKind::Free; - } - if (isSet(F & LLVMRustAllocKindFlags::Uninitialized)) { - AFK |= llvm::AllocFnKind::Uninitialized; - } - if (isSet(F & LLVMRustAllocKindFlags::Zeroed)) { - AFK |= llvm::AllocFnKind::Zeroed; - } - if (isSet(F & LLVMRustAllocKindFlags::Aligned)) { - AFK |= llvm::AllocFnKind::Aligned; - } - return AFK; -} -#endif - -extern "C" LLVMAttributeRef LLVMRustCreateAllocKindAttr(LLVMContextRef C, uint64_t AllocKindArg) { -#if LLVM_VERSION_GE(15, 0) - return wrap(Attribute::get(*unwrap(C), Attribute::AllocKind, - static_cast(allocKindFromRust(static_cast(AllocKindArg))))); -#else - report_fatal_error( - "allockind attributes are new in LLVM 15 and should not be used on older LLVMs"); -#endif -} - -// Enable a fast-math flag -// -// https://llvm.org/docs/LangRef.html#fast-math-flags -extern "C" void LLVMRustSetFastMath(LLVMValueRef V) { - if (auto I = dyn_cast(unwrap(V))) { - I->setFast(true); - } -} - -extern "C" LLVMValueRef -LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source, - const char *Name, LLVMAtomicOrdering Order) { - Value *Ptr = unwrap(Source); - LoadInst *LI = unwrap(B)->CreateLoad(unwrap(Ty), Ptr, Name); - LI->setAtomic(fromRust(Order)); - return wrap(LI); -} - -extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B, - LLVMValueRef V, - LLVMValueRef Target, - LLVMAtomicOrdering Order) { - StoreInst *SI = unwrap(B)->CreateStore(unwrap(V), unwrap(Target)); - SI->setAtomic(fromRust(Order)); - return wrap(SI); -} - -// FIXME: Use the C-API LLVMBuildAtomicCmpXchg and LLVMSetWeak -// once we raise our minimum support to LLVM 10. -extern "C" LLVMValueRef -LLVMRustBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Target, - LLVMValueRef Old, LLVMValueRef Source, - LLVMAtomicOrdering Order, - LLVMAtomicOrdering FailureOrder, LLVMBool Weak) { -#if LLVM_VERSION_GE(13,0) - // Rust probably knows the alignment of the target value and should be able to - // specify something more precise than MaybeAlign here. See also - // https://reviews.llvm.org/D97224 which may be a useful reference. - AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg( - unwrap(Target), unwrap(Old), unwrap(Source), llvm::MaybeAlign(), fromRust(Order), - fromRust(FailureOrder)); -#else - AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg( - unwrap(Target), unwrap(Old), unwrap(Source), fromRust(Order), - fromRust(FailureOrder)); -#endif - ACXI->setWeak(Weak); - return wrap(ACXI); -} - -enum class LLVMRustSynchronizationScope { - SingleThread, - CrossThread, -}; - -static SyncScope::ID fromRust(LLVMRustSynchronizationScope Scope) { - switch (Scope) { - case LLVMRustSynchronizationScope::SingleThread: - return SyncScope::SingleThread; - case LLVMRustSynchronizationScope::CrossThread: - return SyncScope::System; - default: - report_fatal_error("bad SynchronizationScope."); - } -} - -extern "C" LLVMValueRef -LLVMRustBuildAtomicFence(LLVMBuilderRef B, LLVMAtomicOrdering Order, - LLVMRustSynchronizationScope Scope) { - return wrap(unwrap(B)->CreateFence(fromRust(Order), fromRust(Scope))); -} - -enum class LLVMRustAsmDialect { - Att, - Intel, -}; - -static InlineAsm::AsmDialect fromRust(LLVMRustAsmDialect Dialect) { - switch (Dialect) { - case LLVMRustAsmDialect::Att: - return InlineAsm::AD_ATT; - case LLVMRustAsmDialect::Intel: - return InlineAsm::AD_Intel; - default: - report_fatal_error("bad AsmDialect."); - } -} - -extern "C" LLVMValueRef -LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen, - char *Constraints, size_t ConstraintsLen, - LLVMBool HasSideEffects, LLVMBool IsAlignStack, - LLVMRustAsmDialect Dialect, LLVMBool CanThrow) { -#if LLVM_VERSION_GE(13, 0) - return wrap(InlineAsm::get(unwrap(Ty), - StringRef(AsmString, AsmStringLen), - StringRef(Constraints, ConstraintsLen), - HasSideEffects, IsAlignStack, - fromRust(Dialect), CanThrow)); -#else - return wrap(InlineAsm::get(unwrap(Ty), - StringRef(AsmString, AsmStringLen), - StringRef(Constraints, ConstraintsLen), - HasSideEffects, IsAlignStack, - fromRust(Dialect))); -#endif -} - -extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints, - size_t ConstraintsLen) { -#if LLVM_VERSION_LT(15, 0) - return InlineAsm::Verify(unwrap(Ty), - StringRef(Constraints, ConstraintsLen)); -#else - // llvm::Error converts to true if it is an error. - return !llvm::errorToBool(InlineAsm::verify( - unwrap(Ty), StringRef(Constraints, ConstraintsLen))); -#endif -} - -extern "C" void LLVMRustAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm, - size_t AsmLen) { - unwrap(M)->appendModuleInlineAsm(StringRef(Asm, AsmLen)); -} - -typedef DIBuilder *LLVMRustDIBuilderRef; - -template DIT *unwrapDIPtr(LLVMMetadataRef Ref) { - return (DIT *)(Ref ? unwrap(Ref) : nullptr); -} - -#define DIDescriptor DIScope -#define DIArray DINodeArray -#define unwrapDI unwrapDIPtr - -// These values **must** match debuginfo::DIFlags! They also *happen* -// to match LLVM, but that isn't required as we do giant sets of -// matching below. The value shouldn't be directly passed to LLVM. -enum class LLVMRustDIFlags : uint32_t { - FlagZero = 0, - FlagPrivate = 1, - FlagProtected = 2, - FlagPublic = 3, - FlagFwdDecl = (1 << 2), - FlagAppleBlock = (1 << 3), - FlagBlockByrefStruct = (1 << 4), - FlagVirtual = (1 << 5), - FlagArtificial = (1 << 6), - FlagExplicit = (1 << 7), - FlagPrototyped = (1 << 8), - FlagObjcClassComplete = (1 << 9), - FlagObjectPointer = (1 << 10), - FlagVector = (1 << 11), - FlagStaticMember = (1 << 12), - FlagLValueReference = (1 << 13), - FlagRValueReference = (1 << 14), - FlagExternalTypeRef = (1 << 15), - FlagIntroducedVirtual = (1 << 18), - FlagBitField = (1 << 19), - FlagNoReturn = (1 << 20), - // Do not add values that are not supported by the minimum LLVM - // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def -}; - -inline LLVMRustDIFlags operator&(LLVMRustDIFlags A, LLVMRustDIFlags B) { - return static_cast(static_cast(A) & - static_cast(B)); -} - -inline LLVMRustDIFlags operator|(LLVMRustDIFlags A, LLVMRustDIFlags B) { - return static_cast(static_cast(A) | - static_cast(B)); -} - -inline LLVMRustDIFlags &operator|=(LLVMRustDIFlags &A, LLVMRustDIFlags B) { - return A = A | B; -} - -inline bool isSet(LLVMRustDIFlags F) { return F != LLVMRustDIFlags::FlagZero; } - -inline LLVMRustDIFlags visibility(LLVMRustDIFlags F) { - return static_cast(static_cast(F) & 0x3); -} - -static DINode::DIFlags fromRust(LLVMRustDIFlags Flags) { - DINode::DIFlags Result = DINode::DIFlags::FlagZero; - - switch (visibility(Flags)) { - case LLVMRustDIFlags::FlagPrivate: - Result |= DINode::DIFlags::FlagPrivate; - break; - case LLVMRustDIFlags::FlagProtected: - Result |= DINode::DIFlags::FlagProtected; - break; - case LLVMRustDIFlags::FlagPublic: - Result |= DINode::DIFlags::FlagPublic; - break; - default: - // The rest are handled below - break; - } - - if (isSet(Flags & LLVMRustDIFlags::FlagFwdDecl)) { - Result |= DINode::DIFlags::FlagFwdDecl; - } - if (isSet(Flags & LLVMRustDIFlags::FlagAppleBlock)) { - Result |= DINode::DIFlags::FlagAppleBlock; - } - if (isSet(Flags & LLVMRustDIFlags::FlagVirtual)) { - Result |= DINode::DIFlags::FlagVirtual; - } - if (isSet(Flags & LLVMRustDIFlags::FlagArtificial)) { - Result |= DINode::DIFlags::FlagArtificial; - } - if (isSet(Flags & LLVMRustDIFlags::FlagExplicit)) { - Result |= DINode::DIFlags::FlagExplicit; - } - if (isSet(Flags & LLVMRustDIFlags::FlagPrototyped)) { - Result |= DINode::DIFlags::FlagPrototyped; - } - if (isSet(Flags & LLVMRustDIFlags::FlagObjcClassComplete)) { - Result |= DINode::DIFlags::FlagObjcClassComplete; - } - if (isSet(Flags & LLVMRustDIFlags::FlagObjectPointer)) { - Result |= DINode::DIFlags::FlagObjectPointer; - } - if (isSet(Flags & LLVMRustDIFlags::FlagVector)) { - Result |= DINode::DIFlags::FlagVector; - } - if (isSet(Flags & LLVMRustDIFlags::FlagStaticMember)) { - Result |= DINode::DIFlags::FlagStaticMember; - } - if (isSet(Flags & LLVMRustDIFlags::FlagLValueReference)) { - Result |= DINode::DIFlags::FlagLValueReference; - } - if (isSet(Flags & LLVMRustDIFlags::FlagRValueReference)) { - Result |= DINode::DIFlags::FlagRValueReference; - } - if (isSet(Flags & LLVMRustDIFlags::FlagIntroducedVirtual)) { - Result |= DINode::DIFlags::FlagIntroducedVirtual; - } - if (isSet(Flags & LLVMRustDIFlags::FlagBitField)) { - Result |= DINode::DIFlags::FlagBitField; - } - if (isSet(Flags & LLVMRustDIFlags::FlagNoReturn)) { - Result |= DINode::DIFlags::FlagNoReturn; - } - - return Result; -} - -// These values **must** match debuginfo::DISPFlags! They also *happen* -// to match LLVM, but that isn't required as we do giant sets of -// matching below. The value shouldn't be directly passed to LLVM. -enum class LLVMRustDISPFlags : uint32_t { - SPFlagZero = 0, - SPFlagVirtual = 1, - SPFlagPureVirtual = 2, - SPFlagLocalToUnit = (1 << 2), - SPFlagDefinition = (1 << 3), - SPFlagOptimized = (1 << 4), - SPFlagMainSubprogram = (1 << 5), - // Do not add values that are not supported by the minimum LLVM - // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def - // (In LLVM < 8, createFunction supported these as separate bool arguments.) -}; - -inline LLVMRustDISPFlags operator&(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { - return static_cast(static_cast(A) & - static_cast(B)); -} - -inline LLVMRustDISPFlags operator|(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { - return static_cast(static_cast(A) | - static_cast(B)); -} - -inline LLVMRustDISPFlags &operator|=(LLVMRustDISPFlags &A, LLVMRustDISPFlags B) { - return A = A | B; -} - -inline bool isSet(LLVMRustDISPFlags F) { return F != LLVMRustDISPFlags::SPFlagZero; } - -inline LLVMRustDISPFlags virtuality(LLVMRustDISPFlags F) { - return static_cast(static_cast(F) & 0x3); -} - -static DISubprogram::DISPFlags fromRust(LLVMRustDISPFlags SPFlags) { - DISubprogram::DISPFlags Result = DISubprogram::DISPFlags::SPFlagZero; - - switch (virtuality(SPFlags)) { - case LLVMRustDISPFlags::SPFlagVirtual: - Result |= DISubprogram::DISPFlags::SPFlagVirtual; - break; - case LLVMRustDISPFlags::SPFlagPureVirtual: - Result |= DISubprogram::DISPFlags::SPFlagPureVirtual; - break; - default: - // The rest are handled below - break; - } - - if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagLocalToUnit)) { - Result |= DISubprogram::DISPFlags::SPFlagLocalToUnit; - } - if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagDefinition)) { - Result |= DISubprogram::DISPFlags::SPFlagDefinition; - } - if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagOptimized)) { - Result |= DISubprogram::DISPFlags::SPFlagOptimized; - } - if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagMainSubprogram)) { - Result |= DISubprogram::DISPFlags::SPFlagMainSubprogram; - } - - return Result; -} - -enum class LLVMRustDebugEmissionKind { - NoDebug, - FullDebug, - LineTablesOnly, -}; - -static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind) { - switch (Kind) { - case LLVMRustDebugEmissionKind::NoDebug: - return DICompileUnit::DebugEmissionKind::NoDebug; - case LLVMRustDebugEmissionKind::FullDebug: - return DICompileUnit::DebugEmissionKind::FullDebug; - case LLVMRustDebugEmissionKind::LineTablesOnly: - return DICompileUnit::DebugEmissionKind::LineTablesOnly; - default: - report_fatal_error("bad DebugEmissionKind."); - } -} - -enum class LLVMRustChecksumKind { - None, - MD5, - SHA1, - SHA256, -}; - -static Optional fromRust(LLVMRustChecksumKind Kind) { - switch (Kind) { - case LLVMRustChecksumKind::None: - return None; - case LLVMRustChecksumKind::MD5: - return DIFile::ChecksumKind::CSK_MD5; - case LLVMRustChecksumKind::SHA1: - return DIFile::ChecksumKind::CSK_SHA1; - case LLVMRustChecksumKind::SHA256: - return DIFile::ChecksumKind::CSK_SHA256; - default: - report_fatal_error("bad ChecksumKind."); - } -} - -extern "C" uint32_t LLVMRustDebugMetadataVersion() { - return DEBUG_METADATA_VERSION; -} - -extern "C" uint32_t LLVMRustVersionPatch() { return LLVM_VERSION_PATCH; } - -extern "C" uint32_t LLVMRustVersionMinor() { return LLVM_VERSION_MINOR; } - -extern "C" uint32_t LLVMRustVersionMajor() { return LLVM_VERSION_MAJOR; } - -extern "C" void LLVMRustAddModuleFlag( - LLVMModuleRef M, - Module::ModFlagBehavior MergeBehavior, - const char *Name, - uint32_t Value) { - unwrap(M)->addModuleFlag(MergeBehavior, Name, Value); -} - -extern "C" bool LLVMRustHasModuleFlag(LLVMModuleRef M, const char *Name, - size_t Len) { - return unwrap(M)->getModuleFlag(StringRef(Name, Len)) != nullptr; -} - -extern "C" LLVMValueRef LLVMRustMetadataAsValue(LLVMContextRef C, LLVMMetadataRef MD) { - return wrap(MetadataAsValue::get(*unwrap(C), unwrap(MD))); -} - -extern "C" void LLVMRustGlobalAddMetadata( - LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) { - unwrap(Global)->addMetadata(Kind, *unwrap(MD)); -} - -extern "C" LLVMRustDIBuilderRef LLVMRustDIBuilderCreate(LLVMModuleRef M) { - return new DIBuilder(*unwrap(M)); -} - -extern "C" void LLVMRustDIBuilderDispose(LLVMRustDIBuilderRef Builder) { - delete Builder; -} - -extern "C" void LLVMRustDIBuilderFinalize(LLVMRustDIBuilderRef Builder) { - Builder->finalize(); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateCompileUnit( - LLVMRustDIBuilderRef Builder, unsigned Lang, LLVMMetadataRef FileRef, - const char *Producer, size_t ProducerLen, bool isOptimized, - const char *Flags, unsigned RuntimeVer, - const char *SplitName, size_t SplitNameLen, - LLVMRustDebugEmissionKind Kind, - uint64_t DWOId, bool SplitDebugInlining) { - auto *File = unwrapDI(FileRef); - - return wrap(Builder->createCompileUnit(Lang, File, StringRef(Producer, ProducerLen), - isOptimized, Flags, RuntimeVer, - StringRef(SplitName, SplitNameLen), - fromRust(Kind), DWOId, SplitDebugInlining)); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFile( - LLVMRustDIBuilderRef Builder, - const char *Filename, size_t FilenameLen, - const char *Directory, size_t DirectoryLen, LLVMRustChecksumKind CSKind, - const char *Checksum, size_t ChecksumLen) { - Optional llvmCSKind = fromRust(CSKind); - Optional> CSInfo{}; - if (llvmCSKind) - CSInfo.emplace(*llvmCSKind, StringRef{Checksum, ChecksumLen}); - return wrap(Builder->createFile(StringRef(Filename, FilenameLen), - StringRef(Directory, DirectoryLen), - CSInfo)); -} - -extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateSubroutineType(LLVMRustDIBuilderRef Builder, - LLVMMetadataRef ParameterTypes) { - return wrap(Builder->createSubroutineType( - DITypeRefArray(unwrap(ParameterTypes)))); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, - const char *LinkageName, size_t LinkageNameLen, - LLVMMetadataRef File, unsigned LineNo, - LLVMMetadataRef Ty, unsigned ScopeLine, LLVMRustDIFlags Flags, - LLVMRustDISPFlags SPFlags, LLVMValueRef MaybeFn, LLVMMetadataRef TParam, - LLVMMetadataRef Decl) { - DITemplateParameterArray TParams = - DITemplateParameterArray(unwrap(TParam)); - DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); - DINode::DIFlags llvmFlags = fromRust(Flags); - DISubprogram *Sub = Builder->createFunction( - unwrapDI(Scope), - StringRef(Name, NameLen), - StringRef(LinkageName, LinkageNameLen), - unwrapDI(File), LineNo, - unwrapDI(Ty), ScopeLine, llvmFlags, - llvmSPFlags, TParams, unwrapDIPtr(Decl)); - if (MaybeFn) - unwrap(MaybeFn)->setSubprogram(Sub); - return wrap(Sub); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateBasicType( - LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, - uint64_t SizeInBits, unsigned Encoding) { - return wrap(Builder->createBasicType(StringRef(Name, NameLen), SizeInBits, Encoding)); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTypedef( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Type, const char *Name, size_t NameLen, - LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Scope) { - return wrap(Builder->createTypedef( - unwrap(Type), StringRef(Name, NameLen), unwrap(File), - LineNo, unwrapDIPtr(Scope))); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef PointeeTy, - uint64_t SizeInBits, uint32_t AlignInBits, unsigned AddressSpace, - const char *Name, size_t NameLen) { - return wrap(Builder->createPointerType(unwrapDI(PointeeTy), - SizeInBits, AlignInBits, - AddressSpace, - StringRef(Name, NameLen))); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStructType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, - LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, - uint32_t AlignInBits, LLVMRustDIFlags Flags, - LLVMMetadataRef DerivedFrom, LLVMMetadataRef Elements, - unsigned RunTimeLang, LLVMMetadataRef VTableHolder, - const char *UniqueId, size_t UniqueIdLen) { - return wrap(Builder->createStructType( - unwrapDI(Scope), StringRef(Name, NameLen), - unwrapDI(File), LineNumber, - SizeInBits, AlignInBits, fromRust(Flags), unwrapDI(DerivedFrom), - DINodeArray(unwrapDI(Elements)), RunTimeLang, - unwrapDI(VTableHolder), StringRef(UniqueId, UniqueIdLen))); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantPart( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, - LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, - uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Discriminator, - LLVMMetadataRef Elements, const char *UniqueId, size_t UniqueIdLen) { - return wrap(Builder->createVariantPart( - unwrapDI(Scope), StringRef(Name, NameLen), - unwrapDI(File), LineNumber, - SizeInBits, AlignInBits, fromRust(Flags), unwrapDI(Discriminator), - DINodeArray(unwrapDI(Elements)), StringRef(UniqueId, UniqueIdLen))); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMemberType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, - LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits, - uint32_t AlignInBits, uint64_t OffsetInBits, LLVMRustDIFlags Flags, - LLVMMetadataRef Ty) { - return wrap(Builder->createMemberType(unwrapDI(Scope), - StringRef(Name, NameLen), - unwrapDI(File), LineNo, - SizeInBits, AlignInBits, OffsetInBits, - fromRust(Flags), unwrapDI(Ty))); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo, - uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, LLVMValueRef Discriminant, - LLVMRustDIFlags Flags, LLVMMetadataRef Ty) { - llvm::ConstantInt* D = nullptr; - if (Discriminant) { - D = unwrap(Discriminant); - } - return wrap(Builder->createVariantMemberType(unwrapDI(Scope), - StringRef(Name, NameLen), - unwrapDI(File), LineNo, - SizeInBits, AlignInBits, OffsetInBits, D, - fromRust(Flags), unwrapDI(Ty))); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateLexicalBlock( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - LLVMMetadataRef File, unsigned Line, unsigned Col) { - return wrap(Builder->createLexicalBlock(unwrapDI(Scope), - unwrapDI(File), Line, Col)); -} - -extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateLexicalBlockFile(LLVMRustDIBuilderRef Builder, - LLVMMetadataRef Scope, - LLVMMetadataRef File) { - return wrap(Builder->createLexicalBlockFile(unwrapDI(Scope), - unwrapDI(File))); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Context, - const char *Name, size_t NameLen, - const char *LinkageName, size_t LinkageNameLen, - LLVMMetadataRef File, unsigned LineNo, - LLVMMetadataRef Ty, bool IsLocalToUnit, LLVMValueRef V, - LLVMMetadataRef Decl = nullptr, uint32_t AlignInBits = 0) { - llvm::GlobalVariable *InitVal = cast(unwrap(V)); - - llvm::DIExpression *InitExpr = nullptr; - if (llvm::ConstantInt *IntVal = llvm::dyn_cast(InitVal)) { - InitExpr = Builder->createConstantValueExpression( - IntVal->getValue().getSExtValue()); - } else if (llvm::ConstantFP *FPVal = - llvm::dyn_cast(InitVal)) { - InitExpr = Builder->createConstantValueExpression( - FPVal->getValueAPF().bitcastToAPInt().getZExtValue()); - } - - llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression( - unwrapDI(Context), StringRef(Name, NameLen), - StringRef(LinkageName, LinkageNameLen), - unwrapDI(File), LineNo, unwrapDI(Ty), IsLocalToUnit, - /* isDefined */ true, - InitExpr, unwrapDIPtr(Decl), - /* templateParams */ nullptr, - AlignInBits); - - InitVal->setMetadata("dbg", VarExpr); - - return wrap(VarExpr); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable( - LLVMRustDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, - LLVMMetadataRef File, unsigned LineNo, - LLVMMetadataRef Ty, bool AlwaysPreserve, LLVMRustDIFlags Flags, - unsigned ArgNo, uint32_t AlignInBits) { - if (Tag == 0x100) { // DW_TAG_auto_variable - return wrap(Builder->createAutoVariable( - unwrapDI(Scope), StringRef(Name, NameLen), - unwrapDI(File), LineNo, - unwrapDI(Ty), AlwaysPreserve, fromRust(Flags), AlignInBits)); - } else { - return wrap(Builder->createParameterVariable( - unwrapDI(Scope), StringRef(Name, NameLen), ArgNo, - unwrapDI(File), LineNo, - unwrapDI(Ty), AlwaysPreserve, fromRust(Flags))); - } -} - -extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateArrayType(LLVMRustDIBuilderRef Builder, uint64_t Size, - uint32_t AlignInBits, LLVMMetadataRef Ty, - LLVMMetadataRef Subscripts) { - return wrap( - Builder->createArrayType(Size, AlignInBits, unwrapDI(Ty), - DINodeArray(unwrapDI(Subscripts)))); -} - -extern "C" LLVMMetadataRef -LLVMRustDIBuilderGetOrCreateSubrange(LLVMRustDIBuilderRef Builder, int64_t Lo, - int64_t Count) { - return wrap(Builder->getOrCreateSubrange(Lo, Count)); -} - -extern "C" LLVMMetadataRef -LLVMRustDIBuilderGetOrCreateArray(LLVMRustDIBuilderRef Builder, - LLVMMetadataRef *Ptr, unsigned Count) { - Metadata **DataValue = unwrap(Ptr); - return wrap( - Builder->getOrCreateArray(ArrayRef(DataValue, Count)).get()); -} - -extern "C" LLVMValueRef LLVMRustDIBuilderInsertDeclareAtEnd( - LLVMRustDIBuilderRef Builder, LLVMValueRef V, LLVMMetadataRef VarInfo, - uint64_t *AddrOps, unsigned AddrOpsCount, LLVMMetadataRef DL, - LLVMBasicBlockRef InsertAtEnd) { - return wrap(Builder->insertDeclare( - unwrap(V), unwrap(VarInfo), - Builder->createExpression(llvm::ArrayRef(AddrOps, AddrOpsCount)), - DebugLoc(cast(unwrap(DL))), - unwrap(InsertAtEnd))); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerator( - LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, - int64_t Value, bool IsUnsigned) { - return wrap(Builder->createEnumerator(StringRef(Name, NameLen), Value, IsUnsigned)); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, - LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, - uint32_t AlignInBits, LLVMMetadataRef Elements, - LLVMMetadataRef ClassTy, bool IsScoped) { - return wrap(Builder->createEnumerationType( - unwrapDI(Scope), StringRef(Name, NameLen), - unwrapDI(File), LineNumber, - SizeInBits, AlignInBits, DINodeArray(unwrapDI(Elements)), - unwrapDI(ClassTy), "", IsScoped)); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, - LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, - uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Elements, - unsigned RunTimeLang, const char *UniqueId, size_t UniqueIdLen) { - return wrap(Builder->createUnionType( - unwrapDI(Scope), StringRef(Name, NameLen), unwrapDI(File), - LineNumber, SizeInBits, AlignInBits, fromRust(Flags), - DINodeArray(unwrapDI(Elements)), RunTimeLang, - StringRef(UniqueId, UniqueIdLen))); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTemplateTypeParameter( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, LLVMMetadataRef Ty) { - bool IsDefault = false; // FIXME: should we ever set this true? - return wrap(Builder->createTemplateTypeParameter( - unwrapDI(Scope), StringRef(Name, NameLen), unwrapDI(Ty), IsDefault)); -} - -extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateNameSpace( - LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, - const char *Name, size_t NameLen, bool ExportSymbols) { - return wrap(Builder->createNameSpace( - unwrapDI(Scope), StringRef(Name, NameLen), ExportSymbols - )); -} - -extern "C" void -LLVMRustDICompositeTypeReplaceArrays(LLVMRustDIBuilderRef Builder, - LLVMMetadataRef CompositeTy, - LLVMMetadataRef Elements, - LLVMMetadataRef Params) { - DICompositeType *Tmp = unwrapDI(CompositeTy); - Builder->replaceArrays(Tmp, DINodeArray(unwrap(Elements)), - DINodeArray(unwrap(Params))); -} - -extern "C" LLVMMetadataRef -LLVMRustDIBuilderCreateDebugLocation(unsigned Line, unsigned Column, - LLVMMetadataRef ScopeRef, - LLVMMetadataRef InlinedAt) { - MDNode *Scope = unwrapDIPtr(ScopeRef); - DILocation *Loc = DILocation::get( - Scope->getContext(), Line, Column, Scope, - unwrapDIPtr(InlinedAt)); - return wrap(Loc); -} - -extern "C" uint64_t LLVMRustDIBuilderCreateOpDeref() { - return dwarf::DW_OP_deref; -} - -extern "C" uint64_t LLVMRustDIBuilderCreateOpPlusUconst() { - return dwarf::DW_OP_plus_uconst; -} - -extern "C" void LLVMRustWriteTypeToString(LLVMTypeRef Ty, RustStringRef Str) { - RawRustStringOstream OS(Str); - unwrap(Ty)->print(OS); -} - -extern "C" void LLVMRustWriteValueToString(LLVMValueRef V, - RustStringRef Str) { - RawRustStringOstream OS(Str); - if (!V) { - OS << "(null)"; - } else { - OS << "("; - unwrap(V)->getType()->print(OS); - OS << ":"; - unwrap(V)->print(OS); - OS << ")"; - } -} - -// LLVMArrayType function does not support 64-bit ElementCount -extern "C" LLVMTypeRef LLVMRustArrayType(LLVMTypeRef ElementTy, - uint64_t ElementCount) { - return wrap(ArrayType::get(unwrap(ElementTy), ElementCount)); -} - -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Twine, LLVMTwineRef) - -extern "C" void LLVMRustWriteTwineToString(LLVMTwineRef T, RustStringRef Str) { - RawRustStringOstream OS(Str); - unwrap(T)->print(OS); -} - -extern "C" void LLVMRustUnpackOptimizationDiagnostic( - LLVMDiagnosticInfoRef DI, RustStringRef PassNameOut, - LLVMValueRef *FunctionOut, unsigned* Line, unsigned* Column, - RustStringRef FilenameOut, RustStringRef MessageOut) { - // Undefined to call this not on an optimization diagnostic! - llvm::DiagnosticInfoOptimizationBase *Opt = - static_cast(unwrap(DI)); - - RawRustStringOstream PassNameOS(PassNameOut); - PassNameOS << Opt->getPassName(); - *FunctionOut = wrap(&Opt->getFunction()); - - RawRustStringOstream FilenameOS(FilenameOut); - DiagnosticLocation loc = Opt->getLocation(); - if (loc.isValid()) { - *Line = loc.getLine(); - *Column = loc.getColumn(); - FilenameOS << loc.getAbsolutePath(); - } - - RawRustStringOstream MessageOS(MessageOut); - MessageOS << Opt->getMsg(); -} - -enum class LLVMRustDiagnosticLevel { - Error, - Warning, - Note, - Remark, -}; - -extern "C" void -LLVMRustUnpackInlineAsmDiagnostic(LLVMDiagnosticInfoRef DI, - LLVMRustDiagnosticLevel *LevelOut, - unsigned *CookieOut, - LLVMTwineRef *MessageOut) { - // Undefined to call this not on an inline assembly diagnostic! - llvm::DiagnosticInfoInlineAsm *IA = - static_cast(unwrap(DI)); - - *CookieOut = IA->getLocCookie(); - *MessageOut = wrap(&IA->getMsgStr()); - - switch (IA->getSeverity()) { - case DS_Error: - *LevelOut = LLVMRustDiagnosticLevel::Error; - break; - case DS_Warning: - *LevelOut = LLVMRustDiagnosticLevel::Warning; - break; - case DS_Note: - *LevelOut = LLVMRustDiagnosticLevel::Note; - break; - case DS_Remark: - *LevelOut = LLVMRustDiagnosticLevel::Remark; - break; - default: - report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); - } -} - -extern "C" void LLVMRustWriteDiagnosticInfoToString(LLVMDiagnosticInfoRef DI, - RustStringRef Str) { - RawRustStringOstream OS(Str); - DiagnosticPrinterRawOStream DP(OS); - unwrap(DI)->print(DP); -} - -enum class LLVMRustDiagnosticKind { - Other, - InlineAsm, - StackSize, - DebugMetadataVersion, - SampleProfile, - OptimizationRemark, - OptimizationRemarkMissed, - OptimizationRemarkAnalysis, - OptimizationRemarkAnalysisFPCommute, - OptimizationRemarkAnalysisAliasing, - OptimizationRemarkOther, - OptimizationFailure, - PGOProfile, - Linker, - Unsupported, - SrcMgr, -}; - -static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) { - switch (Kind) { - case DK_InlineAsm: - return LLVMRustDiagnosticKind::InlineAsm; - case DK_StackSize: - return LLVMRustDiagnosticKind::StackSize; - case DK_DebugMetadataVersion: - return LLVMRustDiagnosticKind::DebugMetadataVersion; - case DK_SampleProfile: - return LLVMRustDiagnosticKind::SampleProfile; - case DK_OptimizationRemark: - case DK_MachineOptimizationRemark: - return LLVMRustDiagnosticKind::OptimizationRemark; - case DK_OptimizationRemarkMissed: - case DK_MachineOptimizationRemarkMissed: - return LLVMRustDiagnosticKind::OptimizationRemarkMissed; - case DK_OptimizationRemarkAnalysis: - case DK_MachineOptimizationRemarkAnalysis: - return LLVMRustDiagnosticKind::OptimizationRemarkAnalysis; - case DK_OptimizationRemarkAnalysisFPCommute: - return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisFPCommute; - case DK_OptimizationRemarkAnalysisAliasing: - return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisAliasing; - case DK_PGOProfile: - return LLVMRustDiagnosticKind::PGOProfile; - case DK_Linker: - return LLVMRustDiagnosticKind::Linker; - case DK_Unsupported: - return LLVMRustDiagnosticKind::Unsupported; -#if LLVM_VERSION_GE(13, 0) - case DK_SrcMgr: - return LLVMRustDiagnosticKind::SrcMgr; -#endif - default: - return (Kind >= DK_FirstRemark && Kind <= DK_LastRemark) - ? LLVMRustDiagnosticKind::OptimizationRemarkOther - : LLVMRustDiagnosticKind::Other; - } -} - -extern "C" LLVMRustDiagnosticKind -LLVMRustGetDiagInfoKind(LLVMDiagnosticInfoRef DI) { - return toRust((DiagnosticKind)unwrap(DI)->getKind()); -} - -// This is kept distinct from LLVMGetTypeKind, because when -// a new type kind is added, the Rust-side enum must be -// updated or UB will result. -extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) { - switch (unwrap(Ty)->getTypeID()) { - case Type::VoidTyID: - return LLVMVoidTypeKind; - case Type::HalfTyID: - return LLVMHalfTypeKind; - case Type::FloatTyID: - return LLVMFloatTypeKind; - case Type::DoubleTyID: - return LLVMDoubleTypeKind; - case Type::X86_FP80TyID: - return LLVMX86_FP80TypeKind; - case Type::FP128TyID: - return LLVMFP128TypeKind; - case Type::PPC_FP128TyID: - return LLVMPPC_FP128TypeKind; - case Type::LabelTyID: - return LLVMLabelTypeKind; - case Type::MetadataTyID: - return LLVMMetadataTypeKind; - case Type::IntegerTyID: - return LLVMIntegerTypeKind; - case Type::FunctionTyID: - return LLVMFunctionTypeKind; - case Type::StructTyID: - return LLVMStructTypeKind; - case Type::ArrayTyID: - return LLVMArrayTypeKind; - case Type::PointerTyID: - return LLVMPointerTypeKind; - case Type::FixedVectorTyID: - return LLVMVectorTypeKind; - case Type::X86_MMXTyID: - return LLVMX86_MMXTypeKind; - case Type::TokenTyID: - return LLVMTokenTypeKind; - case Type::ScalableVectorTyID: - return LLVMScalableVectorTypeKind; - case Type::BFloatTyID: - return LLVMBFloatTypeKind; - case Type::X86_AMXTyID: - return LLVMX86_AMXTypeKind; -#if LLVM_VERSION_GE(15, 0) && LLVM_VERSION_LT(16, 0) - case Type::DXILPointerTyID: - report_fatal_error("Rust does not support DirectX typed pointers."); - break; -#endif -#if LLVM_VERSION_GE(16, 0) - case Type::TypedPointerTyID: - report_fatal_error("Rust does not support typed pointers."); - break; -#endif - } - report_fatal_error("Unhandled TypeID."); -} - -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef) - -#if LLVM_VERSION_LT(13, 0) -using LLVMInlineAsmDiagHandlerTy = LLVMContext::InlineAsmDiagHandlerTy; -#else -using LLVMInlineAsmDiagHandlerTy = void*; -#endif - -extern "C" void LLVMRustSetInlineAsmDiagnosticHandler( - LLVMContextRef C, LLVMInlineAsmDiagHandlerTy H, void *CX) { - // Diagnostic handlers were unified in LLVM change 5de2d189e6ad, so starting - // with LLVM 13 this function is gone. -#if LLVM_VERSION_LT(13, 0) - unwrap(C)->setInlineAsmDiagnosticHandler(H, CX); -#endif -} - -extern "C" LLVMSMDiagnosticRef LLVMRustGetSMDiagnostic( - LLVMDiagnosticInfoRef DI, unsigned *Cookie) { -#if LLVM_VERSION_GE(13, 0) - llvm::DiagnosticInfoSrcMgr *SM = static_cast(unwrap(DI)); - *Cookie = SM->getLocCookie(); - return wrap(&SM->getSMDiag()); -#else - report_fatal_error("Shouldn't get called on older versions"); -#endif -} - -extern "C" bool LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef, - RustStringRef MessageOut, - RustStringRef BufferOut, - LLVMRustDiagnosticLevel* LevelOut, - unsigned* LocOut, - unsigned* RangesOut, - size_t* NumRanges) { - SMDiagnostic& D = *unwrap(DRef); - RawRustStringOstream MessageOS(MessageOut); - MessageOS << D.getMessage(); - - switch (D.getKind()) { - case SourceMgr::DK_Error: - *LevelOut = LLVMRustDiagnosticLevel::Error; - break; - case SourceMgr::DK_Warning: - *LevelOut = LLVMRustDiagnosticLevel::Warning; - break; - case SourceMgr::DK_Note: - *LevelOut = LLVMRustDiagnosticLevel::Note; - break; - case SourceMgr::DK_Remark: - *LevelOut = LLVMRustDiagnosticLevel::Remark; - break; - default: - report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); - } - - if (D.getLoc() == SMLoc()) - return false; - - const SourceMgr &LSM = *D.getSourceMgr(); - const MemoryBuffer *LBuf = LSM.getMemoryBuffer(LSM.FindBufferContainingLoc(D.getLoc())); - LLVMRustStringWriteImpl(BufferOut, LBuf->getBufferStart(), LBuf->getBufferSize()); - - *LocOut = D.getLoc().getPointer() - LBuf->getBufferStart(); - - *NumRanges = std::min(*NumRanges, D.getRanges().size()); - size_t LineStart = *LocOut - (size_t)D.getColumnNo(); - for (size_t i = 0; i < *NumRanges; i++) { - RangesOut[i * 2] = LineStart + D.getRanges()[i].first; - RangesOut[i * 2 + 1] = LineStart + D.getRanges()[i].second; - } - - return true; -} - -extern "C" LLVMValueRef LLVMRustBuildCleanupPad(LLVMBuilderRef B, - LLVMValueRef ParentPad, - unsigned ArgCount, - LLVMValueRef *LLArgs, - const char *Name) { - Value **Args = unwrap(LLArgs); - if (ParentPad == nullptr) { - Type *Ty = Type::getTokenTy(unwrap(B)->getContext()); - ParentPad = wrap(Constant::getNullValue(Ty)); - } - return wrap(unwrap(B)->CreateCleanupPad( - unwrap(ParentPad), ArrayRef(Args, ArgCount), Name)); -} - -extern "C" LLVMValueRef LLVMRustBuildCleanupRet(LLVMBuilderRef B, - LLVMValueRef CleanupPad, - LLVMBasicBlockRef UnwindBB) { - CleanupPadInst *Inst = cast(unwrap(CleanupPad)); - return wrap(unwrap(B)->CreateCleanupRet(Inst, unwrap(UnwindBB))); -} - -extern "C" LLVMValueRef -LLVMRustBuildCatchPad(LLVMBuilderRef B, LLVMValueRef ParentPad, - unsigned ArgCount, LLVMValueRef *LLArgs, const char *Name) { - Value **Args = unwrap(LLArgs); - return wrap(unwrap(B)->CreateCatchPad( - unwrap(ParentPad), ArrayRef(Args, ArgCount), Name)); -} - -extern "C" LLVMValueRef LLVMRustBuildCatchRet(LLVMBuilderRef B, - LLVMValueRef Pad, - LLVMBasicBlockRef BB) { - return wrap(unwrap(B)->CreateCatchRet(cast(unwrap(Pad)), - unwrap(BB))); -} - -extern "C" LLVMValueRef LLVMRustBuildCatchSwitch(LLVMBuilderRef B, - LLVMValueRef ParentPad, - LLVMBasicBlockRef BB, - unsigned NumHandlers, - const char *Name) { - if (ParentPad == nullptr) { - Type *Ty = Type::getTokenTy(unwrap(B)->getContext()); - ParentPad = wrap(Constant::getNullValue(Ty)); - } - return wrap(unwrap(B)->CreateCatchSwitch(unwrap(ParentPad), unwrap(BB), - NumHandlers, Name)); -} - -extern "C" void LLVMRustAddHandler(LLVMValueRef CatchSwitchRef, - LLVMBasicBlockRef Handler) { - Value *CatchSwitch = unwrap(CatchSwitchRef); - cast(CatchSwitch)->addHandler(unwrap(Handler)); -} - -extern "C" OperandBundleDef *LLVMRustBuildOperandBundleDef(const char *Name, - LLVMValueRef *Inputs, - unsigned NumInputs) { - return new OperandBundleDef(Name, makeArrayRef(unwrap(Inputs), NumInputs)); -} - -extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { - delete Bundle; -} - -extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, - LLVMValueRef *Args, unsigned NumArgs, - OperandBundleDef *Bundle) { - Value *Callee = unwrap(Fn); - FunctionType *FTy = unwrap(Ty); - unsigned Len = Bundle ? 1 : 0; - ArrayRef Bundles = makeArrayRef(Bundle, Len); - return wrap(unwrap(B)->CreateCall( - FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), Bundles)); -} - -extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) { - return wrap(llvm::Intrinsic::getDeclaration(unwrap(M), - (llvm::Intrinsic::ID)llvm::Intrinsic::instrprof_increment)); -} - -extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, - LLVMValueRef Dst, unsigned DstAlign, - LLVMValueRef Src, unsigned SrcAlign, - LLVMValueRef Size, bool IsVolatile) { - return wrap(unwrap(B)->CreateMemCpy( - unwrap(Dst), MaybeAlign(DstAlign), - unwrap(Src), MaybeAlign(SrcAlign), - unwrap(Size), IsVolatile)); -} - -extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B, - LLVMValueRef Dst, unsigned DstAlign, - LLVMValueRef Src, unsigned SrcAlign, - LLVMValueRef Size, bool IsVolatile) { - return wrap(unwrap(B)->CreateMemMove( - unwrap(Dst), MaybeAlign(DstAlign), - unwrap(Src), MaybeAlign(SrcAlign), - unwrap(Size), IsVolatile)); -} - -extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, - LLVMValueRef Dst, unsigned DstAlign, - LLVMValueRef Val, - LLVMValueRef Size, bool IsVolatile) { - return wrap(unwrap(B)->CreateMemSet( - unwrap(Dst), unwrap(Val), unwrap(Size), MaybeAlign(DstAlign), IsVolatile)); -} - -extern "C" LLVMValueRef -LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, - LLVMValueRef *Args, unsigned NumArgs, - LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, - OperandBundleDef *Bundle, const char *Name) { - Value *Callee = unwrap(Fn); - FunctionType *FTy = unwrap(Ty); - unsigned Len = Bundle ? 1 : 0; - ArrayRef Bundles = makeArrayRef(Bundle, Len); - return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch), - makeArrayRef(unwrap(Args), NumArgs), - Bundles, Name)); -} - -extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, - LLVMBasicBlockRef BB) { - auto Point = unwrap(BB)->getFirstInsertionPt(); - unwrap(B)->SetInsertPoint(unwrap(BB), Point); -} - -extern "C" void LLVMRustSetComdat(LLVMModuleRef M, LLVMValueRef V, - const char *Name, size_t NameLen) { - Triple TargetTriple(unwrap(M)->getTargetTriple()); - GlobalObject *GV = unwrap(V); - if (TargetTriple.supportsCOMDAT()) { - StringRef NameRef(Name, NameLen); - GV->setComdat(unwrap(M)->getOrInsertComdat(NameRef)); - } -} - -enum class LLVMRustLinkage { - ExternalLinkage = 0, - AvailableExternallyLinkage = 1, - LinkOnceAnyLinkage = 2, - LinkOnceODRLinkage = 3, - WeakAnyLinkage = 4, - WeakODRLinkage = 5, - AppendingLinkage = 6, - InternalLinkage = 7, - PrivateLinkage = 8, - ExternalWeakLinkage = 9, - CommonLinkage = 10, -}; - -static LLVMRustLinkage toRust(LLVMLinkage Linkage) { - switch (Linkage) { - case LLVMExternalLinkage: - return LLVMRustLinkage::ExternalLinkage; - case LLVMAvailableExternallyLinkage: - return LLVMRustLinkage::AvailableExternallyLinkage; - case LLVMLinkOnceAnyLinkage: - return LLVMRustLinkage::LinkOnceAnyLinkage; - case LLVMLinkOnceODRLinkage: - return LLVMRustLinkage::LinkOnceODRLinkage; - case LLVMWeakAnyLinkage: - return LLVMRustLinkage::WeakAnyLinkage; - case LLVMWeakODRLinkage: - return LLVMRustLinkage::WeakODRLinkage; - case LLVMAppendingLinkage: - return LLVMRustLinkage::AppendingLinkage; - case LLVMInternalLinkage: - return LLVMRustLinkage::InternalLinkage; - case LLVMPrivateLinkage: - return LLVMRustLinkage::PrivateLinkage; - case LLVMExternalWeakLinkage: - return LLVMRustLinkage::ExternalWeakLinkage; - case LLVMCommonLinkage: - return LLVMRustLinkage::CommonLinkage; - default: - report_fatal_error("Invalid LLVMRustLinkage value!"); - } -} - -static LLVMLinkage fromRust(LLVMRustLinkage Linkage) { - switch (Linkage) { - case LLVMRustLinkage::ExternalLinkage: - return LLVMExternalLinkage; - case LLVMRustLinkage::AvailableExternallyLinkage: - return LLVMAvailableExternallyLinkage; - case LLVMRustLinkage::LinkOnceAnyLinkage: - return LLVMLinkOnceAnyLinkage; - case LLVMRustLinkage::LinkOnceODRLinkage: - return LLVMLinkOnceODRLinkage; - case LLVMRustLinkage::WeakAnyLinkage: - return LLVMWeakAnyLinkage; - case LLVMRustLinkage::WeakODRLinkage: - return LLVMWeakODRLinkage; - case LLVMRustLinkage::AppendingLinkage: - return LLVMAppendingLinkage; - case LLVMRustLinkage::InternalLinkage: - return LLVMInternalLinkage; - case LLVMRustLinkage::PrivateLinkage: - return LLVMPrivateLinkage; - case LLVMRustLinkage::ExternalWeakLinkage: - return LLVMExternalWeakLinkage; - case LLVMRustLinkage::CommonLinkage: - return LLVMCommonLinkage; - } - report_fatal_error("Invalid LLVMRustLinkage value!"); -} - -extern "C" LLVMRustLinkage LLVMRustGetLinkage(LLVMValueRef V) { - return toRust(LLVMGetLinkage(V)); -} - -extern "C" void LLVMRustSetLinkage(LLVMValueRef V, - LLVMRustLinkage RustLinkage) { - LLVMSetLinkage(V, fromRust(RustLinkage)); -} - -extern "C" LLVMValueRef LLVMRustConstInBoundsGEP2(LLVMTypeRef Ty, - LLVMValueRef ConstantVal, - LLVMValueRef *ConstantIndices, - unsigned NumIndices) { - ArrayRef IdxList(unwrap(ConstantIndices, NumIndices), - NumIndices); - Constant *Val = unwrap(ConstantVal); - return wrap(ConstantExpr::getInBoundsGetElementPtr(unwrap(Ty), Val, IdxList)); -} - -// Returns true if both high and low were successfully set. Fails in case constant wasn’t any of -// the common sizes (1, 8, 16, 32, 64, 128 bits) -extern "C" bool LLVMRustConstInt128Get(LLVMValueRef CV, bool sext, uint64_t *high, uint64_t *low) -{ - auto C = unwrap(CV); - if (C->getBitWidth() > 128) { return false; } - APInt AP; -#if LLVM_VERSION_GE(15, 0) - if (sext) { - AP = C->getValue().sext(128); - } else { - AP = C->getValue().zext(128); - } -#else - if (sext) { - AP = C->getValue().sextOrSelf(128); - } else { - AP = C->getValue().zextOrSelf(128); - } -#endif - *low = AP.getLoBits(64).getZExtValue(); - *high = AP.getHiBits(64).getZExtValue(); - return true; -} - -enum class LLVMRustVisibility { - Default = 0, - Hidden = 1, - Protected = 2, -}; - -static LLVMRustVisibility toRust(LLVMVisibility Vis) { - switch (Vis) { - case LLVMDefaultVisibility: - return LLVMRustVisibility::Default; - case LLVMHiddenVisibility: - return LLVMRustVisibility::Hidden; - case LLVMProtectedVisibility: - return LLVMRustVisibility::Protected; - } - report_fatal_error("Invalid LLVMRustVisibility value!"); -} - -static LLVMVisibility fromRust(LLVMRustVisibility Vis) { - switch (Vis) { - case LLVMRustVisibility::Default: - return LLVMDefaultVisibility; - case LLVMRustVisibility::Hidden: - return LLVMHiddenVisibility; - case LLVMRustVisibility::Protected: - return LLVMProtectedVisibility; - } - report_fatal_error("Invalid LLVMRustVisibility value!"); -} - -extern "C" LLVMRustVisibility LLVMRustGetVisibility(LLVMValueRef V) { - return toRust(LLVMGetVisibility(V)); -} - -// Oh hey, a binding that makes sense for once? (because LLVM’s own do not) -extern "C" LLVMValueRef LLVMRustBuildIntCast(LLVMBuilderRef B, LLVMValueRef Val, - LLVMTypeRef DestTy, bool isSigned) { - return wrap(unwrap(B)->CreateIntCast(unwrap(Val), unwrap(DestTy), isSigned, "")); -} - -extern "C" void LLVMRustSetVisibility(LLVMValueRef V, - LLVMRustVisibility RustVisibility) { - LLVMSetVisibility(V, fromRust(RustVisibility)); -} - -extern "C" void LLVMRustSetDSOLocal(LLVMValueRef Global, bool is_dso_local) { - unwrap(Global)->setDSOLocal(is_dso_local); -} - -struct LLVMRustModuleBuffer { - std::string data; -}; - -extern "C" LLVMRustModuleBuffer* -LLVMRustModuleBufferCreate(LLVMModuleRef M) { - auto Ret = std::make_unique(); - { - raw_string_ostream OS(Ret->data); - { - legacy::PassManager PM; - PM.add(createBitcodeWriterPass(OS)); - PM.run(*unwrap(M)); - } - } - return Ret.release(); -} - -extern "C" void -LLVMRustModuleBufferFree(LLVMRustModuleBuffer *Buffer) { - delete Buffer; -} - -extern "C" const void* -LLVMRustModuleBufferPtr(const LLVMRustModuleBuffer *Buffer) { - return Buffer->data.data(); -} - -extern "C" size_t -LLVMRustModuleBufferLen(const LLVMRustModuleBuffer *Buffer) { - return Buffer->data.length(); -} - -extern "C" uint64_t -LLVMRustModuleCost(LLVMModuleRef M) { - auto f = unwrap(M)->functions(); - return std::distance(std::begin(f), std::end(f)); -} - -// Vector reductions: -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceFAdd(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { - return wrap(unwrap(B)->CreateFAddReduce(unwrap(Acc),unwrap(Src))); -} -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceFMul(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { - return wrap(unwrap(B)->CreateFMulReduce(unwrap(Acc),unwrap(Src))); -} -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceAdd(LLVMBuilderRef B, LLVMValueRef Src) { - return wrap(unwrap(B)->CreateAddReduce(unwrap(Src))); -} -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceMul(LLVMBuilderRef B, LLVMValueRef Src) { - return wrap(unwrap(B)->CreateMulReduce(unwrap(Src))); -} -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceAnd(LLVMBuilderRef B, LLVMValueRef Src) { - return wrap(unwrap(B)->CreateAndReduce(unwrap(Src))); -} -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceOr(LLVMBuilderRef B, LLVMValueRef Src) { - return wrap(unwrap(B)->CreateOrReduce(unwrap(Src))); -} -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceXor(LLVMBuilderRef B, LLVMValueRef Src) { - return wrap(unwrap(B)->CreateXorReduce(unwrap(Src))); -} -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceMin(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) { - return wrap(unwrap(B)->CreateIntMinReduce(unwrap(Src), IsSigned)); -} -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceMax(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) { - return wrap(unwrap(B)->CreateIntMaxReduce(unwrap(Src), IsSigned)); -} -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceFMin(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { - Instruction *I = unwrap(B)->CreateFPMinReduce(unwrap(Src)); - I->setHasNoNaNs(NoNaN); - return wrap(I); -} -extern "C" LLVMValueRef -LLVMRustBuildVectorReduceFMax(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { - Instruction *I = unwrap(B)->CreateFPMaxReduce(unwrap(Src)); - I->setHasNoNaNs(NoNaN); - return wrap(I); -} - -extern "C" LLVMValueRef -LLVMRustBuildMinNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { - return wrap(unwrap(B)->CreateMinNum(unwrap(LHS),unwrap(RHS))); -} -extern "C" LLVMValueRef -LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { - return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS),unwrap(RHS))); -} - -// This struct contains all necessary info about a symbol exported from a DLL. -struct LLVMRustCOFFShortExport { - const char* name; - bool ordinal_present; - // The value of `ordinal` is only meaningful if `ordinal_present` is true. - uint16_t ordinal; -}; - -// Machine must be a COFF machine type, as defined in PE specs. -extern "C" LLVMRustResult LLVMRustWriteImportLibrary( - const char* ImportName, - const char* Path, - const LLVMRustCOFFShortExport* Exports, - size_t NumExports, - uint16_t Machine, - bool MinGW) -{ - std::vector ConvertedExports; - ConvertedExports.reserve(NumExports); - - for (size_t i = 0; i < NumExports; ++i) { - bool ordinal_present = Exports[i].ordinal_present; - uint16_t ordinal = ordinal_present ? Exports[i].ordinal : 0; - ConvertedExports.push_back(llvm::object::COFFShortExport{ - Exports[i].name, // Name - std::string{}, // ExtName - std::string{}, // SymbolName - std::string{}, // AliasTarget - ordinal, // Ordinal - ordinal_present, // Noname - false, // Data - false, // Private - false // Constant - }); - } - - auto Error = llvm::object::writeImportLibrary( - ImportName, - Path, - ConvertedExports, - static_cast(Machine), - MinGW); - if (Error) { - std::string errorString; - llvm::raw_string_ostream stream(errorString); - stream << Error; - stream.flush(); - LLVMRustSetLastError(errorString.c_str()); - return LLVMRustResult::Failure; - } else { - return LLVMRustResult::Success; - } -} - -// Transfers ownership of DiagnosticHandler unique_ptr to the caller. -extern "C" DiagnosticHandler * -LLVMRustContextGetDiagnosticHandler(LLVMContextRef C) { - std::unique_ptr DH = unwrap(C)->getDiagnosticHandler(); - return DH.release(); -} - -// Sets unique_ptr to object of DiagnosticHandler to provide custom diagnostic -// handling. Ownership of the handler is moved to the LLVMContext. -extern "C" void LLVMRustContextSetDiagnosticHandler(LLVMContextRef C, - DiagnosticHandler *DH) { - unwrap(C)->setDiagnosticHandler(std::unique_ptr(DH)); -} - -using LLVMDiagnosticHandlerTy = DiagnosticHandler::DiagnosticHandlerTy; - -// Configures a diagnostic handler that invokes provided callback when a -// backend needs to emit a diagnostic. -// -// When RemarkAllPasses is true, remarks are enabled for all passes. Otherwise -// the RemarkPasses array specifies individual passes for which remarks will be -// enabled. -extern "C" void LLVMRustContextConfigureDiagnosticHandler( - LLVMContextRef C, LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, - void *DiagnosticHandlerContext, bool RemarkAllPasses, - const char * const * RemarkPasses, size_t RemarkPassesLen) { - - class RustDiagnosticHandler final : public DiagnosticHandler { - public: - RustDiagnosticHandler(LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, - void *DiagnosticHandlerContext, - bool RemarkAllPasses, - std::vector RemarkPasses) - : DiagnosticHandlerCallback(DiagnosticHandlerCallback), - DiagnosticHandlerContext(DiagnosticHandlerContext), - RemarkAllPasses(RemarkAllPasses), - RemarkPasses(RemarkPasses) {} - - virtual bool handleDiagnostics(const DiagnosticInfo &DI) override { - if (DiagnosticHandlerCallback) { - DiagnosticHandlerCallback(DI, DiagnosticHandlerContext); - return true; - } - return false; - } - - bool isAnalysisRemarkEnabled(StringRef PassName) const override { - return isRemarkEnabled(PassName); - } - - bool isMissedOptRemarkEnabled(StringRef PassName) const override { - return isRemarkEnabled(PassName); - } - - bool isPassedOptRemarkEnabled(StringRef PassName) const override { - return isRemarkEnabled(PassName); - } - - bool isAnyRemarkEnabled() const override { - return RemarkAllPasses || !RemarkPasses.empty(); - } - - private: - bool isRemarkEnabled(StringRef PassName) const { - if (RemarkAllPasses) - return true; - - for (auto &Pass : RemarkPasses) - if (Pass == PassName) - return true; - - return false; - } - - LLVMDiagnosticHandlerTy DiagnosticHandlerCallback = nullptr; - void *DiagnosticHandlerContext = nullptr; - - bool RemarkAllPasses = false; - std::vector RemarkPasses; - }; - - std::vector Passes; - for (size_t I = 0; I != RemarkPassesLen; ++I) - Passes.push_back(RemarkPasses[I]); - - unwrap(C)->setDiagnosticHandler(std::make_unique( - DiagnosticHandlerCallback, DiagnosticHandlerContext, RemarkAllPasses, Passes)); -} - -extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) { - RawRustStringOstream OS(Str); - GlobalValue *GV = unwrap(V); - Mangler().getNameWithPrefix(OS, GV, true); -} - -// LLVMGetAggregateElement was added in LLVM 15. For earlier LLVM versions just -// use its implementation. -#if LLVM_VERSION_LT(15, 0) -extern "C" LLVMValueRef LLVMGetAggregateElement(LLVMValueRef C, unsigned Idx) { - return wrap(unwrap(C)->getAggregateElement(Idx)); -} -#endif - -extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) { -#if LLVM_VERSION_GE(15, 0) - auto *CB = unwrap(CallSite); - switch (CB->getIntrinsicID()) { - case Intrinsic::arm_ldrex: - return 0; - case Intrinsic::arm_strex: - return 1; - } -#endif - return -1; -} +// #include "LLVMWrapper.h" +// #include "llvm/IR/DebugInfoMetadata.h" +// #include "llvm/IR/DiagnosticHandler.h" +// #include "llvm/IR/DiagnosticInfo.h" +// #include "llvm/IR/DiagnosticPrinter.h" +// #include "llvm/IR/GlobalVariable.h" +// #include "llvm/IR/Instructions.h" +// #include "llvm/IR/Intrinsics.h" +// #include "llvm/IR/IntrinsicsARM.h" +// #include "llvm/IR/Mangler.h" +// #include "llvm/Object/Archive.h" +// #include "llvm/Object/COFFImportFile.h" +// #include "llvm/Object/ObjectFile.h" +// #include "llvm/Pass.h" +// #include "llvm/Bitcode/BitcodeWriterPass.h" +// #include "llvm/Support/Signals.h" +// #include "llvm/ADT/Optional.h" + +// #include + +// //===----------------------------------------------------------------------=== +// // +// // This file defines alternate interfaces to core functions that are more +// // readily callable by Rust's FFI. +// // +// //===----------------------------------------------------------------------=== + +// using namespace llvm; +// using namespace llvm::sys; +// using namespace llvm::object; + +// // LLVMAtomicOrdering is already an enum - don't create another +// // one. +// static AtomicOrdering fromRust(LLVMAtomicOrdering Ordering) { +// switch (Ordering) { +// case LLVMAtomicOrderingNotAtomic: +// return AtomicOrdering::NotAtomic; +// case LLVMAtomicOrderingUnordered: +// return AtomicOrdering::Unordered; +// case LLVMAtomicOrderingMonotonic: +// return AtomicOrdering::Monotonic; +// case LLVMAtomicOrderingAcquire: +// return AtomicOrdering::Acquire; +// case LLVMAtomicOrderingRelease: +// return AtomicOrdering::Release; +// case LLVMAtomicOrderingAcquireRelease: +// return AtomicOrdering::AcquireRelease; +// case LLVMAtomicOrderingSequentiallyConsistent: +// return AtomicOrdering::SequentiallyConsistent; +// } + +// report_fatal_error("Invalid LLVMAtomicOrdering value!"); +// } + +// static LLVM_THREAD_LOCAL char *LastError; + +// // Custom error handler for fatal LLVM errors. +// // +// // Notably it exits the process with code 101, unlike LLVM's default of 1. +// static void FatalErrorHandler(void *UserData, +// #if LLVM_VERSION_LT(14, 0) +// const std::string& Reason, +// #else +// const char* Reason, +// #endif +// bool GenCrashDiag) { +// // Do the same thing that the default error handler does. +// std::cerr << "LLVM ERROR: " << Reason << std::endl; + +// // Since this error handler exits the process, we have to run any cleanup that +// // LLVM would run after handling the error. This might change with an LLVM +// // upgrade. +// sys::RunInterruptHandlers(); + +// exit(101); +// } + +// extern "C" void LLVMRustInstallFatalErrorHandler() { +// install_fatal_error_handler(FatalErrorHandler); +// } + +// extern "C" void LLVMRustDisableSystemDialogsOnCrash() { +// sys::DisableSystemDialogsOnCrash(); +// } + +// extern "C" char *LLVMRustGetLastError(void) { +// char *Ret = LastError; +// LastError = nullptr; +// return Ret; +// } + +// extern "C" void LLVMRustSetLastError(const char *Err) { +// free((void *)LastError); +// LastError = strdup(Err); +// } + +// extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames) { +// auto ctx = new LLVMContext(); +// ctx->setDiscardValueNames(shouldDiscardNames); +// return wrap(ctx); +// } + +// extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, +// const char *Triple) { +// unwrap(M)->setTargetTriple(Triple::normalize(Triple)); +// } + +// extern "C" void LLVMRustPrintPassTimings() { +// raw_fd_ostream OS(2, false); // stderr. +// TimerGroup::printAll(OS); +// } + +// extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, +// size_t NameLen) { +// return wrap(unwrap(M)->getNamedValue(StringRef(Name, NameLen))); +// } + +// extern "C" LLVMValueRef LLVMRustGetOrInsertFunction(LLVMModuleRef M, +// const char *Name, +// size_t NameLen, +// LLVMTypeRef FunctionTy) { +// return wrap(unwrap(M) +// ->getOrInsertFunction(StringRef(Name, NameLen), +// unwrap(FunctionTy)) +// .getCallee() +// ); +// } + +// extern "C" LLVMValueRef +// LLVMRustGetOrInsertGlobal(LLVMModuleRef M, const char *Name, size_t NameLen, LLVMTypeRef Ty) { +// Module *Mod = unwrap(M); +// StringRef NameRef(Name, NameLen); + +// // We don't use Module::getOrInsertGlobal because that returns a Constant*, +// // which may either be the real GlobalVariable*, or a constant bitcast of it +// // if our type doesn't match the original declaration. We always want the +// // GlobalVariable* so we can access linkage, visibility, etc. +// GlobalVariable *GV = Mod->getGlobalVariable(NameRef, true); +// if (!GV) +// GV = new GlobalVariable(*Mod, unwrap(Ty), false, +// GlobalValue::ExternalLinkage, nullptr, NameRef); +// return wrap(GV); +// } + +// extern "C" LLVMValueRef +// LLVMRustInsertPrivateGlobal(LLVMModuleRef M, LLVMTypeRef Ty) { +// return wrap(new GlobalVariable(*unwrap(M), +// unwrap(Ty), +// false, +// GlobalValue::PrivateLinkage, +// nullptr)); +// } + +// extern "C" LLVMTypeRef LLVMRustMetadataTypeInContext(LLVMContextRef C) { +// return wrap(Type::getMetadataTy(*unwrap(C))); +// } + +// static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { +// switch (Kind) { +// case AlwaysInline: +// return Attribute::AlwaysInline; +// case ByVal: +// return Attribute::ByVal; +// case Cold: +// return Attribute::Cold; +// case InlineHint: +// return Attribute::InlineHint; +// case MinSize: +// return Attribute::MinSize; +// case Naked: +// return Attribute::Naked; +// case NoAlias: +// return Attribute::NoAlias; +// case NoCapture: +// return Attribute::NoCapture; +// case NoCfCheck: +// return Attribute::NoCfCheck; +// case NoInline: +// return Attribute::NoInline; +// case NonNull: +// return Attribute::NonNull; +// case NoRedZone: +// return Attribute::NoRedZone; +// case NoReturn: +// return Attribute::NoReturn; +// case NoUnwind: +// return Attribute::NoUnwind; +// case OptimizeForSize: +// return Attribute::OptimizeForSize; +// case ReadOnly: +// return Attribute::ReadOnly; +// case SExt: +// return Attribute::SExt; +// case StructRet: +// return Attribute::StructRet; +// case UWTable: +// return Attribute::UWTable; +// case ZExt: +// return Attribute::ZExt; +// case InReg: +// return Attribute::InReg; +// case SanitizeThread: +// return Attribute::SanitizeThread; +// case SanitizeAddress: +// return Attribute::SanitizeAddress; +// case SanitizeMemory: +// return Attribute::SanitizeMemory; +// case NonLazyBind: +// return Attribute::NonLazyBind; +// case OptimizeNone: +// return Attribute::OptimizeNone; +// case ReturnsTwice: +// return Attribute::ReturnsTwice; +// case ReadNone: +// return Attribute::ReadNone; +// case InaccessibleMemOnly: +// return Attribute::InaccessibleMemOnly; +// case SanitizeHWAddress: +// return Attribute::SanitizeHWAddress; +// case WillReturn: +// return Attribute::WillReturn; +// case StackProtectReq: +// return Attribute::StackProtectReq; +// case StackProtectStrong: +// return Attribute::StackProtectStrong; +// case StackProtect: +// return Attribute::StackProtect; +// case NoUndef: +// return Attribute::NoUndef; +// case SanitizeMemTag: +// return Attribute::SanitizeMemTag; +// case ShadowCallStack: +// return Attribute::ShadowCallStack; +// case AllocSize: +// return Attribute::AllocSize; +// #if LLVM_VERSION_GE(15, 0) +// case AllocatedPointer: +// return Attribute::AllocatedPointer; +// case AllocAlign: +// return Attribute::AllocAlign; +// #endif +// } +// report_fatal_error("bad AttributeKind"); +// } + +// template static inline void AddAttributes(T *t, unsigned Index, +// LLVMAttributeRef *Attrs, size_t AttrsLen) { +// AttributeList PAL = t->getAttributes(); +// AttributeList PALNew; +// #if LLVM_VERSION_LT(14, 0) +// AttrBuilder B; +// for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen)) +// B.addAttribute(unwrap(Attr)); +// PALNew = PAL.addAttributes(t->getContext(), Index, B); +// #else +// AttrBuilder B(t->getContext()); +// for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen)) +// B.addAttribute(unwrap(Attr)); +// PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B); +// #endif +// t->setAttributes(PALNew); +// } + +// extern "C" void LLVMRustAddFunctionAttributes(LLVMValueRef Fn, unsigned Index, +// LLVMAttributeRef *Attrs, size_t AttrsLen) { +// Function *F = unwrap(Fn); +// AddAttributes(F, Index, Attrs, AttrsLen); +// } + +// extern "C" void LLVMRustAddCallSiteAttributes(LLVMValueRef Instr, unsigned Index, +// LLVMAttributeRef *Attrs, size_t AttrsLen) { +// CallBase *Call = unwrap(Instr); +// AddAttributes(Call, Index, Attrs, AttrsLen); +// } + +// extern "C" LLVMAttributeRef LLVMRustCreateAttrNoValue(LLVMContextRef C, +// LLVMRustAttribute RustAttr) { +// return wrap(Attribute::get(*unwrap(C), fromRust(RustAttr))); +// } + +// extern "C" LLVMAttributeRef LLVMRustCreateAlignmentAttr(LLVMContextRef C, +// uint64_t Bytes) { +// return wrap(Attribute::getWithAlignment(*unwrap(C), llvm::Align(Bytes))); +// } + +// extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableAttr(LLVMContextRef C, +// uint64_t Bytes) { +// return wrap(Attribute::getWithDereferenceableBytes(*unwrap(C), Bytes)); +// } + +// extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableOrNullAttr(LLVMContextRef C, +// uint64_t Bytes) { +// return wrap(Attribute::getWithDereferenceableOrNullBytes(*unwrap(C), Bytes)); +// } + +// extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C, LLVMTypeRef Ty) { +// return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty))); +// } + +// extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C, LLVMTypeRef Ty) { +// return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty))); +// } + +// extern "C" LLVMAttributeRef LLVMRustCreateElementTypeAttr(LLVMContextRef C, LLVMTypeRef Ty) { +// #if LLVM_VERSION_GE(15, 0) +// return wrap(Attribute::get(*unwrap(C), Attribute::ElementType, unwrap(Ty))); +// #else +// report_fatal_error("Should not be needed on LLVM < 15"); +// #endif +// } + +// extern "C" LLVMAttributeRef LLVMRustCreateUWTableAttr(LLVMContextRef C, bool Async) { +// #if LLVM_VERSION_LT(15, 0) +// return wrap(Attribute::get(*unwrap(C), Attribute::UWTable)); +// #else +// return wrap(Attribute::getWithUWTableKind( +// *unwrap(C), Async ? UWTableKind::Async : UWTableKind::Sync)); +// #endif +// } + +// extern "C" LLVMAttributeRef LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) { +// return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, None)); +// } + +// #if LLVM_VERSION_GE(15, 0) + +// // These values **must** match ffi::AllocKindFlags. +// // It _happens_ to match the LLVM values of llvm::AllocFnKind, +// // but that's happenstance and we do explicit conversions before +// // passing them to LLVM. +// enum class LLVMRustAllocKindFlags : uint64_t { +// Unknown = 0, +// Alloc = 1, +// Realloc = 1 << 1, +// Free = 1 << 2, +// Uninitialized = 1 << 3, +// Zeroed = 1 << 4, +// Aligned = 1 << 5, +// }; + +// static LLVMRustAllocKindFlags operator&(LLVMRustAllocKindFlags A, LLVMRustAllocKindFlags B) { +// return static_cast(static_cast(A) & +// static_cast(B)); +// } + +// static bool isSet(LLVMRustAllocKindFlags F) { return F != LLVMRustAllocKindFlags::Unknown; } + +// static llvm::AllocFnKind allocKindFromRust(LLVMRustAllocKindFlags F) { +// llvm::AllocFnKind AFK = llvm::AllocFnKind::Unknown; +// if (isSet(F & LLVMRustAllocKindFlags::Alloc)) { +// AFK |= llvm::AllocFnKind::Alloc; +// } +// if (isSet(F & LLVMRustAllocKindFlags::Realloc)) { +// AFK |= llvm::AllocFnKind::Realloc; +// } +// if (isSet(F & LLVMRustAllocKindFlags::Free)) { +// AFK |= llvm::AllocFnKind::Free; +// } +// if (isSet(F & LLVMRustAllocKindFlags::Uninitialized)) { +// AFK |= llvm::AllocFnKind::Uninitialized; +// } +// if (isSet(F & LLVMRustAllocKindFlags::Zeroed)) { +// AFK |= llvm::AllocFnKind::Zeroed; +// } +// if (isSet(F & LLVMRustAllocKindFlags::Aligned)) { +// AFK |= llvm::AllocFnKind::Aligned; +// } +// return AFK; +// } +// #endif + +// extern "C" LLVMAttributeRef LLVMRustCreateAllocKindAttr(LLVMContextRef C, uint64_t AllocKindArg) { +// #if LLVM_VERSION_GE(15, 0) +// return wrap(Attribute::get(*unwrap(C), Attribute::AllocKind, +// static_cast(allocKindFromRust(static_cast(AllocKindArg))))); +// #else +// report_fatal_error( +// "allockind attributes are new in LLVM 15 and should not be used on older LLVMs"); +// #endif +// } + +// // Enable a fast-math flag +// // +// // https://llvm.org/docs/LangRef.html#fast-math-flags +// extern "C" void LLVMRustSetFastMath(LLVMValueRef V) { +// if (auto I = dyn_cast(unwrap(V))) { +// I->setFast(true); +// } +// } + +// extern "C" LLVMValueRef +// LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source, +// const char *Name, LLVMAtomicOrdering Order) { +// Value *Ptr = unwrap(Source); +// LoadInst *LI = unwrap(B)->CreateLoad(unwrap(Ty), Ptr, Name); +// LI->setAtomic(fromRust(Order)); +// return wrap(LI); +// } + +// extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B, +// LLVMValueRef V, +// LLVMValueRef Target, +// LLVMAtomicOrdering Order) { +// StoreInst *SI = unwrap(B)->CreateStore(unwrap(V), unwrap(Target)); +// SI->setAtomic(fromRust(Order)); +// return wrap(SI); +// } + +// // FIXME: Use the C-API LLVMBuildAtomicCmpXchg and LLVMSetWeak +// // once we raise our minimum support to LLVM 10. +// extern "C" LLVMValueRef +// LLVMRustBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Target, +// LLVMValueRef Old, LLVMValueRef Source, +// LLVMAtomicOrdering Order, +// LLVMAtomicOrdering FailureOrder, LLVMBool Weak) { +// #if LLVM_VERSION_GE(13,0) +// // Rust probably knows the alignment of the target value and should be able to +// // specify something more precise than MaybeAlign here. See also +// // https://reviews.llvm.org/D97224 which may be a useful reference. +// AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg( +// unwrap(Target), unwrap(Old), unwrap(Source), llvm::MaybeAlign(), fromRust(Order), +// fromRust(FailureOrder)); +// #else +// AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg( +// unwrap(Target), unwrap(Old), unwrap(Source), fromRust(Order), +// fromRust(FailureOrder)); +// #endif +// ACXI->setWeak(Weak); +// return wrap(ACXI); +// } + +// enum class LLVMRustSynchronizationScope { +// SingleThread, +// CrossThread, +// }; + +// static SyncScope::ID fromRust(LLVMRustSynchronizationScope Scope) { +// switch (Scope) { +// case LLVMRustSynchronizationScope::SingleThread: +// return SyncScope::SingleThread; +// case LLVMRustSynchronizationScope::CrossThread: +// return SyncScope::System; +// default: +// report_fatal_error("bad SynchronizationScope."); +// } +// } + +// extern "C" LLVMValueRef +// LLVMRustBuildAtomicFence(LLVMBuilderRef B, LLVMAtomicOrdering Order, +// LLVMRustSynchronizationScope Scope) { +// return wrap(unwrap(B)->CreateFence(fromRust(Order), fromRust(Scope))); +// } + +// enum class LLVMRustAsmDialect { +// Att, +// Intel, +// }; + +// static InlineAsm::AsmDialect fromRust(LLVMRustAsmDialect Dialect) { +// switch (Dialect) { +// case LLVMRustAsmDialect::Att: +// return InlineAsm::AD_ATT; +// case LLVMRustAsmDialect::Intel: +// return InlineAsm::AD_Intel; +// default: +// report_fatal_error("bad AsmDialect."); +// } +// } + +// extern "C" LLVMValueRef +// LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen, +// char *Constraints, size_t ConstraintsLen, +// LLVMBool HasSideEffects, LLVMBool IsAlignStack, +// LLVMRustAsmDialect Dialect, LLVMBool CanThrow) { +// #if LLVM_VERSION_GE(13, 0) +// return wrap(InlineAsm::get(unwrap(Ty), +// StringRef(AsmString, AsmStringLen), +// StringRef(Constraints, ConstraintsLen), +// HasSideEffects, IsAlignStack, +// fromRust(Dialect), CanThrow)); +// #else +// return wrap(InlineAsm::get(unwrap(Ty), +// StringRef(AsmString, AsmStringLen), +// StringRef(Constraints, ConstraintsLen), +// HasSideEffects, IsAlignStack, +// fromRust(Dialect))); +// #endif +// } + +// extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints, +// size_t ConstraintsLen) { +// #if LLVM_VERSION_LT(15, 0) +// return InlineAsm::Verify(unwrap(Ty), +// StringRef(Constraints, ConstraintsLen)); +// #else +// // llvm::Error converts to true if it is an error. +// return !llvm::errorToBool(InlineAsm::verify( +// unwrap(Ty), StringRef(Constraints, ConstraintsLen))); +// #endif +// } + +// extern "C" void LLVMRustAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm, +// size_t AsmLen) { +// unwrap(M)->appendModuleInlineAsm(StringRef(Asm, AsmLen)); +// } + +// typedef DIBuilder *LLVMRustDIBuilderRef; + +// template DIT *unwrapDIPtr(LLVMMetadataRef Ref) { +// return (DIT *)(Ref ? unwrap(Ref) : nullptr); +// } + +// #define DIDescriptor DIScope +// #define DIArray DINodeArray +// #define unwrapDI unwrapDIPtr + +// // These values **must** match debuginfo::DIFlags! They also *happen* +// // to match LLVM, but that isn't required as we do giant sets of +// // matching below. The value shouldn't be directly passed to LLVM. +// enum class LLVMRustDIFlags : uint32_t { +// FlagZero = 0, +// FlagPrivate = 1, +// FlagProtected = 2, +// FlagPublic = 3, +// FlagFwdDecl = (1 << 2), +// FlagAppleBlock = (1 << 3), +// FlagBlockByrefStruct = (1 << 4), +// FlagVirtual = (1 << 5), +// FlagArtificial = (1 << 6), +// FlagExplicit = (1 << 7), +// FlagPrototyped = (1 << 8), +// FlagObjcClassComplete = (1 << 9), +// FlagObjectPointer = (1 << 10), +// FlagVector = (1 << 11), +// FlagStaticMember = (1 << 12), +// FlagLValueReference = (1 << 13), +// FlagRValueReference = (1 << 14), +// FlagExternalTypeRef = (1 << 15), +// FlagIntroducedVirtual = (1 << 18), +// FlagBitField = (1 << 19), +// FlagNoReturn = (1 << 20), +// // Do not add values that are not supported by the minimum LLVM +// // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def +// }; + +// inline LLVMRustDIFlags operator&(LLVMRustDIFlags A, LLVMRustDIFlags B) { +// return static_cast(static_cast(A) & +// static_cast(B)); +// } + +// inline LLVMRustDIFlags operator|(LLVMRustDIFlags A, LLVMRustDIFlags B) { +// return static_cast(static_cast(A) | +// static_cast(B)); +// } + +// inline LLVMRustDIFlags &operator|=(LLVMRustDIFlags &A, LLVMRustDIFlags B) { +// return A = A | B; +// } + +// inline bool isSet(LLVMRustDIFlags F) { return F != LLVMRustDIFlags::FlagZero; } + +// inline LLVMRustDIFlags visibility(LLVMRustDIFlags F) { +// return static_cast(static_cast(F) & 0x3); +// } + +// static DINode::DIFlags fromRust(LLVMRustDIFlags Flags) { +// DINode::DIFlags Result = DINode::DIFlags::FlagZero; + +// switch (visibility(Flags)) { +// case LLVMRustDIFlags::FlagPrivate: +// Result |= DINode::DIFlags::FlagPrivate; +// break; +// case LLVMRustDIFlags::FlagProtected: +// Result |= DINode::DIFlags::FlagProtected; +// break; +// case LLVMRustDIFlags::FlagPublic: +// Result |= DINode::DIFlags::FlagPublic; +// break; +// default: +// // The rest are handled below +// break; +// } + +// if (isSet(Flags & LLVMRustDIFlags::FlagFwdDecl)) { +// Result |= DINode::DIFlags::FlagFwdDecl; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagAppleBlock)) { +// Result |= DINode::DIFlags::FlagAppleBlock; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagVirtual)) { +// Result |= DINode::DIFlags::FlagVirtual; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagArtificial)) { +// Result |= DINode::DIFlags::FlagArtificial; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagExplicit)) { +// Result |= DINode::DIFlags::FlagExplicit; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagPrototyped)) { +// Result |= DINode::DIFlags::FlagPrototyped; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagObjcClassComplete)) { +// Result |= DINode::DIFlags::FlagObjcClassComplete; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagObjectPointer)) { +// Result |= DINode::DIFlags::FlagObjectPointer; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagVector)) { +// Result |= DINode::DIFlags::FlagVector; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagStaticMember)) { +// Result |= DINode::DIFlags::FlagStaticMember; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagLValueReference)) { +// Result |= DINode::DIFlags::FlagLValueReference; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagRValueReference)) { +// Result |= DINode::DIFlags::FlagRValueReference; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagIntroducedVirtual)) { +// Result |= DINode::DIFlags::FlagIntroducedVirtual; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagBitField)) { +// Result |= DINode::DIFlags::FlagBitField; +// } +// if (isSet(Flags & LLVMRustDIFlags::FlagNoReturn)) { +// Result |= DINode::DIFlags::FlagNoReturn; +// } + +// return Result; +// } + +// // These values **must** match debuginfo::DISPFlags! They also *happen* +// // to match LLVM, but that isn't required as we do giant sets of +// // matching below. The value shouldn't be directly passed to LLVM. +// enum class LLVMRustDISPFlags : uint32_t { +// SPFlagZero = 0, +// SPFlagVirtual = 1, +// SPFlagPureVirtual = 2, +// SPFlagLocalToUnit = (1 << 2), +// SPFlagDefinition = (1 << 3), +// SPFlagOptimized = (1 << 4), +// SPFlagMainSubprogram = (1 << 5), +// // Do not add values that are not supported by the minimum LLVM +// // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def +// // (In LLVM < 8, createFunction supported these as separate bool arguments.) +// }; + +// inline LLVMRustDISPFlags operator&(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { +// return static_cast(static_cast(A) & +// static_cast(B)); +// } + +// inline LLVMRustDISPFlags operator|(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { +// return static_cast(static_cast(A) | +// static_cast(B)); +// } + +// inline LLVMRustDISPFlags &operator|=(LLVMRustDISPFlags &A, LLVMRustDISPFlags B) { +// return A = A | B; +// } + +// inline bool isSet(LLVMRustDISPFlags F) { return F != LLVMRustDISPFlags::SPFlagZero; } + +// inline LLVMRustDISPFlags virtuality(LLVMRustDISPFlags F) { +// return static_cast(static_cast(F) & 0x3); +// } + +// static DISubprogram::DISPFlags fromRust(LLVMRustDISPFlags SPFlags) { +// DISubprogram::DISPFlags Result = DISubprogram::DISPFlags::SPFlagZero; + +// switch (virtuality(SPFlags)) { +// case LLVMRustDISPFlags::SPFlagVirtual: +// Result |= DISubprogram::DISPFlags::SPFlagVirtual; +// break; +// case LLVMRustDISPFlags::SPFlagPureVirtual: +// Result |= DISubprogram::DISPFlags::SPFlagPureVirtual; +// break; +// default: +// // The rest are handled below +// break; +// } + +// if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagLocalToUnit)) { +// Result |= DISubprogram::DISPFlags::SPFlagLocalToUnit; +// } +// if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagDefinition)) { +// Result |= DISubprogram::DISPFlags::SPFlagDefinition; +// } +// if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagOptimized)) { +// Result |= DISubprogram::DISPFlags::SPFlagOptimized; +// } +// if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagMainSubprogram)) { +// Result |= DISubprogram::DISPFlags::SPFlagMainSubprogram; +// } + +// return Result; +// } + +// enum class LLVMRustDebugEmissionKind { +// NoDebug, +// FullDebug, +// LineTablesOnly, +// }; + +// static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind) { +// switch (Kind) { +// case LLVMRustDebugEmissionKind::NoDebug: +// return DICompileUnit::DebugEmissionKind::NoDebug; +// case LLVMRustDebugEmissionKind::FullDebug: +// return DICompileUnit::DebugEmissionKind::FullDebug; +// case LLVMRustDebugEmissionKind::LineTablesOnly: +// return DICompileUnit::DebugEmissionKind::LineTablesOnly; +// default: +// report_fatal_error("bad DebugEmissionKind."); +// } +// } + +// enum class LLVMRustChecksumKind { +// None, +// MD5, +// SHA1, +// SHA256, +// }; + +// static Optional fromRust(LLVMRustChecksumKind Kind) { +// switch (Kind) { +// case LLVMRustChecksumKind::None: +// return None; +// case LLVMRustChecksumKind::MD5: +// return DIFile::ChecksumKind::CSK_MD5; +// case LLVMRustChecksumKind::SHA1: +// return DIFile::ChecksumKind::CSK_SHA1; +// case LLVMRustChecksumKind::SHA256: +// return DIFile::ChecksumKind::CSK_SHA256; +// default: +// report_fatal_error("bad ChecksumKind."); +// } +// } + +// extern "C" uint32_t LLVMRustDebugMetadataVersion() { +// return DEBUG_METADATA_VERSION; +// } + +// extern "C" uint32_t LLVMRustVersionPatch() { return LLVM_VERSION_PATCH; } + +// extern "C" uint32_t LLVMRustVersionMinor() { return LLVM_VERSION_MINOR; } + +// extern "C" uint32_t LLVMRustVersionMajor() { return LLVM_VERSION_MAJOR; } + +// extern "C" void LLVMRustAddModuleFlag( +// LLVMModuleRef M, +// Module::ModFlagBehavior MergeBehavior, +// const char *Name, +// uint32_t Value) { +// unwrap(M)->addModuleFlag(MergeBehavior, Name, Value); +// } + +// extern "C" bool LLVMRustHasModuleFlag(LLVMModuleRef M, const char *Name, +// size_t Len) { +// return unwrap(M)->getModuleFlag(StringRef(Name, Len)) != nullptr; +// } + +// extern "C" LLVMValueRef LLVMRustMetadataAsValue(LLVMContextRef C, LLVMMetadataRef MD) { +// return wrap(MetadataAsValue::get(*unwrap(C), unwrap(MD))); +// } + +// extern "C" void LLVMRustGlobalAddMetadata( +// LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) { +// unwrap(Global)->addMetadata(Kind, *unwrap(MD)); +// } + +// extern "C" LLVMRustDIBuilderRef LLVMRustDIBuilderCreate(LLVMModuleRef M) { +// return new DIBuilder(*unwrap(M)); +// } + +// extern "C" void LLVMRustDIBuilderDispose(LLVMRustDIBuilderRef Builder) { +// delete Builder; +// } + +// extern "C" void LLVMRustDIBuilderFinalize(LLVMRustDIBuilderRef Builder) { +// Builder->finalize(); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateCompileUnit( +// LLVMRustDIBuilderRef Builder, unsigned Lang, LLVMMetadataRef FileRef, +// const char *Producer, size_t ProducerLen, bool isOptimized, +// const char *Flags, unsigned RuntimeVer, +// const char *SplitName, size_t SplitNameLen, +// LLVMRustDebugEmissionKind Kind, +// uint64_t DWOId, bool SplitDebugInlining) { +// auto *File = unwrapDI(FileRef); + +// return wrap(Builder->createCompileUnit(Lang, File, StringRef(Producer, ProducerLen), +// isOptimized, Flags, RuntimeVer, +// StringRef(SplitName, SplitNameLen), +// fromRust(Kind), DWOId, SplitDebugInlining)); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFile( +// LLVMRustDIBuilderRef Builder, +// const char *Filename, size_t FilenameLen, +// const char *Directory, size_t DirectoryLen, LLVMRustChecksumKind CSKind, +// const char *Checksum, size_t ChecksumLen) { +// Optional llvmCSKind = fromRust(CSKind); +// Optional> CSInfo{}; +// if (llvmCSKind) +// CSInfo.emplace(*llvmCSKind, StringRef{Checksum, ChecksumLen}); +// return wrap(Builder->createFile(StringRef(Filename, FilenameLen), +// StringRef(Directory, DirectoryLen), +// CSInfo)); +// } + +// extern "C" LLVMMetadataRef +// LLVMRustDIBuilderCreateSubroutineType(LLVMRustDIBuilderRef Builder, +// LLVMMetadataRef ParameterTypes) { +// return wrap(Builder->createSubroutineType( +// DITypeRefArray(unwrap(ParameterTypes)))); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, +// const char *Name, size_t NameLen, +// const char *LinkageName, size_t LinkageNameLen, +// LLVMMetadataRef File, unsigned LineNo, +// LLVMMetadataRef Ty, unsigned ScopeLine, LLVMRustDIFlags Flags, +// LLVMRustDISPFlags SPFlags, LLVMValueRef MaybeFn, LLVMMetadataRef TParam, +// LLVMMetadataRef Decl) { +// DITemplateParameterArray TParams = +// DITemplateParameterArray(unwrap(TParam)); +// DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); +// DINode::DIFlags llvmFlags = fromRust(Flags); +// DISubprogram *Sub = Builder->createFunction( +// unwrapDI(Scope), +// StringRef(Name, NameLen), +// StringRef(LinkageName, LinkageNameLen), +// unwrapDI(File), LineNo, +// unwrapDI(Ty), ScopeLine, llvmFlags, +// llvmSPFlags, TParams, unwrapDIPtr(Decl)); +// if (MaybeFn) +// unwrap(MaybeFn)->setSubprogram(Sub); +// return wrap(Sub); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateBasicType( +// LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, +// uint64_t SizeInBits, unsigned Encoding) { +// return wrap(Builder->createBasicType(StringRef(Name, NameLen), SizeInBits, Encoding)); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTypedef( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Type, const char *Name, size_t NameLen, +// LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Scope) { +// return wrap(Builder->createTypedef( +// unwrap(Type), StringRef(Name, NameLen), unwrap(File), +// LineNo, unwrapDIPtr(Scope))); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef PointeeTy, +// uint64_t SizeInBits, uint32_t AlignInBits, unsigned AddressSpace, +// const char *Name, size_t NameLen) { +// return wrap(Builder->createPointerType(unwrapDI(PointeeTy), +// SizeInBits, AlignInBits, +// AddressSpace, +// StringRef(Name, NameLen))); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStructType( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, +// const char *Name, size_t NameLen, +// LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, +// uint32_t AlignInBits, LLVMRustDIFlags Flags, +// LLVMMetadataRef DerivedFrom, LLVMMetadataRef Elements, +// unsigned RunTimeLang, LLVMMetadataRef VTableHolder, +// const char *UniqueId, size_t UniqueIdLen) { +// return wrap(Builder->createStructType( +// unwrapDI(Scope), StringRef(Name, NameLen), +// unwrapDI(File), LineNumber, +// SizeInBits, AlignInBits, fromRust(Flags), unwrapDI(DerivedFrom), +// DINodeArray(unwrapDI(Elements)), RunTimeLang, +// unwrapDI(VTableHolder), StringRef(UniqueId, UniqueIdLen))); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantPart( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, +// const char *Name, size_t NameLen, +// LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, +// uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Discriminator, +// LLVMMetadataRef Elements, const char *UniqueId, size_t UniqueIdLen) { +// return wrap(Builder->createVariantPart( +// unwrapDI(Scope), StringRef(Name, NameLen), +// unwrapDI(File), LineNumber, +// SizeInBits, AlignInBits, fromRust(Flags), unwrapDI(Discriminator), +// DINodeArray(unwrapDI(Elements)), StringRef(UniqueId, UniqueIdLen))); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMemberType( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, +// const char *Name, size_t NameLen, +// LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits, +// uint32_t AlignInBits, uint64_t OffsetInBits, LLVMRustDIFlags Flags, +// LLVMMetadataRef Ty) { +// return wrap(Builder->createMemberType(unwrapDI(Scope), +// StringRef(Name, NameLen), +// unwrapDI(File), LineNo, +// SizeInBits, AlignInBits, OffsetInBits, +// fromRust(Flags), unwrapDI(Ty))); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, +// const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo, +// uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, LLVMValueRef Discriminant, +// LLVMRustDIFlags Flags, LLVMMetadataRef Ty) { +// llvm::ConstantInt* D = nullptr; +// if (Discriminant) { +// D = unwrap(Discriminant); +// } +// return wrap(Builder->createVariantMemberType(unwrapDI(Scope), +// StringRef(Name, NameLen), +// unwrapDI(File), LineNo, +// SizeInBits, AlignInBits, OffsetInBits, D, +// fromRust(Flags), unwrapDI(Ty))); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateLexicalBlock( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, +// LLVMMetadataRef File, unsigned Line, unsigned Col) { +// return wrap(Builder->createLexicalBlock(unwrapDI(Scope), +// unwrapDI(File), Line, Col)); +// } + +// extern "C" LLVMMetadataRef +// LLVMRustDIBuilderCreateLexicalBlockFile(LLVMRustDIBuilderRef Builder, +// LLVMMetadataRef Scope, +// LLVMMetadataRef File) { +// return wrap(Builder->createLexicalBlockFile(unwrapDI(Scope), +// unwrapDI(File))); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Context, +// const char *Name, size_t NameLen, +// const char *LinkageName, size_t LinkageNameLen, +// LLVMMetadataRef File, unsigned LineNo, +// LLVMMetadataRef Ty, bool IsLocalToUnit, LLVMValueRef V, +// LLVMMetadataRef Decl = nullptr, uint32_t AlignInBits = 0) { +// llvm::GlobalVariable *InitVal = cast(unwrap(V)); + +// llvm::DIExpression *InitExpr = nullptr; +// if (llvm::ConstantInt *IntVal = llvm::dyn_cast(InitVal)) { +// InitExpr = Builder->createConstantValueExpression( +// IntVal->getValue().getSExtValue()); +// } else if (llvm::ConstantFP *FPVal = +// llvm::dyn_cast(InitVal)) { +// InitExpr = Builder->createConstantValueExpression( +// FPVal->getValueAPF().bitcastToAPInt().getZExtValue()); +// } + +// llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression( +// unwrapDI(Context), StringRef(Name, NameLen), +// StringRef(LinkageName, LinkageNameLen), +// unwrapDI(File), LineNo, unwrapDI(Ty), IsLocalToUnit, +// /* isDefined */ true, +// InitExpr, unwrapDIPtr(Decl), +// /* templateParams */ nullptr, +// AlignInBits); + +// InitVal->setMetadata("dbg", VarExpr); + +// return wrap(VarExpr); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable( +// LLVMRustDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope, +// const char *Name, size_t NameLen, +// LLVMMetadataRef File, unsigned LineNo, +// LLVMMetadataRef Ty, bool AlwaysPreserve, LLVMRustDIFlags Flags, +// unsigned ArgNo, uint32_t AlignInBits) { +// if (Tag == 0x100) { // DW_TAG_auto_variable +// return wrap(Builder->createAutoVariable( +// unwrapDI(Scope), StringRef(Name, NameLen), +// unwrapDI(File), LineNo, +// unwrapDI(Ty), AlwaysPreserve, fromRust(Flags), AlignInBits)); +// } else { +// return wrap(Builder->createParameterVariable( +// unwrapDI(Scope), StringRef(Name, NameLen), ArgNo, +// unwrapDI(File), LineNo, +// unwrapDI(Ty), AlwaysPreserve, fromRust(Flags))); +// } +// } + +// extern "C" LLVMMetadataRef +// LLVMRustDIBuilderCreateArrayType(LLVMRustDIBuilderRef Builder, uint64_t Size, +// uint32_t AlignInBits, LLVMMetadataRef Ty, +// LLVMMetadataRef Subscripts) { +// return wrap( +// Builder->createArrayType(Size, AlignInBits, unwrapDI(Ty), +// DINodeArray(unwrapDI(Subscripts)))); +// } + +// extern "C" LLVMMetadataRef +// LLVMRustDIBuilderGetOrCreateSubrange(LLVMRustDIBuilderRef Builder, int64_t Lo, +// int64_t Count) { +// return wrap(Builder->getOrCreateSubrange(Lo, Count)); +// } + +// extern "C" LLVMMetadataRef +// LLVMRustDIBuilderGetOrCreateArray(LLVMRustDIBuilderRef Builder, +// LLVMMetadataRef *Ptr, unsigned Count) { +// Metadata **DataValue = unwrap(Ptr); +// return wrap( +// Builder->getOrCreateArray(ArrayRef(DataValue, Count)).get()); +// } + +// extern "C" LLVMValueRef LLVMRustDIBuilderInsertDeclareAtEnd( +// LLVMRustDIBuilderRef Builder, LLVMValueRef V, LLVMMetadataRef VarInfo, +// uint64_t *AddrOps, unsigned AddrOpsCount, LLVMMetadataRef DL, +// LLVMBasicBlockRef InsertAtEnd) { +// return wrap(Builder->insertDeclare( +// unwrap(V), unwrap(VarInfo), +// Builder->createExpression(llvm::ArrayRef(AddrOps, AddrOpsCount)), +// DebugLoc(cast(unwrap(DL))), +// unwrap(InsertAtEnd))); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerator( +// LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, +// int64_t Value, bool IsUnsigned) { +// return wrap(Builder->createEnumerator(StringRef(Name, NameLen), Value, IsUnsigned)); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, +// const char *Name, size_t NameLen, +// LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, +// uint32_t AlignInBits, LLVMMetadataRef Elements, +// LLVMMetadataRef ClassTy, bool IsScoped) { +// return wrap(Builder->createEnumerationType( +// unwrapDI(Scope), StringRef(Name, NameLen), +// unwrapDI(File), LineNumber, +// SizeInBits, AlignInBits, DINodeArray(unwrapDI(Elements)), +// unwrapDI(ClassTy), "", IsScoped)); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, +// const char *Name, size_t NameLen, +// LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, +// uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Elements, +// unsigned RunTimeLang, const char *UniqueId, size_t UniqueIdLen) { +// return wrap(Builder->createUnionType( +// unwrapDI(Scope), StringRef(Name, NameLen), unwrapDI(File), +// LineNumber, SizeInBits, AlignInBits, fromRust(Flags), +// DINodeArray(unwrapDI(Elements)), RunTimeLang, +// StringRef(UniqueId, UniqueIdLen))); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTemplateTypeParameter( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, +// const char *Name, size_t NameLen, LLVMMetadataRef Ty) { +// bool IsDefault = false; // FIXME: should we ever set this true? +// return wrap(Builder->createTemplateTypeParameter( +// unwrapDI(Scope), StringRef(Name, NameLen), unwrapDI(Ty), IsDefault)); +// } + +// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateNameSpace( +// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, +// const char *Name, size_t NameLen, bool ExportSymbols) { +// return wrap(Builder->createNameSpace( +// unwrapDI(Scope), StringRef(Name, NameLen), ExportSymbols +// )); +// } + +// extern "C" void +// LLVMRustDICompositeTypeReplaceArrays(LLVMRustDIBuilderRef Builder, +// LLVMMetadataRef CompositeTy, +// LLVMMetadataRef Elements, +// LLVMMetadataRef Params) { +// DICompositeType *Tmp = unwrapDI(CompositeTy); +// Builder->replaceArrays(Tmp, DINodeArray(unwrap(Elements)), +// DINodeArray(unwrap(Params))); +// } + +// extern "C" LLVMMetadataRef +// LLVMRustDIBuilderCreateDebugLocation(unsigned Line, unsigned Column, +// LLVMMetadataRef ScopeRef, +// LLVMMetadataRef InlinedAt) { +// MDNode *Scope = unwrapDIPtr(ScopeRef); +// DILocation *Loc = DILocation::get( +// Scope->getContext(), Line, Column, Scope, +// unwrapDIPtr(InlinedAt)); +// return wrap(Loc); +// } + +// extern "C" uint64_t LLVMRustDIBuilderCreateOpDeref() { +// return dwarf::DW_OP_deref; +// } + +// extern "C" uint64_t LLVMRustDIBuilderCreateOpPlusUconst() { +// return dwarf::DW_OP_plus_uconst; +// } + +// extern "C" void LLVMRustWriteTypeToString(LLVMTypeRef Ty, RustStringRef Str) { +// RawRustStringOstream OS(Str); +// unwrap(Ty)->print(OS); +// } + +// extern "C" void LLVMRustWriteValueToString(LLVMValueRef V, +// RustStringRef Str) { +// RawRustStringOstream OS(Str); +// if (!V) { +// OS << "(null)"; +// } else { +// OS << "("; +// unwrap(V)->getType()->print(OS); +// OS << ":"; +// unwrap(V)->print(OS); +// OS << ")"; +// } +// } + +// // LLVMArrayType function does not support 64-bit ElementCount +// extern "C" LLVMTypeRef LLVMRustArrayType(LLVMTypeRef ElementTy, +// uint64_t ElementCount) { +// return wrap(ArrayType::get(unwrap(ElementTy), ElementCount)); +// } + +// DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Twine, LLVMTwineRef) + +// extern "C" void LLVMRustWriteTwineToString(LLVMTwineRef T, RustStringRef Str) { +// RawRustStringOstream OS(Str); +// unwrap(T)->print(OS); +// } + +// extern "C" void LLVMRustUnpackOptimizationDiagnostic( +// LLVMDiagnosticInfoRef DI, RustStringRef PassNameOut, +// LLVMValueRef *FunctionOut, unsigned* Line, unsigned* Column, +// RustStringRef FilenameOut, RustStringRef MessageOut) { +// // Undefined to call this not on an optimization diagnostic! +// llvm::DiagnosticInfoOptimizationBase *Opt = +// static_cast(unwrap(DI)); + +// RawRustStringOstream PassNameOS(PassNameOut); +// PassNameOS << Opt->getPassName(); +// *FunctionOut = wrap(&Opt->getFunction()); + +// RawRustStringOstream FilenameOS(FilenameOut); +// DiagnosticLocation loc = Opt->getLocation(); +// if (loc.isValid()) { +// *Line = loc.getLine(); +// *Column = loc.getColumn(); +// FilenameOS << loc.getAbsolutePath(); +// } + +// RawRustStringOstream MessageOS(MessageOut); +// MessageOS << Opt->getMsg(); +// } + +// enum class LLVMRustDiagnosticLevel { +// Error, +// Warning, +// Note, +// Remark, +// }; + +// extern "C" void +// LLVMRustUnpackInlineAsmDiagnostic(LLVMDiagnosticInfoRef DI, +// LLVMRustDiagnosticLevel *LevelOut, +// unsigned *CookieOut, +// LLVMTwineRef *MessageOut) { +// // Undefined to call this not on an inline assembly diagnostic! +// llvm::DiagnosticInfoInlineAsm *IA = +// static_cast(unwrap(DI)); + +// *CookieOut = IA->getLocCookie(); +// *MessageOut = wrap(&IA->getMsgStr()); + +// switch (IA->getSeverity()) { +// case DS_Error: +// *LevelOut = LLVMRustDiagnosticLevel::Error; +// break; +// case DS_Warning: +// *LevelOut = LLVMRustDiagnosticLevel::Warning; +// break; +// case DS_Note: +// *LevelOut = LLVMRustDiagnosticLevel::Note; +// break; +// case DS_Remark: +// *LevelOut = LLVMRustDiagnosticLevel::Remark; +// break; +// default: +// report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); +// } +// } + +// extern "C" void LLVMRustWriteDiagnosticInfoToString(LLVMDiagnosticInfoRef DI, +// RustStringRef Str) { +// RawRustStringOstream OS(Str); +// DiagnosticPrinterRawOStream DP(OS); +// unwrap(DI)->print(DP); +// } + +// enum class LLVMRustDiagnosticKind { +// Other, +// InlineAsm, +// StackSize, +// DebugMetadataVersion, +// SampleProfile, +// OptimizationRemark, +// OptimizationRemarkMissed, +// OptimizationRemarkAnalysis, +// OptimizationRemarkAnalysisFPCommute, +// OptimizationRemarkAnalysisAliasing, +// OptimizationRemarkOther, +// OptimizationFailure, +// PGOProfile, +// Linker, +// Unsupported, +// SrcMgr, +// }; + +// static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) { +// switch (Kind) { +// case DK_InlineAsm: +// return LLVMRustDiagnosticKind::InlineAsm; +// case DK_StackSize: +// return LLVMRustDiagnosticKind::StackSize; +// case DK_DebugMetadataVersion: +// return LLVMRustDiagnosticKind::DebugMetadataVersion; +// case DK_SampleProfile: +// return LLVMRustDiagnosticKind::SampleProfile; +// case DK_OptimizationRemark: +// case DK_MachineOptimizationRemark: +// return LLVMRustDiagnosticKind::OptimizationRemark; +// case DK_OptimizationRemarkMissed: +// case DK_MachineOptimizationRemarkMissed: +// return LLVMRustDiagnosticKind::OptimizationRemarkMissed; +// case DK_OptimizationRemarkAnalysis: +// case DK_MachineOptimizationRemarkAnalysis: +// return LLVMRustDiagnosticKind::OptimizationRemarkAnalysis; +// case DK_OptimizationRemarkAnalysisFPCommute: +// return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisFPCommute; +// case DK_OptimizationRemarkAnalysisAliasing: +// return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisAliasing; +// case DK_PGOProfile: +// return LLVMRustDiagnosticKind::PGOProfile; +// case DK_Linker: +// return LLVMRustDiagnosticKind::Linker; +// case DK_Unsupported: +// return LLVMRustDiagnosticKind::Unsupported; +// #if LLVM_VERSION_GE(13, 0) +// case DK_SrcMgr: +// return LLVMRustDiagnosticKind::SrcMgr; +// #endif +// default: +// return (Kind >= DK_FirstRemark && Kind <= DK_LastRemark) +// ? LLVMRustDiagnosticKind::OptimizationRemarkOther +// : LLVMRustDiagnosticKind::Other; +// } +// } + +// extern "C" LLVMRustDiagnosticKind +// LLVMRustGetDiagInfoKind(LLVMDiagnosticInfoRef DI) { +// return toRust((DiagnosticKind)unwrap(DI)->getKind()); +// } + +// // This is kept distinct from LLVMGetTypeKind, because when +// // a new type kind is added, the Rust-side enum must be +// // updated or UB will result. +// extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) { +// switch (unwrap(Ty)->getTypeID()) { +// case Type::VoidTyID: +// return LLVMVoidTypeKind; +// case Type::HalfTyID: +// return LLVMHalfTypeKind; +// case Type::FloatTyID: +// return LLVMFloatTypeKind; +// case Type::DoubleTyID: +// return LLVMDoubleTypeKind; +// case Type::X86_FP80TyID: +// return LLVMX86_FP80TypeKind; +// case Type::FP128TyID: +// return LLVMFP128TypeKind; +// case Type::PPC_FP128TyID: +// return LLVMPPC_FP128TypeKind; +// case Type::LabelTyID: +// return LLVMLabelTypeKind; +// case Type::MetadataTyID: +// return LLVMMetadataTypeKind; +// case Type::IntegerTyID: +// return LLVMIntegerTypeKind; +// case Type::FunctionTyID: +// return LLVMFunctionTypeKind; +// case Type::StructTyID: +// return LLVMStructTypeKind; +// case Type::ArrayTyID: +// return LLVMArrayTypeKind; +// case Type::PointerTyID: +// return LLVMPointerTypeKind; +// case Type::FixedVectorTyID: +// return LLVMVectorTypeKind; +// case Type::X86_MMXTyID: +// return LLVMX86_MMXTypeKind; +// case Type::TokenTyID: +// return LLVMTokenTypeKind; +// case Type::ScalableVectorTyID: +// return LLVMScalableVectorTypeKind; +// case Type::BFloatTyID: +// return LLVMBFloatTypeKind; +// case Type::X86_AMXTyID: +// return LLVMX86_AMXTypeKind; +// #if LLVM_VERSION_GE(15, 0) && LLVM_VERSION_LT(16, 0) +// case Type::DXILPointerTyID: +// report_fatal_error("Rust does not support DirectX typed pointers."); +// break; +// #endif +// #if LLVM_VERSION_GE(16, 0) +// case Type::TypedPointerTyID: +// report_fatal_error("Rust does not support typed pointers."); +// break; +// #endif +// } +// report_fatal_error("Unhandled TypeID."); +// } + +// DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef) + +// #if LLVM_VERSION_LT(13, 0) +// using LLVMInlineAsmDiagHandlerTy = LLVMContext::InlineAsmDiagHandlerTy; +// #else +// using LLVMInlineAsmDiagHandlerTy = void*; +// #endif + +// extern "C" void LLVMRustSetInlineAsmDiagnosticHandler( +// LLVMContextRef C, LLVMInlineAsmDiagHandlerTy H, void *CX) { +// // Diagnostic handlers were unified in LLVM change 5de2d189e6ad, so starting +// // with LLVM 13 this function is gone. +// #if LLVM_VERSION_LT(13, 0) +// unwrap(C)->setInlineAsmDiagnosticHandler(H, CX); +// #endif +// } + +// extern "C" LLVMSMDiagnosticRef LLVMRustGetSMDiagnostic( +// LLVMDiagnosticInfoRef DI, unsigned *Cookie) { +// #if LLVM_VERSION_GE(13, 0) +// llvm::DiagnosticInfoSrcMgr *SM = static_cast(unwrap(DI)); +// *Cookie = SM->getLocCookie(); +// return wrap(&SM->getSMDiag()); +// #else +// report_fatal_error("Shouldn't get called on older versions"); +// #endif +// } + +// extern "C" bool LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef, +// RustStringRef MessageOut, +// RustStringRef BufferOut, +// LLVMRustDiagnosticLevel* LevelOut, +// unsigned* LocOut, +// unsigned* RangesOut, +// size_t* NumRanges) { +// SMDiagnostic& D = *unwrap(DRef); +// RawRustStringOstream MessageOS(MessageOut); +// MessageOS << D.getMessage(); + +// switch (D.getKind()) { +// case SourceMgr::DK_Error: +// *LevelOut = LLVMRustDiagnosticLevel::Error; +// break; +// case SourceMgr::DK_Warning: +// *LevelOut = LLVMRustDiagnosticLevel::Warning; +// break; +// case SourceMgr::DK_Note: +// *LevelOut = LLVMRustDiagnosticLevel::Note; +// break; +// case SourceMgr::DK_Remark: +// *LevelOut = LLVMRustDiagnosticLevel::Remark; +// break; +// default: +// report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); +// } + +// if (D.getLoc() == SMLoc()) +// return false; + +// const SourceMgr &LSM = *D.getSourceMgr(); +// const MemoryBuffer *LBuf = LSM.getMemoryBuffer(LSM.FindBufferContainingLoc(D.getLoc())); +// LLVMRustStringWriteImpl(BufferOut, LBuf->getBufferStart(), LBuf->getBufferSize()); + +// *LocOut = D.getLoc().getPointer() - LBuf->getBufferStart(); + +// *NumRanges = std::min(*NumRanges, D.getRanges().size()); +// size_t LineStart = *LocOut - (size_t)D.getColumnNo(); +// for (size_t i = 0; i < *NumRanges; i++) { +// RangesOut[i * 2] = LineStart + D.getRanges()[i].first; +// RangesOut[i * 2 + 1] = LineStart + D.getRanges()[i].second; +// } + +// return true; +// } + +// extern "C" LLVMValueRef LLVMRustBuildCleanupPad(LLVMBuilderRef B, +// LLVMValueRef ParentPad, +// unsigned ArgCount, +// LLVMValueRef *LLArgs, +// const char *Name) { +// Value **Args = unwrap(LLArgs); +// if (ParentPad == nullptr) { +// Type *Ty = Type::getTokenTy(unwrap(B)->getContext()); +// ParentPad = wrap(Constant::getNullValue(Ty)); +// } +// return wrap(unwrap(B)->CreateCleanupPad( +// unwrap(ParentPad), ArrayRef(Args, ArgCount), Name)); +// } + +// extern "C" LLVMValueRef LLVMRustBuildCleanupRet(LLVMBuilderRef B, +// LLVMValueRef CleanupPad, +// LLVMBasicBlockRef UnwindBB) { +// CleanupPadInst *Inst = cast(unwrap(CleanupPad)); +// return wrap(unwrap(B)->CreateCleanupRet(Inst, unwrap(UnwindBB))); +// } + +// extern "C" LLVMValueRef +// LLVMRustBuildCatchPad(LLVMBuilderRef B, LLVMValueRef ParentPad, +// unsigned ArgCount, LLVMValueRef *LLArgs, const char *Name) { +// Value **Args = unwrap(LLArgs); +// return wrap(unwrap(B)->CreateCatchPad( +// unwrap(ParentPad), ArrayRef(Args, ArgCount), Name)); +// } + +// extern "C" LLVMValueRef LLVMRustBuildCatchRet(LLVMBuilderRef B, +// LLVMValueRef Pad, +// LLVMBasicBlockRef BB) { +// return wrap(unwrap(B)->CreateCatchRet(cast(unwrap(Pad)), +// unwrap(BB))); +// } + +// extern "C" LLVMValueRef LLVMRustBuildCatchSwitch(LLVMBuilderRef B, +// LLVMValueRef ParentPad, +// LLVMBasicBlockRef BB, +// unsigned NumHandlers, +// const char *Name) { +// if (ParentPad == nullptr) { +// Type *Ty = Type::getTokenTy(unwrap(B)->getContext()); +// ParentPad = wrap(Constant::getNullValue(Ty)); +// } +// return wrap(unwrap(B)->CreateCatchSwitch(unwrap(ParentPad), unwrap(BB), +// NumHandlers, Name)); +// } + +// extern "C" void LLVMRustAddHandler(LLVMValueRef CatchSwitchRef, +// LLVMBasicBlockRef Handler) { +// Value *CatchSwitch = unwrap(CatchSwitchRef); +// cast(CatchSwitch)->addHandler(unwrap(Handler)); +// } + +// extern "C" OperandBundleDef *LLVMRustBuildOperandBundleDef(const char *Name, +// LLVMValueRef *Inputs, +// unsigned NumInputs) { +// return new OperandBundleDef(Name, makeArrayRef(unwrap(Inputs), NumInputs)); +// } + +// extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { +// delete Bundle; +// } + +// extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, +// LLVMValueRef *Args, unsigned NumArgs, +// OperandBundleDef *Bundle) { +// Value *Callee = unwrap(Fn); +// FunctionType *FTy = unwrap(Ty); +// unsigned Len = Bundle ? 1 : 0; +// ArrayRef Bundles = makeArrayRef(Bundle, Len); +// return wrap(unwrap(B)->CreateCall( +// FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), Bundles)); +// } + +// extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) { +// return wrap(llvm::Intrinsic::getDeclaration(unwrap(M), +// (llvm::Intrinsic::ID)llvm::Intrinsic::instrprof_increment)); +// } + +// extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, +// LLVMValueRef Dst, unsigned DstAlign, +// LLVMValueRef Src, unsigned SrcAlign, +// LLVMValueRef Size, bool IsVolatile) { +// return wrap(unwrap(B)->CreateMemCpy( +// unwrap(Dst), MaybeAlign(DstAlign), +// unwrap(Src), MaybeAlign(SrcAlign), +// unwrap(Size), IsVolatile)); +// } + +// extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B, +// LLVMValueRef Dst, unsigned DstAlign, +// LLVMValueRef Src, unsigned SrcAlign, +// LLVMValueRef Size, bool IsVolatile) { +// return wrap(unwrap(B)->CreateMemMove( +// unwrap(Dst), MaybeAlign(DstAlign), +// unwrap(Src), MaybeAlign(SrcAlign), +// unwrap(Size), IsVolatile)); +// } + +// extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, +// LLVMValueRef Dst, unsigned DstAlign, +// LLVMValueRef Val, +// LLVMValueRef Size, bool IsVolatile) { +// return wrap(unwrap(B)->CreateMemSet( +// unwrap(Dst), unwrap(Val), unwrap(Size), MaybeAlign(DstAlign), IsVolatile)); +// } + +// extern "C" LLVMValueRef +// LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, +// LLVMValueRef *Args, unsigned NumArgs, +// LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, +// OperandBundleDef *Bundle, const char *Name) { +// Value *Callee = unwrap(Fn); +// FunctionType *FTy = unwrap(Ty); +// unsigned Len = Bundle ? 1 : 0; +// ArrayRef Bundles = makeArrayRef(Bundle, Len); +// return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch), +// makeArrayRef(unwrap(Args), NumArgs), +// Bundles, Name)); +// } + +// extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, +// LLVMBasicBlockRef BB) { +// auto Point = unwrap(BB)->getFirstInsertionPt(); +// unwrap(B)->SetInsertPoint(unwrap(BB), Point); +// } + +// extern "C" void LLVMRustSetComdat(LLVMModuleRef M, LLVMValueRef V, +// const char *Name, size_t NameLen) { +// Triple TargetTriple(unwrap(M)->getTargetTriple()); +// GlobalObject *GV = unwrap(V); +// if (TargetTriple.supportsCOMDAT()) { +// StringRef NameRef(Name, NameLen); +// GV->setComdat(unwrap(M)->getOrInsertComdat(NameRef)); +// } +// } + +// enum class LLVMRustLinkage { +// ExternalLinkage = 0, +// AvailableExternallyLinkage = 1, +// LinkOnceAnyLinkage = 2, +// LinkOnceODRLinkage = 3, +// WeakAnyLinkage = 4, +// WeakODRLinkage = 5, +// AppendingLinkage = 6, +// InternalLinkage = 7, +// PrivateLinkage = 8, +// ExternalWeakLinkage = 9, +// CommonLinkage = 10, +// }; + +// static LLVMRustLinkage toRust(LLVMLinkage Linkage) { +// switch (Linkage) { +// case LLVMExternalLinkage: +// return LLVMRustLinkage::ExternalLinkage; +// case LLVMAvailableExternallyLinkage: +// return LLVMRustLinkage::AvailableExternallyLinkage; +// case LLVMLinkOnceAnyLinkage: +// return LLVMRustLinkage::LinkOnceAnyLinkage; +// case LLVMLinkOnceODRLinkage: +// return LLVMRustLinkage::LinkOnceODRLinkage; +// case LLVMWeakAnyLinkage: +// return LLVMRustLinkage::WeakAnyLinkage; +// case LLVMWeakODRLinkage: +// return LLVMRustLinkage::WeakODRLinkage; +// case LLVMAppendingLinkage: +// return LLVMRustLinkage::AppendingLinkage; +// case LLVMInternalLinkage: +// return LLVMRustLinkage::InternalLinkage; +// case LLVMPrivateLinkage: +// return LLVMRustLinkage::PrivateLinkage; +// case LLVMExternalWeakLinkage: +// return LLVMRustLinkage::ExternalWeakLinkage; +// case LLVMCommonLinkage: +// return LLVMRustLinkage::CommonLinkage; +// default: +// report_fatal_error("Invalid LLVMRustLinkage value!"); +// } +// } + +// static LLVMLinkage fromRust(LLVMRustLinkage Linkage) { +// switch (Linkage) { +// case LLVMRustLinkage::ExternalLinkage: +// return LLVMExternalLinkage; +// case LLVMRustLinkage::AvailableExternallyLinkage: +// return LLVMAvailableExternallyLinkage; +// case LLVMRustLinkage::LinkOnceAnyLinkage: +// return LLVMLinkOnceAnyLinkage; +// case LLVMRustLinkage::LinkOnceODRLinkage: +// return LLVMLinkOnceODRLinkage; +// case LLVMRustLinkage::WeakAnyLinkage: +// return LLVMWeakAnyLinkage; +// case LLVMRustLinkage::WeakODRLinkage: +// return LLVMWeakODRLinkage; +// case LLVMRustLinkage::AppendingLinkage: +// return LLVMAppendingLinkage; +// case LLVMRustLinkage::InternalLinkage: +// return LLVMInternalLinkage; +// case LLVMRustLinkage::PrivateLinkage: +// return LLVMPrivateLinkage; +// case LLVMRustLinkage::ExternalWeakLinkage: +// return LLVMExternalWeakLinkage; +// case LLVMRustLinkage::CommonLinkage: +// return LLVMCommonLinkage; +// } +// report_fatal_error("Invalid LLVMRustLinkage value!"); +// } + +// extern "C" LLVMRustLinkage LLVMRustGetLinkage(LLVMValueRef V) { +// return toRust(LLVMGetLinkage(V)); +// } + +// extern "C" void LLVMRustSetLinkage(LLVMValueRef V, +// LLVMRustLinkage RustLinkage) { +// LLVMSetLinkage(V, fromRust(RustLinkage)); +// } + +// extern "C" LLVMValueRef LLVMRustConstInBoundsGEP2(LLVMTypeRef Ty, +// LLVMValueRef ConstantVal, +// LLVMValueRef *ConstantIndices, +// unsigned NumIndices) { +// ArrayRef IdxList(unwrap(ConstantIndices, NumIndices), +// NumIndices); +// Constant *Val = unwrap(ConstantVal); +// return wrap(ConstantExpr::getInBoundsGetElementPtr(unwrap(Ty), Val, IdxList)); +// } + +// // Returns true if both high and low were successfully set. Fails in case constant wasn’t any of +// // the common sizes (1, 8, 16, 32, 64, 128 bits) +// extern "C" bool LLVMRustConstInt128Get(LLVMValueRef CV, bool sext, uint64_t *high, uint64_t *low) +// { +// auto C = unwrap(CV); +// if (C->getBitWidth() > 128) { return false; } +// APInt AP; +// #if LLVM_VERSION_GE(15, 0) +// if (sext) { +// AP = C->getValue().sext(128); +// } else { +// AP = C->getValue().zext(128); +// } +// #else +// if (sext) { +// AP = C->getValue().sextOrSelf(128); +// } else { +// AP = C->getValue().zextOrSelf(128); +// } +// #endif +// *low = AP.getLoBits(64).getZExtValue(); +// *high = AP.getHiBits(64).getZExtValue(); +// return true; +// } + +// enum class LLVMRustVisibility { +// Default = 0, +// Hidden = 1, +// Protected = 2, +// }; + +// static LLVMRustVisibility toRust(LLVMVisibility Vis) { +// switch (Vis) { +// case LLVMDefaultVisibility: +// return LLVMRustVisibility::Default; +// case LLVMHiddenVisibility: +// return LLVMRustVisibility::Hidden; +// case LLVMProtectedVisibility: +// return LLVMRustVisibility::Protected; +// } +// report_fatal_error("Invalid LLVMRustVisibility value!"); +// } + +// static LLVMVisibility fromRust(LLVMRustVisibility Vis) { +// switch (Vis) { +// case LLVMRustVisibility::Default: +// return LLVMDefaultVisibility; +// case LLVMRustVisibility::Hidden: +// return LLVMHiddenVisibility; +// case LLVMRustVisibility::Protected: +// return LLVMProtectedVisibility; +// } +// report_fatal_error("Invalid LLVMRustVisibility value!"); +// } + +// extern "C" LLVMRustVisibility LLVMRustGetVisibility(LLVMValueRef V) { +// return toRust(LLVMGetVisibility(V)); +// } + +// // Oh hey, a binding that makes sense for once? (because LLVM’s own do not) +// extern "C" LLVMValueRef LLVMRustBuildIntCast(LLVMBuilderRef B, LLVMValueRef Val, +// LLVMTypeRef DestTy, bool isSigned) { +// return wrap(unwrap(B)->CreateIntCast(unwrap(Val), unwrap(DestTy), isSigned, "")); +// } + +// extern "C" void LLVMRustSetVisibility(LLVMValueRef V, +// LLVMRustVisibility RustVisibility) { +// LLVMSetVisibility(V, fromRust(RustVisibility)); +// } + +// extern "C" void LLVMRustSetDSOLocal(LLVMValueRef Global, bool is_dso_local) { +// unwrap(Global)->setDSOLocal(is_dso_local); +// } + +// struct LLVMRustModuleBuffer { +// std::string data; +// }; + +// extern "C" LLVMRustModuleBuffer* +// LLVMRustModuleBufferCreate(LLVMModuleRef M) { +// auto Ret = std::make_unique(); +// { +// raw_string_ostream OS(Ret->data); +// { +// legacy::PassManager PM; +// PM.add(createBitcodeWriterPass(OS)); +// PM.run(*unwrap(M)); +// } +// } +// return Ret.release(); +// } + +// extern "C" void +// LLVMRustModuleBufferFree(LLVMRustModuleBuffer *Buffer) { +// delete Buffer; +// } + +// extern "C" const void* +// LLVMRustModuleBufferPtr(const LLVMRustModuleBuffer *Buffer) { +// return Buffer->data.data(); +// } + +// extern "C" size_t +// LLVMRustModuleBufferLen(const LLVMRustModuleBuffer *Buffer) { +// return Buffer->data.length(); +// } + +// extern "C" uint64_t +// LLVMRustModuleCost(LLVMModuleRef M) { +// auto f = unwrap(M)->functions(); +// return std::distance(std::begin(f), std::end(f)); +// } + +// // Vector reductions: +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceFAdd(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { +// return wrap(unwrap(B)->CreateFAddReduce(unwrap(Acc),unwrap(Src))); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceFMul(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { +// return wrap(unwrap(B)->CreateFMulReduce(unwrap(Acc),unwrap(Src))); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceAdd(LLVMBuilderRef B, LLVMValueRef Src) { +// return wrap(unwrap(B)->CreateAddReduce(unwrap(Src))); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceMul(LLVMBuilderRef B, LLVMValueRef Src) { +// return wrap(unwrap(B)->CreateMulReduce(unwrap(Src))); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceAnd(LLVMBuilderRef B, LLVMValueRef Src) { +// return wrap(unwrap(B)->CreateAndReduce(unwrap(Src))); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceOr(LLVMBuilderRef B, LLVMValueRef Src) { +// return wrap(unwrap(B)->CreateOrReduce(unwrap(Src))); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceXor(LLVMBuilderRef B, LLVMValueRef Src) { +// return wrap(unwrap(B)->CreateXorReduce(unwrap(Src))); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceMin(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) { +// return wrap(unwrap(B)->CreateIntMinReduce(unwrap(Src), IsSigned)); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceMax(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) { +// return wrap(unwrap(B)->CreateIntMaxReduce(unwrap(Src), IsSigned)); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceFMin(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { +// Instruction *I = unwrap(B)->CreateFPMinReduce(unwrap(Src)); +// I->setHasNoNaNs(NoNaN); +// return wrap(I); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildVectorReduceFMax(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { +// Instruction *I = unwrap(B)->CreateFPMaxReduce(unwrap(Src)); +// I->setHasNoNaNs(NoNaN); +// return wrap(I); +// } + +// extern "C" LLVMValueRef +// LLVMRustBuildMinNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { +// return wrap(unwrap(B)->CreateMinNum(unwrap(LHS),unwrap(RHS))); +// } +// extern "C" LLVMValueRef +// LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { +// return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS),unwrap(RHS))); +// } + +// // This struct contains all necessary info about a symbol exported from a DLL. +// struct LLVMRustCOFFShortExport { +// const char* name; +// bool ordinal_present; +// // The value of `ordinal` is only meaningful if `ordinal_present` is true. +// uint16_t ordinal; +// }; + +// // Machine must be a COFF machine type, as defined in PE specs. +// extern "C" LLVMRustResult LLVMRustWriteImportLibrary( +// const char* ImportName, +// const char* Path, +// const LLVMRustCOFFShortExport* Exports, +// size_t NumExports, +// uint16_t Machine, +// bool MinGW) +// { +// std::vector ConvertedExports; +// ConvertedExports.reserve(NumExports); + +// for (size_t i = 0; i < NumExports; ++i) { +// bool ordinal_present = Exports[i].ordinal_present; +// uint16_t ordinal = ordinal_present ? Exports[i].ordinal : 0; +// ConvertedExports.push_back(llvm::object::COFFShortExport{ +// Exports[i].name, // Name +// std::string{}, // ExtName +// std::string{}, // SymbolName +// std::string{}, // AliasTarget +// ordinal, // Ordinal +// ordinal_present, // Noname +// false, // Data +// false, // Private +// false // Constant +// }); +// } + +// auto Error = llvm::object::writeImportLibrary( +// ImportName, +// Path, +// ConvertedExports, +// static_cast(Machine), +// MinGW); +// if (Error) { +// std::string errorString; +// llvm::raw_string_ostream stream(errorString); +// stream << Error; +// stream.flush(); +// LLVMRustSetLastError(errorString.c_str()); +// return LLVMRustResult::Failure; +// } else { +// return LLVMRustResult::Success; +// } +// } + +// // Transfers ownership of DiagnosticHandler unique_ptr to the caller. +// extern "C" DiagnosticHandler * +// LLVMRustContextGetDiagnosticHandler(LLVMContextRef C) { +// std::unique_ptr DH = unwrap(C)->getDiagnosticHandler(); +// return DH.release(); +// } + +// // Sets unique_ptr to object of DiagnosticHandler to provide custom diagnostic +// // handling. Ownership of the handler is moved to the LLVMContext. +// extern "C" void LLVMRustContextSetDiagnosticHandler(LLVMContextRef C, +// DiagnosticHandler *DH) { +// unwrap(C)->setDiagnosticHandler(std::unique_ptr(DH)); +// } + +// using LLVMDiagnosticHandlerTy = DiagnosticHandler::DiagnosticHandlerTy; + +// // Configures a diagnostic handler that invokes provided callback when a +// // backend needs to emit a diagnostic. +// // +// // When RemarkAllPasses is true, remarks are enabled for all passes. Otherwise +// // the RemarkPasses array specifies individual passes for which remarks will be +// // enabled. +// extern "C" void LLVMRustContextConfigureDiagnosticHandler( +// LLVMContextRef C, LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, +// void *DiagnosticHandlerContext, bool RemarkAllPasses, +// const char * const * RemarkPasses, size_t RemarkPassesLen) { + +// class RustDiagnosticHandler final : public DiagnosticHandler { +// public: +// RustDiagnosticHandler(LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, +// void *DiagnosticHandlerContext, +// bool RemarkAllPasses, +// std::vector RemarkPasses) +// : DiagnosticHandlerCallback(DiagnosticHandlerCallback), +// DiagnosticHandlerContext(DiagnosticHandlerContext), +// RemarkAllPasses(RemarkAllPasses), +// RemarkPasses(RemarkPasses) {} + +// virtual bool handleDiagnostics(const DiagnosticInfo &DI) override { +// if (DiagnosticHandlerCallback) { +// DiagnosticHandlerCallback(DI, DiagnosticHandlerContext); +// return true; +// } +// return false; +// } + +// bool isAnalysisRemarkEnabled(StringRef PassName) const override { +// return isRemarkEnabled(PassName); +// } + +// bool isMissedOptRemarkEnabled(StringRef PassName) const override { +// return isRemarkEnabled(PassName); +// } + +// bool isPassedOptRemarkEnabled(StringRef PassName) const override { +// return isRemarkEnabled(PassName); +// } + +// bool isAnyRemarkEnabled() const override { +// return RemarkAllPasses || !RemarkPasses.empty(); +// } + +// private: +// bool isRemarkEnabled(StringRef PassName) const { +// if (RemarkAllPasses) +// return true; + +// for (auto &Pass : RemarkPasses) +// if (Pass == PassName) +// return true; + +// return false; +// } + +// LLVMDiagnosticHandlerTy DiagnosticHandlerCallback = nullptr; +// void *DiagnosticHandlerContext = nullptr; + +// bool RemarkAllPasses = false; +// std::vector RemarkPasses; +// }; + +// std::vector Passes; +// for (size_t I = 0; I != RemarkPassesLen; ++I) +// Passes.push_back(RemarkPasses[I]); + +// unwrap(C)->setDiagnosticHandler(std::make_unique( +// DiagnosticHandlerCallback, DiagnosticHandlerContext, RemarkAllPasses, Passes)); +// } + +// extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) { +// RawRustStringOstream OS(Str); +// GlobalValue *GV = unwrap(V); +// Mangler().getNameWithPrefix(OS, GV, true); +// } + +// // LLVMGetAggregateElement was added in LLVM 15. For earlier LLVM versions just +// // use its implementation. +// #if LLVM_VERSION_LT(15, 0) +// extern "C" LLVMValueRef LLVMGetAggregateElement(LLVMValueRef C, unsigned Idx) { +// return wrap(unwrap(C)->getAggregateElement(Idx)); +// } +// #endif + +// extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) { +// #if LLVM_VERSION_GE(15, 0) +// auto *CB = unwrap(CallSite); +// switch (CB->getIntrinsicID()) { +// case Intrinsic::arm_ldrex: +// return 0; +// case Intrinsic::arm_strex: +// return 1; +// } +// #endif +// return -1; +// } diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs index cc712da5ef0..20a8fe6b3ec 100644 --- a/compiler/rustc_llvm/src/lib.rs +++ b/compiler/rustc_llvm/src/lib.rs @@ -6,6 +6,8 @@ use libc::{c_char, size_t}; use std::cell::RefCell; use std::slice; +// NOTE: Trying to import this RustString will result in a BAD TIME. +// Use the definition from `rustc_llvm_coverage` instead! #[repr(C)] pub struct RustString { pub bytes: RefCell>, @@ -29,156 +31,156 @@ pub unsafe extern "C" fn LLVMRustStringWriteImpl(sr: &RustString, ptr: *const c_ sr.bytes.borrow_mut().extend_from_slice(slice); } -/// Initialize targets enabled by the build script via `cfg(llvm_component = "...")`. -/// N.B., this function can't be moved to `rustc_codegen_llvm` because of the `cfg`s. -pub fn initialize_available_targets() { - macro_rules! init_target( - ($cfg:meta, $($method:ident),*) => { { - #[cfg($cfg)] - fn init() { - extern "C" { - $(fn $method();)* - } - unsafe { - $($method();)* - } - } - #[cfg(not($cfg))] - fn init() { } - init(); - } } - ); - init_target!( - llvm_component = "x86", - LLVMInitializeX86TargetInfo, - LLVMInitializeX86Target, - LLVMInitializeX86TargetMC, - LLVMInitializeX86AsmPrinter, - LLVMInitializeX86AsmParser - ); - init_target!( - llvm_component = "arm", - LLVMInitializeARMTargetInfo, - LLVMInitializeARMTarget, - LLVMInitializeARMTargetMC, - LLVMInitializeARMAsmPrinter, - LLVMInitializeARMAsmParser - ); - init_target!( - llvm_component = "aarch64", - LLVMInitializeAArch64TargetInfo, - LLVMInitializeAArch64Target, - LLVMInitializeAArch64TargetMC, - LLVMInitializeAArch64AsmPrinter, - LLVMInitializeAArch64AsmParser - ); - init_target!( - llvm_component = "amdgpu", - LLVMInitializeAMDGPUTargetInfo, - LLVMInitializeAMDGPUTarget, - LLVMInitializeAMDGPUTargetMC, - LLVMInitializeAMDGPUAsmPrinter, - LLVMInitializeAMDGPUAsmParser - ); - init_target!( - llvm_component = "avr", - LLVMInitializeAVRTargetInfo, - LLVMInitializeAVRTarget, - LLVMInitializeAVRTargetMC, - LLVMInitializeAVRAsmPrinter, - LLVMInitializeAVRAsmParser - ); - init_target!( - llvm_component = "m68k", - LLVMInitializeM68kTargetInfo, - LLVMInitializeM68kTarget, - LLVMInitializeM68kTargetMC, - LLVMInitializeM68kAsmPrinter, - LLVMInitializeM68kAsmParser - ); - init_target!( - llvm_component = "mips", - LLVMInitializeMipsTargetInfo, - LLVMInitializeMipsTarget, - LLVMInitializeMipsTargetMC, - LLVMInitializeMipsAsmPrinter, - LLVMInitializeMipsAsmParser - ); - init_target!( - llvm_component = "powerpc", - LLVMInitializePowerPCTargetInfo, - LLVMInitializePowerPCTarget, - LLVMInitializePowerPCTargetMC, - LLVMInitializePowerPCAsmPrinter, - LLVMInitializePowerPCAsmParser - ); - init_target!( - llvm_component = "systemz", - LLVMInitializeSystemZTargetInfo, - LLVMInitializeSystemZTarget, - LLVMInitializeSystemZTargetMC, - LLVMInitializeSystemZAsmPrinter, - LLVMInitializeSystemZAsmParser - ); - init_target!( - llvm_component = "jsbackend", - LLVMInitializeJSBackendTargetInfo, - LLVMInitializeJSBackendTarget, - LLVMInitializeJSBackendTargetMC - ); - init_target!( - llvm_component = "msp430", - LLVMInitializeMSP430TargetInfo, - LLVMInitializeMSP430Target, - LLVMInitializeMSP430TargetMC, - LLVMInitializeMSP430AsmPrinter, - LLVMInitializeMSP430AsmParser - ); - init_target!( - llvm_component = "riscv", - LLVMInitializeRISCVTargetInfo, - LLVMInitializeRISCVTarget, - LLVMInitializeRISCVTargetMC, - LLVMInitializeRISCVAsmPrinter, - LLVMInitializeRISCVAsmParser - ); - init_target!( - llvm_component = "sparc", - LLVMInitializeSparcTargetInfo, - LLVMInitializeSparcTarget, - LLVMInitializeSparcTargetMC, - LLVMInitializeSparcAsmPrinter, - LLVMInitializeSparcAsmParser - ); - init_target!( - llvm_component = "nvptx", - LLVMInitializeNVPTXTargetInfo, - LLVMInitializeNVPTXTarget, - LLVMInitializeNVPTXTargetMC, - LLVMInitializeNVPTXAsmPrinter - ); - init_target!( - llvm_component = "hexagon", - LLVMInitializeHexagonTargetInfo, - LLVMInitializeHexagonTarget, - LLVMInitializeHexagonTargetMC, - LLVMInitializeHexagonAsmPrinter, - LLVMInitializeHexagonAsmParser - ); - init_target!( - llvm_component = "webassembly", - LLVMInitializeWebAssemblyTargetInfo, - LLVMInitializeWebAssemblyTarget, - LLVMInitializeWebAssemblyTargetMC, - LLVMInitializeWebAssemblyAsmPrinter, - LLVMInitializeWebAssemblyAsmParser - ); - init_target!( - llvm_component = "bpf", - LLVMInitializeBPFTargetInfo, - LLVMInitializeBPFTarget, - LLVMInitializeBPFTargetMC, - LLVMInitializeBPFAsmPrinter, - LLVMInitializeBPFAsmParser - ); -} +// /// Initialize targets enabled by the build script via `cfg(llvm_component = "...")`. +// /// N.B., this function can't be moved to `rustc_codegen_llvm` because of the `cfg`s. +// pub fn initialize_available_targets() { +// macro_rules! init_target( +// ($cfg:meta, $($method:ident),*) => { { +// #[cfg($cfg)] +// fn init() { +// extern "C" { +// $(fn $method();)* +// } +// unsafe { +// $($method();)* +// } +// } +// #[cfg(not($cfg))] +// fn init() { } +// init(); +// } } +// ); +// init_target!( +// llvm_component = "x86", +// LLVMInitializeX86TargetInfo, +// LLVMInitializeX86Target, +// LLVMInitializeX86TargetMC, +// LLVMInitializeX86AsmPrinter, +// LLVMInitializeX86AsmParser +// ); +// init_target!( +// llvm_component = "arm", +// LLVMInitializeARMTargetInfo, +// LLVMInitializeARMTarget, +// LLVMInitializeARMTargetMC, +// LLVMInitializeARMAsmPrinter, +// LLVMInitializeARMAsmParser +// ); +// init_target!( +// llvm_component = "aarch64", +// LLVMInitializeAArch64TargetInfo, +// LLVMInitializeAArch64Target, +// LLVMInitializeAArch64TargetMC, +// LLVMInitializeAArch64AsmPrinter, +// LLVMInitializeAArch64AsmParser +// ); +// init_target!( +// llvm_component = "amdgpu", +// LLVMInitializeAMDGPUTargetInfo, +// LLVMInitializeAMDGPUTarget, +// LLVMInitializeAMDGPUTargetMC, +// LLVMInitializeAMDGPUAsmPrinter, +// LLVMInitializeAMDGPUAsmParser +// ); +// init_target!( +// llvm_component = "avr", +// LLVMInitializeAVRTargetInfo, +// LLVMInitializeAVRTarget, +// LLVMInitializeAVRTargetMC, +// LLVMInitializeAVRAsmPrinter, +// LLVMInitializeAVRAsmParser +// ); +// init_target!( +// llvm_component = "m68k", +// LLVMInitializeM68kTargetInfo, +// LLVMInitializeM68kTarget, +// LLVMInitializeM68kTargetMC, +// LLVMInitializeM68kAsmPrinter, +// LLVMInitializeM68kAsmParser +// ); +// init_target!( +// llvm_component = "mips", +// LLVMInitializeMipsTargetInfo, +// LLVMInitializeMipsTarget, +// LLVMInitializeMipsTargetMC, +// LLVMInitializeMipsAsmPrinter, +// LLVMInitializeMipsAsmParser +// ); +// init_target!( +// llvm_component = "powerpc", +// LLVMInitializePowerPCTargetInfo, +// LLVMInitializePowerPCTarget, +// LLVMInitializePowerPCTargetMC, +// LLVMInitializePowerPCAsmPrinter, +// LLVMInitializePowerPCAsmParser +// ); +// init_target!( +// llvm_component = "systemz", +// LLVMInitializeSystemZTargetInfo, +// LLVMInitializeSystemZTarget, +// LLVMInitializeSystemZTargetMC, +// LLVMInitializeSystemZAsmPrinter, +// LLVMInitializeSystemZAsmParser +// ); +// init_target!( +// llvm_component = "jsbackend", +// LLVMInitializeJSBackendTargetInfo, +// LLVMInitializeJSBackendTarget, +// LLVMInitializeJSBackendTargetMC +// ); +// init_target!( +// llvm_component = "msp430", +// LLVMInitializeMSP430TargetInfo, +// LLVMInitializeMSP430Target, +// LLVMInitializeMSP430TargetMC, +// LLVMInitializeMSP430AsmPrinter, +// LLVMInitializeMSP430AsmParser +// ); +// init_target!( +// llvm_component = "riscv", +// LLVMInitializeRISCVTargetInfo, +// LLVMInitializeRISCVTarget, +// LLVMInitializeRISCVTargetMC, +// LLVMInitializeRISCVAsmPrinter, +// LLVMInitializeRISCVAsmParser +// ); +// init_target!( +// llvm_component = "sparc", +// LLVMInitializeSparcTargetInfo, +// LLVMInitializeSparcTarget, +// LLVMInitializeSparcTargetMC, +// LLVMInitializeSparcAsmPrinter, +// LLVMInitializeSparcAsmParser +// ); +// init_target!( +// llvm_component = "nvptx", +// LLVMInitializeNVPTXTargetInfo, +// LLVMInitializeNVPTXTarget, +// LLVMInitializeNVPTXTargetMC, +// LLVMInitializeNVPTXAsmPrinter +// ); +// init_target!( +// llvm_component = "hexagon", +// LLVMInitializeHexagonTargetInfo, +// LLVMInitializeHexagonTarget, +// LLVMInitializeHexagonTargetMC, +// LLVMInitializeHexagonAsmPrinter, +// LLVMInitializeHexagonAsmParser +// ); +// init_target!( +// llvm_component = "webassembly", +// LLVMInitializeWebAssemblyTargetInfo, +// LLVMInitializeWebAssemblyTarget, +// LLVMInitializeWebAssemblyTargetMC, +// LLVMInitializeWebAssemblyAsmPrinter, +// LLVMInitializeWebAssemblyAsmParser +// ); +// init_target!( +// llvm_component = "bpf", +// LLVMInitializeBPFTargetInfo, +// LLVMInitializeBPFTarget, +// LLVMInitializeBPFTargetMC, +// LLVMInitializeBPFAsmPrinter, +// LLVMInitializeBPFAsmParser +// ); +// } From 5609edba61674baa671a759fcec1ae83c9100ca7 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 6 Dec 2023 22:24:51 +0000 Subject: [PATCH 12/70] fix build depend issue --- compiler/rustc_llvm_coverage/Cargo.toml | 2 +- compiler/rustc_llvm_coverage/src/ffi.rs | 3 --- compiler/rustc_llvm_coverage/src/types.rs | 18 ++++++++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_llvm_coverage/Cargo.toml b/compiler/rustc_llvm_coverage/Cargo.toml index 5248cb58f9b..0f89c132d3b 100644 --- a/compiler/rustc_llvm_coverage/Cargo.toml +++ b/compiler/rustc_llvm_coverage/Cargo.toml @@ -6,5 +6,5 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -libc = "0.2" +libc = "0.2.73" rustc_llvm = { path = "../rustc_llvm" } diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index ac06f94cd97..b01f35e2afe 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -6,13 +6,10 @@ // use libc::{c_char, c_uint, c_void, size_t}; -use rustc_llvm::RustString; - use super::types::*; #[repr(C)] pub struct Module(c_void); - #[repr(C)] pub struct Value(c_void); diff --git a/compiler/rustc_llvm_coverage/src/types.rs b/compiler/rustc_llvm_coverage/src/types.rs index 3318ce5a6cb..e39d53e665a 100644 --- a/compiler/rustc_llvm_coverage/src/types.rs +++ b/compiler/rustc_llvm_coverage/src/types.rs @@ -7,6 +7,24 @@ // - [`rustc_codegen_llvm/src/coverageinfo/ffi.rs`](https://github.com/rust-lang/rust/blob/56278a6e2824acc96b222e5816bf2d74e85dab93/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs#L4) // - [`rustc_middle/src/mir/coverage.rs`](https://github.com/rust-lang/rust/blob/56278a6e2824acc96b222e5816bf2d74e85dab93/compiler/rustc_middle/src/mir/coverage.rs#L9) // + +use std::cell::RefCell; + +#[repr(C)] +pub struct RustString { + pub bytes: RefCell>, +} + +impl RustString { + pub fn len(&self) -> usize { + self.bytes.borrow().len() + } + + pub fn is_empty(&self) -> bool { + self.bytes.borrow().is_empty() + } +} + #[derive(Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] pub struct CounterId(u32); From d588285d451af1d2e6793c8d3849d70803904ea4 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 6 Dec 2023 22:25:35 +0000 Subject: [PATCH 13/70] package updates --- Cargo.lock | 10 +++++++++- Cargo.toml | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce4361a8c2a..49197f87018 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2249,6 +2249,14 @@ dependencies = [ "libc", ] +[[package]] +name = "rustc_llvm_coverage" +version = "0.1.0" +dependencies = [ + "libc", + "rustc_llvm", +] + [[package]] name = "rustix" version = "0.37.25" @@ -2304,7 +2312,7 @@ dependencies = [ "plc_xml", "pretty_assertions", "regex", - "rustc_llvm", + "rustc_llvm_coverage", "serde", "serde_json", "serial_test", diff --git a/Cargo.toml b/Cargo.toml index fa3f50484f5..812f6b1cb12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ plc_source = { path = "./compiler/plc_source" } plc_ast = { path = "./compiler/plc_ast" } plc_util = { path = "./compiler/plc_util" } plc_diagnostics = { path = "./compiler/plc_diagnostics" } -rustc_llvm = { path = "./compiler/rustc_llvm" } +rustc_llvm_coverage = { path = "./compiler/rustc_llvm_coverage" } logos = "0.12.0" thiserror = "1.0" clap = { version = "3.0", features = ["derive"] } @@ -71,6 +71,7 @@ members = [ "compiler/plc_xml", "compiler/plc_derive", "compiler/rustc_llvm", + "compiler/rustc_llvm_coverage", ] default-members = [".", "compiler/plc_driver", "compiler/plc_xml"] From f415792258d2e74479eaf36a3e12d50b439274d5 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 6 Dec 2023 22:25:50 +0000 Subject: [PATCH 14/70] call from codegen --- src/codegen.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/codegen.rs b/src/codegen.rs index fb9f5ce00e7..6028123eb1b 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -38,6 +38,7 @@ use inkwell::{ use plc_ast::ast::{CompilationUnit, LinkageType}; use plc_diagnostics::diagnostics::Diagnostic; use plc_source::source_location::SourceLocation; +use rustc_llvm_coverage; mod debug; pub(crate) mod generators; @@ -212,6 +213,9 @@ impl<'ink> CodeGen<'ink> { self.debug.finalize(); log::debug!("{}", self.module.to_string()); + println!("Done generating POUs"); + println!("Cov mapping version: {}", rustc_llvm_coverage::mapping_version()); + #[cfg(feature = "verify")] { self.module From 19d30244b6b378ceb9fad93655028fb8b1bae1cd Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 6 Dec 2023 22:26:20 +0000 Subject: [PATCH 15/70] temp updates to build scripts --- scripts/build.sh | 16 +++++++++++----- scripts/common.sh | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 3f5f35837cf..d720a8c1fed 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -5,12 +5,12 @@ vendor=0 offline=0 check=0 check_style=0 -build=0 +build=1 doc=0 test=0 coverage=0 release=0 -debug=0 +debug=1 container=0 assume_linux=0 junit=0 @@ -18,8 +18,7 @@ package=0 target="" CONTAINER_NAME='rust-llvm' -# There may be a better way to set these -export REAL_LIBRARY_PATH_VAR="/usr/lib/llvm-14/lib" +# There may be a better way to set these (i.e. in Dockerfile) export PATH="$PATH:/usr/lib/llvm-14/bin" source "${BASH_SOURCE%/*}/common.sh" @@ -67,6 +66,11 @@ function run_build() { # Run cargo build with release or debug flags echo "Build starting" echo "-----------------------------------" +# # export REAL_LIBRARY_PATH_VAR="/usr/lib/llvm-14/lib" +# # export PATH="$PATH:/usr/lib/llvm-14/bin" +# cmd="REAL_LIBRARY_PATH_VAR=/usr/lib/llvm-14/lib PATH=$PATH:/usr/lib/llvm-14/bin cargo build -p rustc_llvm_coverage $CARGO_OPTIONS " +# eval "$cmd" + cmd="cargo build $CARGO_OPTIONS " log "Running $cmd" eval "$cmd" @@ -475,8 +479,10 @@ fi if [[ $build -ne 0 ]]; then run_build + # Test a program + ./target/debug/plc ./examples/simple_program.st #Build the standard functions - run_std_build + # run_std_build fi if [[ $package -ne 0 ]]; then diff --git a/scripts/common.sh b/scripts/common.sh index ec6abae1fc2..d9e122f6511 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -1,6 +1,6 @@ #!/bin/bash -debug=0 +debug=1 function log() { if [[ $debug -ne 0 ]]; then >&2 echo "$1" From c5b389d1d0bfaecb3653e3c32684704a50e934a6 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Thu, 7 Dec 2023 16:03:05 +0000 Subject: [PATCH 16/70] disable lib.rs functions, moved to rustc_llvm_coverage --- compiler/rustc_llvm/src/lib.rs | 50 +++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs index 20a8fe6b3ec..989e4bc2ee7 100644 --- a/compiler/rustc_llvm/src/lib.rs +++ b/compiler/rustc_llvm/src/lib.rs @@ -1,35 +1,35 @@ -#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] +// #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -// NOTE: This crate only exists to allow linking on mingw targets. +// // NOTE: This crate only exists to allow linking on mingw targets. -use libc::{c_char, size_t}; -use std::cell::RefCell; -use std::slice; +// use libc::{c_char, size_t}; +// use std::cell::RefCell; +// use std::slice; -// NOTE: Trying to import this RustString will result in a BAD TIME. -// Use the definition from `rustc_llvm_coverage` instead! -#[repr(C)] -pub struct RustString { - pub bytes: RefCell>, -} +// // NOTE: Trying to import this RustString will result in a BAD TIME. +// // Use the definition from `rustc_llvm_coverage` instead! +// #[repr(C)] +// pub struct RustString { +// pub bytes: RefCell>, +// } -impl RustString { - pub fn len(&self) -> usize { - self.bytes.borrow().len() - } +// impl RustString { +// pub fn len(&self) -> usize { +// self.bytes.borrow().len() +// } - pub fn is_empty(&self) -> bool { - self.bytes.borrow().is_empty() - } -} +// pub fn is_empty(&self) -> bool { +// self.bytes.borrow().is_empty() +// } +// } -/// Appending to a Rust string -- used by RawRustStringOstream. -#[no_mangle] -pub unsafe extern "C" fn LLVMRustStringWriteImpl(sr: &RustString, ptr: *const c_char, size: size_t) { - let slice = slice::from_raw_parts(ptr as *const u8, size as usize); +// /// Appending to a Rust string -- used by RawRustStringOstream. +// #[no_mangle] +// pub unsafe extern "C" fn LLVMRustStringWriteImpl(sr: &RustString, ptr: *const c_char, size: size_t) { +// let slice = slice::from_raw_parts(ptr as *const u8, size as usize); - sr.bytes.borrow_mut().extend_from_slice(slice); -} +// sr.bytes.borrow_mut().extend_from_slice(slice); +// } // /// Initialize targets enabled by the build script via `cfg(llvm_component = "...")`. // /// N.B., this function can't be moved to `rustc_codegen_llvm` because of the `cfg`s. From f4c8658a642d1fc0a71d47163a9f475b8b4f3f20 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Thu, 7 Dec 2023 16:04:00 +0000 Subject: [PATCH 17/70] enable hashstring and writing to buffer --- compiler/rustc_llvm_coverage/src/ffi.rs | 12 +++++++- compiler/rustc_llvm_coverage/src/lib.rs | 38 +++++++++++++------------ 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index b01f35e2afe..6e7dc699b40 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -5,15 +5,25 @@ // Function interface definitions are taken from [here](https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_llvm/src/llvm/ffi.rs#L1864). // use libc::{c_char, c_uint, c_void, size_t}; +use std::slice; use super::types::*; #[repr(C)] pub struct Module(c_void); + #[repr(C)] pub struct Value(c_void); -#[link(name = "llvm-wrapper")] +/// Appending to a Rust string -- used by RawRustStringOstream. +#[no_mangle] +pub unsafe extern "C" fn LLVMRustStringWriteImpl(sr: &RustString, ptr: *const c_char, size: size_t) { + let slice = slice::from_raw_parts(ptr as *const u8, size as usize); + + sr.bytes.borrow_mut().extend_from_slice(slice); +} + +#[link(name = "llvm-wrapper", kind = "static")] extern "C" { #[allow(improper_ctypes)] pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 977296498ec..77892cc80cb 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -1,13 +1,15 @@ // use rustc_llvm::RustString; mod ffi; -mod types; +pub mod types; + +use types::*; // use crate::abi::Abi; // use crate::llvm; // use libc::c_uint; -// use std::ffi::CString; +use std::ffi::CString; // /* // * Many of the functions in this file have been adapted from the @@ -31,15 +33,15 @@ mod types; // // unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } // // } -// pub(crate) fn write_filenames_section_to_buffer<'a>( -// filenames: impl IntoIterator, -// buffer: &RustString, -// ) { -// let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::>(); -// unsafe { -// llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer(c_str_vec.as_ptr(), c_str_vec.len(), buffer); -// } -// } +pub fn write_filenames_section_to_buffer<'a>( + filenames: impl IntoIterator, + buffer: &RustString, +) { + let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::>(); + unsafe { + ffi::LLVMRustCoverageWriteFilenamesSectionToBuffer(c_str_vec.as_ptr(), c_str_vec.len(), buffer); + } +} // pub(crate) fn write_mapping_to_buffer( // virtual_file_mapping: Vec, @@ -60,14 +62,14 @@ mod types; // } // } -// pub(crate) fn hash_str(strval: &str) -> u64 { -// let strval = CString::new(strval).expect("null error converting hashable str to C string"); -// unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } -// } +pub fn hash_str(strval: &str) -> u64 { + let strval = CString::new(strval).expect("null error converting hashable str to C string"); + unsafe { ffi::LLVMRustCoverageHashCString(strval.as_ptr()) } +} -// pub(crate) fn hash_bytes(bytes: Vec) -> u64 { -// unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } -// } +pub fn hash_bytes(bytes: Vec) -> u64 { + unsafe { ffi::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } +} pub fn mapping_version() -> u32 { unsafe { ffi::LLVMRustCoverageMappingVersion() } From cdd8194b4921b554217ded1c94e8d6ec511aef5e Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Thu, 7 Dec 2023 16:08:04 +0000 Subject: [PATCH 18/70] convenience auto build --- scripts/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/build.sh b/scripts/build.sh index d720a8c1fed..b6295d4c9f0 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -480,7 +480,10 @@ fi if [[ $build -ne 0 ]]; then run_build # Test a program + echo "-----------------------------------" + echo "Running on example program:" ./target/debug/plc ./examples/simple_program.st + echo "-----------------------------------" #Build the standard functions # run_std_build fi From b060f29d6dd305f19d03fca07a92cc26335a4b01 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Thu, 7 Dec 2023 16:08:24 +0000 Subject: [PATCH 19/70] add buf write example --- src/codegen.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/codegen.rs b/src/codegen.rs index 6028123eb1b..5a2a99bb7c2 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -1,6 +1,7 @@ // Copyright (c) 2020 Ghaith Hachem and Mathias Rieder use std::{ cell::RefCell, + ffi::CString, ops::Deref, path::{Path, PathBuf}, }; @@ -215,6 +216,26 @@ impl<'ink> CodeGen<'ink> { println!("Done generating POUs"); println!("Cov mapping version: {}", rustc_llvm_coverage::mapping_version()); + println!("Hash string: {:#?}", rustc_llvm_coverage::hash_str("asdf")); + let byte_string: Vec = vec![0x01, 0x02, 0x03, 0x04]; + println!("Hash bytes: {:#?}", rustc_llvm_coverage::hash_bytes(byte_string)); + + let rust_string = rustc_llvm_coverage::types::RustString { bytes: RefCell::new(Vec::new()) }; + let filenames = vec![CString::new("test.c").unwrap()]; + rustc_llvm_coverage::write_filenames_section_to_buffer(&filenames, &rust_string); + // print buffer + println!("Filenames: {:#?}", rust_string.bytes.borrow()); + // print buffer as hex string + println!( + "Filenames: {:#?}", + rust_string + .bytes + .borrow() + .iter() + .map(|it| format!("{:02x}", it)) + .collect::>() + .join("") + ); #[cfg(feature = "verify")] { From 0106d693f3fd6a9bce603bb9f25c32643ce44a3b Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Thu, 7 Dec 2023 23:54:24 +0000 Subject: [PATCH 20/70] update to inkwell fork --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- compiler/rustc_llvm_coverage/Cargo.toml | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49197f87018..7b7009c5082 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1243,8 +1243,7 @@ dependencies = [ [[package]] name = "inkwell" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f4fcb4a4fa0b8f7b4178e24e6317d6f8b95ab500d8e6e1bd4283b6860e369c1" +source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" dependencies = [ "either", "inkwell_internals", @@ -1257,8 +1256,7 @@ dependencies = [ [[package]] name = "inkwell_internals" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b185e7d068d6820411502efa14d8fbf010750485399402156b72dd2a548ef8e9" +source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" dependencies = [ "proc-macro2 1.0.67", "quote 1.0.33", @@ -2253,7 +2251,9 @@ dependencies = [ name = "rustc_llvm_coverage" version = "0.1.0" dependencies = [ + "inkwell", "libc", + "llvm-sys", "rustc_llvm", ] diff --git a/Cargo.toml b/Cargo.toml index 812f6b1cb12..e5e455fc305 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ members = [ default-members = [".", "compiler/plc_driver", "compiler/plc_xml"] [workspace.dependencies] -inkwell = { version = "0.2", features = ["llvm14-0"] } +inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } encoding_rs = "0.8" encoding_rs_io = "0.1" log = "0.4" diff --git a/compiler/rustc_llvm_coverage/Cargo.toml b/compiler/rustc_llvm_coverage/Cargo.toml index 0f89c132d3b..0def989f990 100644 --- a/compiler/rustc_llvm_coverage/Cargo.toml +++ b/compiler/rustc_llvm_coverage/Cargo.toml @@ -8,3 +8,5 @@ edition = "2021" [dependencies] libc = "0.2.73" rustc_llvm = { path = "../rustc_llvm" } +inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } +llvm-sys = { package = "llvm-sys", version = "140.0.2" } From 4d2ff285b0ac0def03c0e8c7cd056f0bda0822f3 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Thu, 7 Dec 2023 23:55:25 +0000 Subject: [PATCH 21/70] pgo calls working --- compiler/rustc_llvm_coverage/src/ffi.rs | 7 ++-- compiler/rustc_llvm_coverage/src/lib.rs | 49 ++++++++++++------------- src/codegen.rs | 11 +++++- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index 6e7dc699b40..2a6b7277d5d 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -8,12 +8,13 @@ use libc::{c_char, c_uint, c_void, size_t}; use std::slice; use super::types::*; +use llvm_sys::prelude::LLVMValueRef; #[repr(C)] pub struct Module(c_void); -#[repr(C)] -pub struct Value(c_void); +// #[repr(C)] +// pub struct Value(c_void); /// Appending to a Rust string -- used by RawRustStringOstream. #[no_mangle] @@ -43,7 +44,7 @@ extern "C" { BufferOut: &RustString, ); - pub fn LLVMRustCoverageCreatePGOFuncNameVar(F: &Value, FuncName: *const c_char) -> &Value; + pub fn LLVMRustCoverageCreatePGOFuncNameVar(F: LLVMValueRef, FuncName: *const c_char) -> LLVMValueRef; pub fn LLVMRustCoverageHashCString(StrVal: *const c_char) -> u64; pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 77892cc80cb..61a452fb248 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -1,37 +1,34 @@ -// use rustc_llvm::RustString; +/* + * Many of the functions in this file have been adapted from the + * `rustc` implementation of LLVM code coverage. + * + * https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs#L220-L221 + */ mod ffi; pub mod types; use types::*; -// use crate::abi::Abi; -// use crate::llvm; -// use libc::c_uint; - +use inkwell::values::{AsValueRef, FunctionValue, GlobalValue}; use std::ffi::CString; -// /* -// * Many of the functions in this file have been adapted from the -// * `rustc` implementation of LLVM code coverage. -// * -// * https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs#L220-L221 -// */ -// /// Calls llvm::createPGOFuncNameVar() with the given function instance's -// /// mangled function name. The LLVM API returns an llvm::GlobalVariable -// /// containing the function name, with the specific variable name and linkage -// /// required by LLVM InstrProf source-based coverage instrumentation. Use -// /// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per -// /// `Instance`. -// // fn create_pgo_func_name_var<'ll, 'tcx>( -// // cx: &CodegenCx<'ll, 'tcx>, -// // instance: Instance<'tcx>, -// // ) -> &'ll llvm::Value { -// // let mangled_fn_name = -// // CString::new(cx.tcx.symbol_name(instance).name).expect("error converting function name to C string"); -// // let llfn = cx.get_fn(instance); -// // unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } -// // } +/// Calls llvm::createPGOFuncNameVar() with the given function instance's +/// mangled function name. The LLVM API returns an llvm::GlobalVariable +/// containing the function name, with the specific variable name and linkage +/// required by LLVM InstrProf source-based coverage instrumentation. Use +/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per +/// `Instance`. +pub fn create_pgo_func_name_var<'ctx>(func: &FunctionValue<'ctx>) -> GlobalValue<'ctx> { + // let mangled_fn_name = + // CString::new(cx.tcx.symbol_name(instance).name).expect("error converting function name to C string"); + let mangled_fn_name = CString::new("asdf_fake_program").unwrap(); + // let llfn = cx.get_fn(instance); + let llvm_val_ref = + unsafe { ffi::LLVMRustCoverageCreatePGOFuncNameVar(func.as_value_ref(), mangled_fn_name.as_ptr()) }; + assert!(!llvm_val_ref.is_null()); + unsafe { GlobalValue::new(llvm_val_ref) } +} pub fn write_filenames_section_to_buffer<'a>( filenames: impl IntoIterator, diff --git a/src/codegen.rs b/src/codegen.rs index 5a2a99bb7c2..9a39f7b1fcf 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -224,7 +224,7 @@ impl<'ink> CodeGen<'ink> { let filenames = vec![CString::new("test.c").unwrap()]; rustc_llvm_coverage::write_filenames_section_to_buffer(&filenames, &rust_string); // print buffer - println!("Filenames: {:#?}", rust_string.bytes.borrow()); + // println!("Filenames: {:#?}", rust_string.bytes.borrow()); // print buffer as hex string println!( "Filenames: {:#?}", @@ -237,6 +237,15 @@ impl<'ink> CodeGen<'ink> { .join("") ); + // println!("{:#?}", llvm_index); + let prg_func = llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); + println!("prg: {:#?}", prg_func); + let global_func_var = rustc_llvm_coverage::create_pgo_func_name_var(&prg_func); + println!("global_func_var: {:#?}", global_func_var); + + let global_vars: Vec<_> = self.module.get_globals().collect(); + println!("global_vars: {:#?}", global_vars); + #[cfg(feature = "verify")] { self.module From 15784e70210ea2186b047558f81e9cfc7752910a Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Thu, 14 Dec 2023 04:39:02 +0000 Subject: [PATCH 22/70] cleanup pgo --- compiler/rustc_llvm_coverage/src/lib.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 61a452fb248..0204bf8a0c7 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -20,14 +20,10 @@ use std::ffi::CString; /// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per /// `Instance`. pub fn create_pgo_func_name_var<'ctx>(func: &FunctionValue<'ctx>) -> GlobalValue<'ctx> { - // let mangled_fn_name = - // CString::new(cx.tcx.symbol_name(instance).name).expect("error converting function name to C string"); - let mangled_fn_name = CString::new("asdf_fake_program").unwrap(); - // let llfn = cx.get_fn(instance); - let llvm_val_ref = - unsafe { ffi::LLVMRustCoverageCreatePGOFuncNameVar(func.as_value_ref(), mangled_fn_name.as_ptr()) }; - assert!(!llvm_val_ref.is_null()); - unsafe { GlobalValue::new(llvm_val_ref) } + let pgo_function_ref = + unsafe { ffi::LLVMRustCoverageCreatePGOFuncNameVar(func.as_value_ref(), func.get_name().as_ptr()) }; + assert!(!pgo_function_ref.is_null()); + unsafe { GlobalValue::new(pgo_function_ref) } } pub fn write_filenames_section_to_buffer<'a>( From bbcb66a12e6655a59b6f76aaa593fcad223f4db8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 13 Dec 2023 08:52:24 +0000 Subject: [PATCH 23/70] some changes --- Cargo.lock | 1031 +++++++++++++-------- Cargo.toml | 4 +- compiler/rustc_llvm_coverage/src/ffi.rs | 3 + compiler/rustc_llvm_coverage/src/lib.rs | 91 +- compiler/rustc_llvm_coverage/src/types.rs | 4 +- src/codegen.rs | 45 +- 6 files changed, 723 insertions(+), 455 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b7009c5082..66a02b809f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,22 +19,23 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", "getrandom", "once_cell", "serde", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.0.5" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -62,9 +63,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" dependencies = [ "anstyle", "anstyle-parse", @@ -76,36 +77,36 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -121,36 +122,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue", - "event-listener", + "event-listener 2.5.3", "futures-core", ] +[[package]] +name = "async-channel" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" +dependencies = [ + "concurrent-queue", + "event-listener 4.0.0", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-executor" -version = "1.5.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa3dc5f2a8564f07759c008b9109dc0d39de92a88d5588b8a5036d286383afb" +checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" dependencies = [ - "async-lock", + "async-lock 3.2.0", "async-task", "concurrent-queue", - "fastrand 1.9.0", - "futures-lite", + "fastrand 2.0.1", + "futures-lite 2.1.0", "slab", ] [[package]] name = "async-global-executor" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel", + "async-channel 2.1.1", "async-executor", - "async-io", - "async-lock", + "async-io 2.2.2", + "async-lock 3.2.0", "blocking", - "futures-lite", + "futures-lite 2.1.0", "once_cell", ] @@ -160,27 +174,57 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ - "async-lock", + "async-lock 2.8.0", "autocfg", "cfg-if", "concurrent-queue", - "futures-lite", + "futures-lite 1.13.0", "log", "parking", - "polling", - "rustix 0.37.25", + "polling 2.8.0", + "rustix 0.37.27", "slab", - "socket2 0.4.9", + "socket2 0.4.10", "waker-fn", ] +[[package]] +name = "async-io" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6afaa937395a620e33dc6a742c593c01aced20aa376ffb0f628121198578ccc7" +dependencies = [ + "async-lock 3.2.0", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.1.0", + "parking", + "polling 3.3.1", + "rustix 0.38.28", + "slab", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "async-lock" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" dependencies = [ - "event-listener", + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" +dependencies = [ + "event-listener 4.0.0", + "event-listener-strategy", + "pin-project-lite", ] [[package]] @@ -189,15 +233,15 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" dependencies = [ - "async-channel", + "async-channel 1.9.0", "async-global-executor", - "async-io", - "async-lock", + "async-io 1.13.0", + "async-lock 2.8.0", "crossbeam-utils", "futures-channel", "futures-core", "futures-io", - "futures-lite", + "futures-lite 1.13.0", "gloo-timers", "kv-log-macro", "log", @@ -211,9 +255,9 @@ dependencies = [ [[package]] name = "async-task" -version = "4.4.0" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" +checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" [[package]] name = "atoi" @@ -226,9 +270,19 @@ dependencies = [ [[package]] name = "atomic-waker" -version = "1.1.1" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "atomic-write-file" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" +checksum = "edcdbedc2236483ab103a53415653d6b4442ea6141baf1ffa85df29635e88436" +dependencies = [ + "nix", + "rand", +] [[package]] name = "atty" @@ -264,9 +318,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "base64ct" @@ -303,9 +357,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" dependencies = [ "serde", ] @@ -321,36 +375,37 @@ dependencies = [ [[package]] name = "blocking" -version = "1.3.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77231a1c8f801696fc0123ec6150ce92cffb8e164a02afb9c8ddee0e9b65ad65" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "async-channel", - "async-lock", + "async-channel 2.1.1", + "async-lock 3.2.0", "async-task", - "atomic-waker", - "fastrand 1.9.0", - "futures-lite", - "log", + "fastrand 2.0.1", + "futures-io", + "futures-lite 2.1.0", + "piper", + "tracing", ] [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytecount" -version = "0.6.4" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad152d03a2c813c80bb94fedbf3a3f02b28f793e39e7c214c8a0bcc196343de7" +checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -375,9 +430,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.30" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", @@ -406,23 +461,23 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.3" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" dependencies = [ "clap_builder", - "clap_derive 4.4.2", + "clap_derive 4.4.7", ] [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" dependencies = [ "anstream", "anstyle", - "clap_lex 0.5.1", + "clap_lex 0.6.0", "strsim", ] @@ -434,21 +489,21 @@ checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", ] [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.33", + "syn 2.0.41", ] [[package]] @@ -462,9 +517,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "codespan-reporting" @@ -484,9 +539,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "concurrent-queue" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] @@ -511,9 +566,9 @@ checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -521,15 +576,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -545,19 +600,9 @@ dependencies = [ [[package]] name = "crc-catalog" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" - -[[package]] -name = "crossbeam-channel" -version = "0.5.8" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crossbeam-deque" @@ -619,7 +664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core", @@ -638,9 +683,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +dependencies = [ + "powerfmt", +] [[package]] name = "diff" @@ -701,9 +749,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" dependencies = [ "humantime", "is-terminal", @@ -720,23 +768,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] @@ -756,6 +793,27 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "event-listener" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.0", + "pin-project-lite", +] + [[package]] name = "fancy-regex" version = "0.11.0" @@ -777,9 +835,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "finl_unicode" @@ -789,13 +847,12 @@ checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" [[package]] name = "flume" -version = "0.10.14" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" dependencies = [ "futures-core", "futures-sink", - "pin-project", "spin 0.9.8", ] @@ -822,9 +879,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -841,9 +898,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" dependencies = [ "futures-channel", "futures-core", @@ -856,9 +913,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -866,15 +923,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ "futures-core", "futures-task", @@ -894,9 +951,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-lite" @@ -913,23 +970,36 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "futures-lite" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143" +dependencies = [ + "fastrand 2.0.1", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -963,9 +1033,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "js-sys", @@ -976,9 +1046,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -1000,9 +1070,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" dependencies = [ "bytes", "fnv", @@ -1010,7 +1080,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -1025,9 +1095,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ "ahash", "allocator-api2", @@ -1039,7 +1109,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.0", + "hashbrown 0.14.3", ] [[package]] @@ -1062,9 +1132,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hex" @@ -1110,9 +1180,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -1121,9 +1191,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -1165,7 +1235,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -1174,16 +1244,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -1197,9 +1267,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1232,12 +1302,25 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.3", +] + +[[package]] +name = "inkwell" +version = "0.2.0" +source = "git+https://github.com/HaithemLamri/inkwell.git?branch=master#955375fcacd3bfc04e2862ca4e375e30c07392dd" +dependencies = [ + "either", + "inkwell_internals 0.8.0 (git+https://github.com/HaithemLamri/inkwell.git?branch=master)", + "libc", + "llvm-sys", + "once_cell", + "thiserror", ] [[package]] @@ -1246,28 +1329,38 @@ version = "0.2.0" source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" dependencies = [ "either", - "inkwell_internals", + "inkwell_internals 0.8.0 (git+https://github.com/corbanvilla/inkwell.git?branch=v2.0)", "libc", "llvm-sys", "once_cell", "parking_lot", ] +[[package]] +name = "inkwell_internals" +version = "0.8.0" +source = "git+https://github.com/HaithemLamri/inkwell.git?branch=master#955375fcacd3bfc04e2862ca4e375e30c07392dd" +dependencies = [ + "proc-macro2 1.0.70", + "quote 1.0.33", + "syn 2.0.41", +] + [[package]] name = "inkwell_internals" version = "0.8.0" source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.33", + "syn 2.0.41", ] [[package]] name = "insta" -version = "1.31.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0770b0a3d4c70567f0d58331f3088b0e4c4f56c9b8d764efe654b4a5d46de3a" +checksum = "5d64600be34b2fcfc267740a243fa7744441bb4947a619ac4e5bb6507f35fbfc" dependencies = [ "console", "lazy_static", @@ -1291,16 +1384,16 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi 0.3.3", "libc", "windows-sys 0.48.0", ] [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" @@ -1308,8 +1401,8 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.2", - "rustix 0.38.13", + "hermit-abi 0.3.3", + "rustix 0.38.28", "windows-sys 0.48.0", ] @@ -1331,17 +1424,26 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" dependencies = [ "wasm-bindgen", ] @@ -1356,7 +1458,7 @@ dependencies = [ "anyhow", "base64", "bytecount", - "clap 4.4.3", + "clap 4.4.11", "fancy-regex", "fraction", "getrandom", @@ -1396,21 +1498,21 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.148" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libm" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libsqlite3-sys" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" dependencies = [ "cc", "pkg-config", @@ -1431,9 +1533,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.7" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lld_rs" @@ -1450,9 +1552,9 @@ dependencies = [ [[package]] name = "llvm-sys" -version = "140.1.2" +version = "140.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69b285f8682531b9b394dd9891977a2a28c47006e491bda944e1ca62ebab2664" +checksum = "e3dc78e9857c0231ec11e3bdccf63870493fdc7d0570b0ea7d50bf5df0cb1a0c" dependencies = [ "cc", "lazy_static", @@ -1463,9 +1565,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -1497,7 +1599,7 @@ checksum = "a1d849148dbaf9661a6151d1ca82b13bb4c4c128146a88d05253b38d4e2f496c" dependencies = [ "beef", "fnv", - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", "regex-syntax 0.6.29", "syn 1.0.109", @@ -1505,18 +1607,19 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -1550,9 +1653,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "wasi", @@ -1577,6 +1680,17 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -1688,9 +1802,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", "libm", @@ -1702,7 +1816,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.2", + "hermit-abi 0.3.3", "libc", ] @@ -1717,17 +1831,17 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -1742,9 +1856,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.33", + "syn 2.0.41", ] [[package]] @@ -1755,9 +1869,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" dependencies = [ "cc", "libc", @@ -1767,15 +1881,15 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.5.1" +version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" [[package]] name = "parking" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" @@ -1789,9 +1903,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", @@ -1817,29 +1931,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" - -[[package]] -name = "pin-project" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.3" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" -dependencies = [ - "proc-macro2 1.0.67", - "quote 1.0.33", - "syn 2.0.33", -] +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project-lite" @@ -1853,6 +1947,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -1916,7 +2021,7 @@ dependencies = [ "encoding_rs", "encoding_rs_io", "env_logger", - "indexmap 2.0.0", + "indexmap 2.1.0", "insta", "log", "plc_ast", @@ -1967,9 +2072,9 @@ name = "plc_xml" version = "0.1.0" dependencies = [ "html-escape", - "indexmap 2.0.0", + "indexmap 2.1.0", "insta", - "itertools", + "itertools 0.11.0", "logos", "plc_ast", "plc_diagnostics", @@ -1994,6 +2099,26 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "polling" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf63fa624ab313c11656b4cda960bfc46c410187ad493c41f6ba2d8c1e991c9e" +dependencies = [ + "cfg-if", + "concurrent-queue", + "pin-project-lite", + "rustix 0.38.28", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2017,7 +2142,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", "syn 1.0.109", "version_check", @@ -2029,7 +2154,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", "version_check", ] @@ -2045,9 +2170,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" dependencies = [ "unicode-ident", ] @@ -2077,7 +2202,7 @@ version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", ] [[package]] @@ -2112,9 +2237,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -2122,46 +2247,44 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.9.5" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", ] [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", ] [[package]] @@ -2172,9 +2295,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" @@ -2213,16 +2336,14 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.2" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" dependencies = [ - "byteorder", "const-oid", "digest", "num-bigint-dig", "num-integer", - "num-iter", "num-traits", "pkcs1", "pkcs8", @@ -2251,7 +2372,7 @@ dependencies = [ name = "rustc_llvm_coverage" version = "0.1.0" dependencies = [ - "inkwell", + "inkwell 0.2.0 (git+https://github.com/corbanvilla/inkwell.git?branch=v2.0)", "libc", "llvm-sys", "rustc_llvm", @@ -2259,9 +2380,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.25" +version = "0.37.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4eb579851244c2c03e7c24f501c3432bed80b8f720af1d6e5b0e0f01555a035" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" dependencies = [ "bitflags 1.3.2", "errno", @@ -2273,15 +2394,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.13" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", - "linux-raw-sys 0.4.7", - "windows-sys 0.48.0", + "linux-raw-sys 0.4.12", + "windows-sys 0.52.0", ] [[package]] @@ -2293,10 +2414,10 @@ dependencies = [ "encoding_rs", "encoding_rs_io", "generational-arena", - "indexmap 2.0.0", - "inkwell", + "indexmap 2.1.0", + "inkwell 0.2.0 (git+https://github.com/HaithemLamri/inkwell.git?branch=master)", "insta", - "itertools", + "itertools 0.11.0", "lazy_static", "lld_rs", "log", @@ -2325,9 +2446,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "schannel" @@ -2369,35 +2490,35 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.33", + "syn 2.0.41", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -2436,16 +2557,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.33", + "syn 2.0.41", ] [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -2454,9 +2575,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -2471,9 +2592,9 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "signature" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", "rand_core", @@ -2481,9 +2602,9 @@ dependencies = [ [[package]] name = "similar" -version = "2.2.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" +checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597" [[package]] name = "slab" @@ -2496,15 +2617,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -2512,9 +2633,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2537,9 +2658,9 @@ dependencies = [ [[package]] name = "spki" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", "der", @@ -2547,20 +2668,20 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" dependencies = [ - "itertools", + "itertools 0.12.0", "nom", "unicode_categories", ] [[package]] name = "sqlx" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" +checksum = "dba03c279da73694ef99763320dea58b51095dfe87d001b1d4b5fe78ba8763cf" dependencies = [ "sqlx-core", "sqlx-macros", @@ -2571,12 +2692,12 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" +checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" dependencies = [ "ahash", - "async-io", + "async-io 1.13.0", "async-std", "atoi", "byteorder", @@ -2585,7 +2706,7 @@ dependencies = [ "crossbeam-queue", "dotenvy", "either", - "event-listener", + "event-listener 2.5.3", "futures-channel", "futures-core", "futures-intrusive", @@ -2593,7 +2714,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.0.0", + "indexmap 2.1.0", "log", "memchr", "native-tls", @@ -2612,11 +2733,11 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" +checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", "sqlx-core", "sqlx-macros-core", @@ -2625,17 +2746,18 @@ dependencies = [ [[package]] name = "sqlx-macros-core" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" +checksum = "d0bd4519486723648186a08785143599760f7cc81c52334a55d6a83ea1e20841" dependencies = [ "async-std", + "atomic-write-file", "dotenvy", "either", "heck", "hex", "once_cell", - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", "serde", "serde_json", @@ -2650,13 +2772,13 @@ dependencies = [ [[package]] name = "sqlx-mysql" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" +checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" dependencies = [ "atoi", "base64", - "bitflags 2.4.0", + "bitflags 2.4.1", "byteorder", "bytes", "crc", @@ -2692,13 +2814,13 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" +checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" dependencies = [ "atoi", "base64", - "bitflags 2.4.0", + "bitflags 2.4.1", "byteorder", "crc", "dotenvy", @@ -2731,9 +2853,9 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" +checksum = "210976b7d948c7ba9fced8ca835b11cbb2d677c59c79de41ac0d397e14547490" dependencies = [ "atoi", "flume", @@ -2749,6 +2871,7 @@ dependencies = [ "sqlx-core", "tracing", "url", + "urlencoding", ] [[package]] @@ -2791,27 +2914,27 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.33" +version = "2.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9caece70c63bfba29ec2fed841a09851b14a235c60010fa4de58089b6c025668" +checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", "unicode-ident", ] [[package]] name = "sysinfo" -version = "0.29.10" +version = "0.29.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5" +checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2845,22 +2968,22 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", - "fastrand 2.0.0", + "fastrand 2.0.1", "redox_syscall", - "rustix 0.38.13", + "rustix 0.38.28", "windows-sys 0.48.0", ] [[package]] name = "termcolor" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" dependencies = [ "winapi-util", ] @@ -2873,31 +2996,32 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.33", + "syn 2.0.41", ] [[package]] name = "time" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", + "powerfmt", "serde", "time-core", "time-macros", @@ -2935,9 +3059,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" dependencies = [ "backtrace", "bytes", @@ -2945,15 +3069,15 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", - "socket2 0.5.4", + "socket2 0.5.5", "windows-sys 0.48.0", ] [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -2980,11 +3104,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -2993,41 +3116,41 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.33", + "syn 2.0.41", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" [[package]] name = "unicode-ident" @@ -3052,9 +3175,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unicode-xid" @@ -3070,20 +3193,26 @@ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8-width" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" [[package]] name = "utf8parse" @@ -3093,15 +3222,15 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" [[package]] name = "value-bag" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" +checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" [[package]] name = "vcpkg" @@ -3117,9 +3246,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "waker-fn" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" [[package]] name = "want" @@ -3138,9 +3267,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3148,24 +3277,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.33", + "syn 2.0.41", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" dependencies = [ "cfg-if", "js-sys", @@ -3175,9 +3304,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" dependencies = [ "quote 1.0.33", "wasm-bindgen-macro-support", @@ -3185,28 +3314,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ - "proc-macro2 1.0.67", + "proc-macro2 1.0.70", "quote 1.0.33", - "syn 2.0.33", + "syn 2.0.41", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" dependencies = [ "js-sys", "wasm-bindgen", @@ -3221,7 +3350,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.13", + "rustix 0.38.28", ] [[package]] @@ -3248,9 +3377,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -3262,10 +3391,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ "windows-targets 0.48.5", ] @@ -3288,6 +3417,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -3318,6 +3456,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -3330,6 +3483,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -3342,6 +3501,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -3354,6 +3519,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -3366,6 +3537,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -3378,6 +3555,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -3390,6 +3573,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -3402,6 +3591,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winreg" version = "0.50.0" @@ -3433,7 +3628,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-std", - "clap 4.4.3", + "clap 4.4.11", "plc_ast", "plc_source", "rusty", @@ -3460,8 +3655,28 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "zerocopy" +version = "0.7.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" +dependencies = [ + "proc-macro2 1.0.70", + "quote 1.0.33", + "syn 2.0.41", +] + [[package]] name = "zeroize" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/Cargo.toml b/Cargo.toml index e5e455fc305..32eda3902a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,9 @@ members = [ default-members = [".", "compiler/plc_driver", "compiler/plc_xml"] [workspace.dependencies] -inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } +#inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } +inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", features = ["llvm14-0"] } + encoding_rs = "0.8" encoding_rs_io = "0.1" log = "0.4" diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index 2a6b7277d5d..596c0b09292 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -58,4 +58,7 @@ extern "C" { pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); pub fn LLVMRustCoverageMappingVersion() -> u32; + + + } diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 0204bf8a0c7..468adcb7288 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -5,6 +5,8 @@ * https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs#L220-L221 */ +const VAR_ALIGN_BYTES: u32 = 8; + mod ffi; pub mod types; @@ -12,6 +14,11 @@ use types::*; use inkwell::values::{AsValueRef, FunctionValue, GlobalValue}; use std::ffi::CString; +use libc::c_uint; + +use inkwell::module::Module; + + /// Calls llvm::createPGOFuncNameVar() with the given function instance's /// mangled function name. The LLVM API returns an llvm::GlobalVariable @@ -35,25 +42,25 @@ pub fn write_filenames_section_to_buffer<'a>( ffi::LLVMRustCoverageWriteFilenamesSectionToBuffer(c_str_vec.as_ptr(), c_str_vec.len(), buffer); } } - -// pub(crate) fn write_mapping_to_buffer( -// virtual_file_mapping: Vec, -// expressions: Vec, -// mapping_regions: Vec, -// buffer: &RustString, -// ) { -// unsafe { -// llvm::LLVMRustCoverageWriteMappingToBuffer( -// virtual_file_mapping.as_ptr(), -// virtual_file_mapping.len() as c_uint, -// expressions.as_ptr(), -// expressions.len() as c_uint, -// mapping_regions.as_ptr(), -// mapping_regions.len() as c_uint, -// buffer, -// ); -// } -// } +//create params , call fucntion in codegen, print the buffer +pub fn write_mapping_to_buffer( + virtual_file_mapping: Vec, + expressions: Vec, + mapping_regions: Vec, + buffer: &RustString, +) { + unsafe { + ffi::LLVMRustCoverageWriteMappingToBuffer( + virtual_file_mapping.as_ptr(), + virtual_file_mapping.len() as c_uint, + expressions.as_ptr(), + expressions.len() as c_uint, + mapping_regions.as_ptr(), + mapping_regions.len() as c_uint, + buffer, + ); + } +} pub fn hash_str(strval: &str) -> u64 { let strval = CString::new(strval).expect("null error converting hashable str to C string"); @@ -68,27 +75,33 @@ pub fn mapping_version() -> u32 { unsafe { ffi::LLVMRustCoverageMappingVersion() } } -// pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, cov_data_val: &'ll llvm::Value) { -// let covmap_var_name = llvm::build_string(|s| unsafe { -// llvm::LLVMRustCoverageWriteMappingVarNameToString(s); -// }) -// .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); -// debug!("covmap var name: {:?}", covmap_var_name); - -// let covmap_section_name = llvm::build_string(|s| unsafe { -// llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); -// }) -// .expect("Rust Coverage section name failed UTF-8 conversion"); -// debug!("covmap section name: {:?}", covmap_section_name); +pub fn save_cov_data_to_mod<'ctx>(func: &FunctionValue<'ctx>, cov_data_val: &GlobalValue<'ctx>) { + //global_value + let covmap_var_name = GlobalValue::build_string(|s: &RustString| unsafe { + ffi::LLVMRustCoverageWriteMappingVarNameToString(s); + }); + //.expect("Rust Coverage Mapping var name failed UTF-8 conversion"); + //debug!("covmap var name: {:?}", covmap_var_name); + //build_string not found + // let covmap_section_name = + + // GlobalValue::build_string(|s : &RustString| unsafe { + // ffi::LLVMRustCoverageWriteMapSectionNameToString(func, s); + // }) ; -// let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); -// llvm::set_initializer(llglobal, cov_data_val); -// llvm::set_global_constant(llglobal, true); -// llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); -// llvm::set_section(llglobal, &covmap_section_name); -// llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); -// cx.add_used_global(llglobal); -// } + // //.expect("Rust Coverage section name failed UTF-8 conversion"); + // //debug!("covmap section name: {:?}", covmap_section_name); + // // add_global not found in inkwell global value and func.llmod doesn't exist in func + // + // let llglobal = Module::add_global(func, func.val_ty(cov_data_val), &covmap_var_name); + // GlobalValue::set_initializer(llglobal, cov_data_val); + // //set_global_cst not found in inkwell global value + // GlobalValue::set_constant(llglobal, true); + // GlobalValue::set_linkage(llglobal, GlobalValue::Linkage::PrivateLinkage); + // GlobalValue::set_section(llglobal, Some(&covmap_section_name)); + // GlobalValue::set_alignment(llglobal, VAR_ALIGN_BYTES); + //func.add_used_global(llglobal); +} // pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( // cx: &CodegenCx<'ll, 'tcx>, diff --git a/compiler/rustc_llvm_coverage/src/types.rs b/compiler/rustc_llvm_coverage/src/types.rs index e39d53e665a..9956ee9d8a6 100644 --- a/compiler/rustc_llvm_coverage/src/types.rs +++ b/compiler/rustc_llvm_coverage/src/types.rs @@ -139,7 +139,7 @@ pub struct CounterMappingRegion { } impl CounterMappingRegion { - pub(crate) fn code_region( + pub fn code_region( counter: Counter, file_id: u32, start_line: u32, @@ -294,7 +294,7 @@ impl Counter { } /// Constructs a new `Counter` of kind `Expression`. - pub(crate) fn expression(expression_id: ExpressionId) -> Self { + pub fn expression(expression_id: ExpressionId) -> Self { Self { kind: CounterKind::Expression, id: expression_id.as_u32() } } } diff --git a/src/codegen.rs b/src/codegen.rs index 9a39f7b1fcf..b7e1bebe92f 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -39,7 +39,7 @@ use inkwell::{ use plc_ast::ast::{CompilationUnit, LinkageType}; use plc_diagnostics::diagnostics::Diagnostic; use plc_source::source_location::SourceLocation; -use rustc_llvm_coverage; +use rustc_llvm_coverage::{self, types::{CounterExpression, CounterMappingRegion}}; mod debug; pub(crate) mod generators; @@ -47,6 +47,9 @@ mod llvm_index; mod llvm_typesystem; #[cfg(test)] mod tests; +use types::*; +use rustc_llvm_coverage::*; + /// A wrapper around the LLVM context to allow passing it without exposing the inkwell dependencies pub struct CodegenContext(Context); @@ -237,15 +240,47 @@ impl<'ink> CodeGen<'ink> { .join("") ); - // println!("{:#?}", llvm_index); - let prg_func = llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); + println!("{:#?}", llvm_index); + let prg_func: inkwell::values::FunctionValue<'_> = llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); println!("prg: {:#?}", prg_func); let global_func_var = rustc_llvm_coverage::create_pgo_func_name_var(&prg_func); println!("global_func_var: {:#?}", global_func_var); - + let global_vars: Vec<_> = self.module.get_globals().collect(); println!("global_vars: {:#?}", global_vars); - + //create write_mapping_to_buffer params + //virtual_file_mapping + let virtual_file_mapping : Vec = vec![0x0] ; + + let counter1 = Counter::counter_value_reference(CounterId::new(1)); + let counter2 = Counter::expression(ExpressionId::new(2)); + let counter3 = Counter::counter_value_reference(CounterId::new(3)); + + // Creating a vector of CounterExpression instances + let expressions: Vec = vec![ + CounterExpression::new(counter1, ExprKind::Add, counter2), + CounterExpression::new(counter2, ExprKind::Subtract, counter3), + // Add more CounterExpression instances as needed + ]; + //mapping_regions + let mapping_regions : Vec = vec![CounterMappingRegion::code_region(counter1,0,0,0,3,10), + CounterMappingRegion::code_region(counter3,0,4,0,9,10) + ]; + //buffer + let buffer = rustc_llvm_coverage::types::RustString { bytes: RefCell::new(Vec::new()) }; + write_mapping_to_buffer(virtual_file_mapping, expressions, mapping_regions, &buffer); + //print the buffer + //print the buffer + println!( + "Buffer: {:#?}", + buffer + .bytes + .borrow() + .iter() + .map(|it| format!("{:02x}", it)) + .collect::>() + .join("") + ); #[cfg(feature = "verify")] { self.module From 458e314f67dff967ab76faaceab27013582f2e71 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 13 Dec 2023 09:56:28 +0000 Subject: [PATCH 24/70] commit last modifications' --- Cargo.toml | 2 +- compiler/rustc_llvm_coverage/src/lib.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 32eda3902a3..25882178b4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,7 @@ default-members = [".", "compiler/plc_driver", "compiler/plc_xml"] [workspace.dependencies] #inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } -inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", features = ["llvm14-0"] } +inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } encoding_rs = "0.8" encoding_rs_io = "0.1" diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 468adcb7288..db761d85235 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -13,6 +13,7 @@ pub mod types; use types::*; use inkwell::values::{AsValueRef, FunctionValue, GlobalValue}; + use std::ffi::CString; use libc::c_uint; @@ -77,7 +78,7 @@ pub fn mapping_version() -> u32 { pub fn save_cov_data_to_mod<'ctx>(func: &FunctionValue<'ctx>, cov_data_val: &GlobalValue<'ctx>) { //global_value - let covmap_var_name = GlobalValue::build_string(|s: &RustString| unsafe { + let covmap_var_nam = GlobalValue::build_string(s | unsafe { ffi::LLVMRustCoverageWriteMappingVarNameToString(s); }); //.expect("Rust Coverage Mapping var name failed UTF-8 conversion"); From c66ec4508a28d1981ba3b7b8f366c9ebe8f467fe Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 15 Dec 2023 05:54:41 +0000 Subject: [PATCH 25/70] last changes --- Cargo.lock | 62 ++++++++----------------- compiler/rustc_llvm_coverage/Cargo.toml | 4 +- compiler/rustc_llvm_coverage/src/lib.rs | 21 ++++----- 3 files changed, 32 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66a02b809f4..a8ba82ef267 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -606,9 +606,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -617,22 +617,21 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" dependencies = [ "cfg-if", "crossbeam-utils", @@ -640,9 +639,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" dependencies = [ "cfg-if", ] @@ -1144,9 +1143,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hkdf" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ "hmac", ] @@ -1313,43 +1312,20 @@ dependencies = [ [[package]] name = "inkwell" version = "0.2.0" -source = "git+https://github.com/HaithemLamri/inkwell.git?branch=master#955375fcacd3bfc04e2862ca4e375e30c07392dd" +source = "git+https://github.com/HaithemLamri/inkwell.git?branch=master#fa4597b5905c64c49422f0fbae05451129b5d401" dependencies = [ "either", - "inkwell_internals 0.8.0 (git+https://github.com/HaithemLamri/inkwell.git?branch=master)", + "inkwell_internals", "libc", "llvm-sys", "once_cell", "thiserror", ] -[[package]] -name = "inkwell" -version = "0.2.0" -source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" -dependencies = [ - "either", - "inkwell_internals 0.8.0 (git+https://github.com/corbanvilla/inkwell.git?branch=v2.0)", - "libc", - "llvm-sys", - "once_cell", - "parking_lot", -] - -[[package]] -name = "inkwell_internals" -version = "0.8.0" -source = "git+https://github.com/HaithemLamri/inkwell.git?branch=master#955375fcacd3bfc04e2862ca4e375e30c07392dd" -dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.41", -] - [[package]] name = "inkwell_internals" version = "0.8.0" -source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" +source = "git+https://github.com/HaithemLamri/inkwell.git?branch=master#fa4597b5905c64c49422f0fbae05451129b5d401" dependencies = [ "proc-macro2 1.0.70", "quote 1.0.33", @@ -2372,7 +2348,7 @@ dependencies = [ name = "rustc_llvm_coverage" version = "0.1.0" dependencies = [ - "inkwell 0.2.0 (git+https://github.com/corbanvilla/inkwell.git?branch=v2.0)", + "inkwell", "libc", "llvm-sys", "rustc_llvm", @@ -2415,7 +2391,7 @@ dependencies = [ "encoding_rs_io", "generational-arena", "indexmap 2.1.0", - "inkwell 0.2.0 (git+https://github.com/HaithemLamri/inkwell.git?branch=master)", + "inkwell", "insta", "itertools 0.11.0", "lazy_static", @@ -3657,18 +3633,18 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zerocopy" -version = "0.7.30" +version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" +checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.30" +version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" +checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2 1.0.70", "quote 1.0.33", diff --git a/compiler/rustc_llvm_coverage/Cargo.toml b/compiler/rustc_llvm_coverage/Cargo.toml index 0def989f990..a082e702ad6 100644 --- a/compiler/rustc_llvm_coverage/Cargo.toml +++ b/compiler/rustc_llvm_coverage/Cargo.toml @@ -8,5 +8,7 @@ edition = "2021" [dependencies] libc = "0.2.73" rustc_llvm = { path = "../rustc_llvm" } -inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } +#inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } +inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } + llvm-sys = { package = "llvm-sys", version = "140.0.2" } diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index db761d85235..1909cff340d 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -12,12 +12,13 @@ pub mod types; use types::*; -use inkwell::values::{AsValueRef, FunctionValue, GlobalValue}; +use inkwell::{values::{AsValueRef, FunctionValue, GlobalValue}, types::{AsTypeRef, AnyType}}; use std::ffi::CString; use libc::c_uint; use inkwell::module::Module; +use inkwell::types::BasicType; @@ -78,24 +79,22 @@ pub fn mapping_version() -> u32 { pub fn save_cov_data_to_mod<'ctx>(func: &FunctionValue<'ctx>, cov_data_val: &GlobalValue<'ctx>) { //global_value - let covmap_var_nam = GlobalValue::build_string(s | unsafe { + let covmap_var_nam = unsafe { ffi::LLVMRustCoverageWriteMappingVarNameToString(s); - }); + }; //.expect("Rust Coverage Mapping var name failed UTF-8 conversion"); //debug!("covmap var name: {:?}", covmap_var_name); //build_string not found - // let covmap_section_name = - - // GlobalValue::build_string(|s : &RustString| unsafe { - // ffi::LLVMRustCoverageWriteMapSectionNameToString(func, s); - // }) ; + let covmap_section_name = unsafe { + ffi::LLVMRustCoverageWriteMapSectionNameToString(func.llmod(), s); + } ; // //.expect("Rust Coverage section name failed UTF-8 conversion"); // //debug!("covmap section name: {:?}", covmap_section_name); // // add_global not found in inkwell global value and func.llmod doesn't exist in func - // - // let llglobal = Module::add_global(func, func.val_ty(cov_data_val), &covmap_var_name); - // GlobalValue::set_initializer(llglobal, cov_data_val); + // func.ty + let llglobal = Module::add_global(&func.llmod(), func.val_ty(cov_data_val.as_value_ref()), &covmap_var_name,func.get_name().to_str().unwrap()); + //GlobalValue::set_initializer(llglobal, cov_data_val); // //set_global_cst not found in inkwell global value // GlobalValue::set_constant(llglobal, true); // GlobalValue::set_linkage(llglobal, GlobalValue::Linkage::PrivateLinkage); From 25a8a2cad3cb44f80b86ea0f69d9a6708b12792c Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Fri, 15 Dec 2023 15:28:17 +0000 Subject: [PATCH 26/70] get write func mod working --- Cargo.lock | 6 +-- Cargo.toml | 4 +- compiler/rustc_llvm_coverage/Cargo.toml | 4 +- compiler/rustc_llvm_coverage/src/ffi.rs | 14 ++--- compiler/rustc_llvm_coverage/src/lib.rs | 63 ++++++++++++---------- compiler/rustc_llvm_coverage/src/types.rs | 4 ++ scripts/build.sh | 4 +- src/codegen.rs | 64 +++++++++++++++-------- 8 files changed, 95 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8ba82ef267..8647de0571f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1312,20 +1312,20 @@ dependencies = [ [[package]] name = "inkwell" version = "0.2.0" -source = "git+https://github.com/HaithemLamri/inkwell.git?branch=master#fa4597b5905c64c49422f0fbae05451129b5d401" +source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" dependencies = [ "either", "inkwell_internals", "libc", "llvm-sys", "once_cell", - "thiserror", + "parking_lot", ] [[package]] name = "inkwell_internals" version = "0.8.0" -source = "git+https://github.com/HaithemLamri/inkwell.git?branch=master#fa4597b5905c64c49422f0fbae05451129b5d401" +source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" dependencies = [ "proc-macro2 1.0.70", "quote 1.0.33", diff --git a/Cargo.toml b/Cargo.toml index 25882178b4f..3025e4a0fb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,8 +76,8 @@ members = [ default-members = [".", "compiler/plc_driver", "compiler/plc_xml"] [workspace.dependencies] -#inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } -inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } +inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } +# inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } encoding_rs = "0.8" encoding_rs_io = "0.1" diff --git a/compiler/rustc_llvm_coverage/Cargo.toml b/compiler/rustc_llvm_coverage/Cargo.toml index a082e702ad6..4e81d32120f 100644 --- a/compiler/rustc_llvm_coverage/Cargo.toml +++ b/compiler/rustc_llvm_coverage/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" [dependencies] libc = "0.2.73" rustc_llvm = { path = "../rustc_llvm" } -#inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } -inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } +inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } +# inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } llvm-sys = { package = "llvm-sys", version = "140.0.2" } diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index 596c0b09292..800a9ec82e7 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -8,13 +8,7 @@ use libc::{c_char, c_uint, c_void, size_t}; use std::slice; use super::types::*; -use llvm_sys::prelude::LLVMValueRef; - -#[repr(C)] -pub struct Module(c_void); - -// #[repr(C)] -// pub struct Value(c_void); +use llvm_sys::prelude::{LLVMModuleRef, LLVMValueRef}; /// Appending to a Rust string -- used by RawRustStringOstream. #[no_mangle] @@ -49,16 +43,14 @@ extern "C" { pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); + pub fn LLVMRustCoverageWriteMapSectionNameToString(M: LLVMModuleRef, Str: &RustString); #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); + pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: LLVMModuleRef, Str: &RustString); #[allow(improper_ctypes)] pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); pub fn LLVMRustCoverageMappingVersion() -> u32; - - } diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 1909cff340d..3ac85abad34 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -7,21 +7,25 @@ const VAR_ALIGN_BYTES: u32 = 8; +use std::string::FromUtf8Error; + mod ffi; pub mod types; use types::*; -use inkwell::{values::{AsValueRef, FunctionValue, GlobalValue}, types::{AsTypeRef, AnyType}}; +use inkwell::{ + module::Linkage, + types::{AnyType, AsTypeRef, StructType}, + values::{AsValueRef, FunctionValue, GlobalValue, StructValue}, +}; -use std::ffi::CString; use libc::c_uint; +use std::ffi::CString; use inkwell::module::Module; use inkwell::types::BasicType; - - /// Calls llvm::createPGOFuncNameVar() with the given function instance's /// mangled function name. The LLVM API returns an llvm::GlobalVariable /// containing the function name, with the specific variable name and linkage @@ -77,30 +81,35 @@ pub fn mapping_version() -> u32 { unsafe { ffi::LLVMRustCoverageMappingVersion() } } -pub fn save_cov_data_to_mod<'ctx>(func: &FunctionValue<'ctx>, cov_data_val: &GlobalValue<'ctx>) { - //global_value - let covmap_var_nam = unsafe { - ffi::LLVMRustCoverageWriteMappingVarNameToString(s); +/* == TODO - Refactor these helpers out */ +pub fn build_string(sr: &RustString) -> Result { + String::from_utf8(sr.bytes.borrow().clone()) +} +/* == END TODO */ + +pub fn save_cov_data_to_mod<'ctx>(module: &Module<'ctx>, cov_data_val: StructValue<'ctx>) { + let covmap_var_name = { + let mut s = RustString::new(); + unsafe { + ffi::LLVMRustCoverageWriteMappingVarNameToString(&mut s); + } + build_string(&mut s).expect("Rust Coverage Mapping var name failed UTF-8 conversion") + }; + + let covmap_section_name = { + let mut s = RustString::new(); + unsafe { + ffi::LLVMRustCoverageWriteMapSectionNameToString(module.as_mut_ptr(), &mut s); + } + build_string(&mut s).expect("Rust Coverage Mapping section name failed UTF-8 conversion") }; - //.expect("Rust Coverage Mapping var name failed UTF-8 conversion"); - //debug!("covmap var name: {:?}", covmap_var_name); - //build_string not found - let covmap_section_name = unsafe { - ffi::LLVMRustCoverageWriteMapSectionNameToString(func.llmod(), s); - } ; - - // //.expect("Rust Coverage section name failed UTF-8 conversion"); - // //debug!("covmap section name: {:?}", covmap_section_name); - // // add_global not found in inkwell global value and func.llmod doesn't exist in func - // func.ty - let llglobal = Module::add_global(&func.llmod(), func.val_ty(cov_data_val.as_value_ref()), &covmap_var_name,func.get_name().to_str().unwrap()); - //GlobalValue::set_initializer(llglobal, cov_data_val); - // //set_global_cst not found in inkwell global value - // GlobalValue::set_constant(llglobal, true); - // GlobalValue::set_linkage(llglobal, GlobalValue::Linkage::PrivateLinkage); - // GlobalValue::set_section(llglobal, Some(&covmap_section_name)); - // GlobalValue::set_alignment(llglobal, VAR_ALIGN_BYTES); - //func.add_used_global(llglobal); + + let llglobal = module.add_global(cov_data_val.get_type(), None, covmap_var_name.as_str()); + llglobal.set_initializer(&cov_data_val); + llglobal.set_constant(true); + llglobal.set_linkage(Linkage::Private); + llglobal.set_section(Some(&covmap_section_name)); + llglobal.set_alignment(VAR_ALIGN_BYTES); } // pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( diff --git a/compiler/rustc_llvm_coverage/src/types.rs b/compiler/rustc_llvm_coverage/src/types.rs index 9956ee9d8a6..2ee55556654 100644 --- a/compiler/rustc_llvm_coverage/src/types.rs +++ b/compiler/rustc_llvm_coverage/src/types.rs @@ -16,6 +16,10 @@ pub struct RustString { } impl RustString { + pub fn new() -> Self { + Self { bytes: RefCell::new(Vec::new()) } + } + pub fn len(&self) -> usize { self.bytes.borrow().len() } diff --git a/scripts/build.sh b/scripts/build.sh index b6295d4c9f0..b0309bbe4a4 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -482,7 +482,9 @@ if [[ $build -ne 0 ]]; then # Test a program echo "-----------------------------------" echo "Running on example program:" - ./target/debug/plc ./examples/simple_program.st + ./target/debug/plc --ir ./examples/simple_program.st + echo "-----------------------------------" + cat ./simple_program.st.ll echo "-----------------------------------" #Build the standard functions # run_std_build diff --git a/src/codegen.rs b/src/codegen.rs index b7e1bebe92f..ea7ed088b87 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -39,7 +39,10 @@ use inkwell::{ use plc_ast::ast::{CompilationUnit, LinkageType}; use plc_diagnostics::diagnostics::Diagnostic; use plc_source::source_location::SourceLocation; -use rustc_llvm_coverage::{self, types::{CounterExpression, CounterMappingRegion}}; +use rustc_llvm_coverage::{ + self, + types::{CounterExpression, CounterMappingRegion}, +}; mod debug; pub(crate) mod generators; @@ -47,9 +50,8 @@ mod llvm_index; mod llvm_typesystem; #[cfg(test)] mod tests; -use types::*; use rustc_llvm_coverage::*; - +use types::*; /// A wrapper around the LLVM context to allow passing it without exposing the inkwell dependencies pub struct CodegenContext(Context); @@ -240,47 +242,65 @@ impl<'ink> CodeGen<'ink> { .join("") ); - println!("{:#?}", llvm_index); - let prg_func: inkwell::values::FunctionValue<'_> = llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); + // println!("{:#?}", llvm_index); + let prg_func: inkwell::values::FunctionValue<'_> = + llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); println!("prg: {:#?}", prg_func); let global_func_var = rustc_llvm_coverage::create_pgo_func_name_var(&prg_func); println!("global_func_var: {:#?}", global_func_var); - + let global_vars: Vec<_> = self.module.get_globals().collect(); println!("global_vars: {:#?}", global_vars); //create write_mapping_to_buffer params //virtual_file_mapping - let virtual_file_mapping : Vec = vec![0x0] ; - + let virtual_file_mapping: Vec = vec![0x0]; + let counter1 = Counter::counter_value_reference(CounterId::new(1)); let counter2 = Counter::expression(ExpressionId::new(2)); let counter3 = Counter::counter_value_reference(CounterId::new(3)); - + // Creating a vector of CounterExpression instances let expressions: Vec = vec![ CounterExpression::new(counter1, ExprKind::Add, counter2), CounterExpression::new(counter2, ExprKind::Subtract, counter3), // Add more CounterExpression instances as needed ]; - //mapping_regions - let mapping_regions : Vec = vec![CounterMappingRegion::code_region(counter1,0,0,0,3,10), - CounterMappingRegion::code_region(counter3,0,4,0,9,10) + //mapping_regions + let mapping_regions: Vec = vec![ + CounterMappingRegion::code_region(counter1, 0, 0, 0, 3, 10), + CounterMappingRegion::code_region(counter3, 0, 4, 0, 9, 10), ]; - //buffer + //buffer let buffer = rustc_llvm_coverage::types::RustString { bytes: RefCell::new(Vec::new()) }; write_mapping_to_buffer(virtual_file_mapping, expressions, mapping_regions, &buffer); - //print the buffer - //print the buffer + //print the buffer + //print the buffer println!( "Buffer: {:#?}", - buffer - .bytes - .borrow() - .iter() - .map(|it| format!("{:02x}", it)) - .collect::>() - .join("") + buffer.bytes.borrow().iter().map(|it| format!("{:02x}", it)).collect::>().join("") ); + + // cov mappping + // let struct_type = context.opaque_struct_type("my_struct"); + let i32_type = context.i32_type(); + let i32_zero = i32_type.const_int(0, false); + // TODO - generate this dynamically + let i32_42 = i32_type.const_int(42, false); + // TODO - replace this w/ cov mapping version + let i32_5 = i32_type.const_int(5, false); + + let cov_data_header = + context.const_struct(&[i32_zero.into(), i32_42.into(), i32_zero.into(), i32_5.into()], false); + + // TODO - generate this dynamically from cov mapping data + let i8_type = context.i8_type(); + let i8_zero = i8_type.const_int(0, false); + let cov_mapping_data = i8_type.const_array(&[i8_zero, i8_zero, i8_zero]); + + let cov_data_val = context.const_struct(&[cov_data_header.into(), cov_mapping_data.into()], false); + + rustc_llvm_coverage::save_cov_data_to_mod(&self.module, cov_data_val); + #[cfg(feature = "verify")] { self.module From f86f14157651c93260435dfc4631d873f111e50b Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Fri, 15 Dec 2023 17:45:49 +0000 Subject: [PATCH 27/70] start write func record --- compiler/rustc_llvm_coverage/src/lib.rs | 76 ++++++++++++++----------- src/codegen.rs | 17 +++++- 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 3ac85abad34..782fc596d70 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -110,38 +110,48 @@ pub fn save_cov_data_to_mod<'ctx>(module: &Module<'ctx>, cov_data_val: StructVal llglobal.set_linkage(Linkage::Private); llglobal.set_section(Some(&covmap_section_name)); llglobal.set_alignment(VAR_ALIGN_BYTES); + // We will skip this for now... I don't think it's necessary (-Corban) + // cx.add_used_global(llglobal); } -// pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( -// cx: &CodegenCx<'ll, 'tcx>, -// func_name_hash: u64, -// func_record_val: &'ll llvm::Value, -// is_used: bool, -// ) { -// // Assign a name to the function record. This is used to merge duplicates. -// // -// // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that -// // are included-but-not-used. If (or when) Rust generates functions that are -// // included-but-not-used, note that a dummy description for a function included-but-not-used -// // in a Crate can be replaced by full description provided by a different Crate. The two kinds -// // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by -// // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. -// let func_record_var_name = format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); -// debug!("function record var name: {:?}", func_record_var_name); - -// let func_record_section_name = llvm::build_string(|s| unsafe { -// llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); -// }) -// .expect("Rust Coverage function record section name failed UTF-8 conversion"); -// debug!("function record section name: {:?}", func_record_section_name); - -// let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); -// llvm::set_initializer(llglobal, func_record_val); -// llvm::set_global_constant(llglobal, true); -// llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); -// llvm::set_visibility(llglobal, llvm::Visibility::Hidden); -// llvm::set_section(llglobal, &func_record_section_name); -// llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); -// llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); -// cx.add_used_global(llglobal); -// } +pub fn save_func_record_to_mod<'ctx>( + module: &Module<'ctx>, + func_name_hash: u64, + func_record_val: StructValue<'ctx>, + is_used: bool, +) { + // Assign a name to the function record. This is used to merge duplicates. + // + // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that + // are included-but-not-used. If (or when) Rust generates functions that are + // included-but-not-used, note that a dummy description for a function included-but-not-used + // in a Crate can be replaced by full description provided by a different Crate. The two kinds + // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by + // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. + let func_record_var_name = format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); + println!("function record var name: {:?}", func_record_var_name); + + let func_record_section_name = { + let mut s = RustString::new(); + unsafe { + ffi::LLVMRustCoverageWriteFuncSectionNameToString(module.as_mut_ptr(), &mut s); + } + build_string(&mut s).expect("Rust Coverage function record section name failed UTF-8 conversion") + }; + println!("function record section name: {:?}", func_record_section_name); + + let llglobal = module.add_global(func_record_val.get_type(), None, func_record_var_name.as_str()); + + // llvm::set_initializer(llglobal, func_record_val); + llglobal.set_initializer(&func_record_val); + + // llvm::set_global_constant(llglobal, true); + // llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); + // llvm::set_visibility(llglobal, llvm::Visibility::Hidden); + // llvm::set_section(llglobal, &func_record_section_name); + // llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); + // Use https://thedan64.github.io/inkwell/inkwell/values/struct.GlobalValue.html#method.set_comdat + // llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); + // We will skip this for now... I don't think it's necessary (-Corban) + // cx.add_used_global(llglobal); +} diff --git a/src/codegen.rs b/src/codegen.rs index ea7ed088b87..a07a36d4a2d 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -243,8 +243,9 @@ impl<'ink> CodeGen<'ink> { ); // println!("{:#?}", llvm_index); - let prg_func: inkwell::values::FunctionValue<'_> = - llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); + // let prg_func: inkwell::values::FunctionValue<'_> = + // llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); + let prg_func = self.module.get_function("prg").expect("Unable to get prg"); println!("prg: {:#?}", prg_func); let global_func_var = rustc_llvm_coverage::create_pgo_func_name_var(&prg_func); println!("global_func_var: {:#?}", global_func_var); @@ -301,6 +302,18 @@ impl<'ink> CodeGen<'ink> { rustc_llvm_coverage::save_cov_data_to_mod(&self.module, cov_data_val); + // func record + + let i64_type = context.i64_type(); + let i64_zero = i64_type.const_int(0, false); + let i64_one = i64_type.const_int(1, false); + let func_mapping_data = i8_type.const_array(&[i8_zero, i8_zero, i8_zero]); + let func_data_val = context.const_struct( + &[i64_one.into(), i32_zero.into(), i64_zero.into(), i64_zero.into(), func_mapping_data.into()], + false, + ); + rustc_llvm_coverage::save_func_record_to_mod(&self.module, 0x1234, func_data_val, true); + #[cfg(feature = "verify")] { self.module From fb3dba4eef732206dde46cc802f50e1723013f9a Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 20 Dec 2023 09:22:52 +0000 Subject: [PATCH 28/70] commit to merge --- Cargo.lock | 83 ++++++++++++++++--------- compiler/rustc_llvm_coverage/Cargo.toml | 5 ++ compiler/rustc_llvm_coverage/src/lib.rs | 53 +++++++--------- inkwell | 1 + 4 files changed, 82 insertions(+), 60 deletions(-) create mode 160000 inkwell diff --git a/Cargo.lock b/Cargo.lock index 8647de0571f..73cb1d54f7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,7 +133,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" dependencies = [ "concurrent-queue", - "event-listener 4.0.0", + "event-listener 4.0.1", "event-listener-strategy", "futures-core", "pin-project-lite", @@ -222,7 +222,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" dependencies = [ - "event-listener 4.0.0", + "event-listener 4.0.1", "event-listener-strategy", "pin-project-lite", ] @@ -255,9 +255,9 @@ dependencies = [ [[package]] name = "async-task" -version = "4.5.0" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" +checksum = "e1d90cd0b264dfdd8eb5bad0a2c217c1f88fa96a8573f40e7b12de23fb468f46" [[package]] name = "atoi" @@ -560,9 +560,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "core-foundation" @@ -794,9 +794,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" +checksum = "84f2cdcf274580f2d63697192d744727b3198894b1bf02923643bf59e2c26712" dependencies = [ "concurrent-queue", "parking", @@ -809,7 +809,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" dependencies = [ - "event-listener 4.0.0", + "event-listener 4.0.1", "pin-project-lite", ] @@ -1161,11 +1161,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1219,9 +1219,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -1234,7 +1234,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -1315,7 +1315,20 @@ version = "0.2.0" source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" dependencies = [ "either", - "inkwell_internals", + "inkwell_internals 0.8.0 (git+https://github.com/HaithemLamri/inkwell.git?branch=master)", + "libc", + "llvm-sys", + "once_cell", + "parking_lot", +] + +[[package]] +name = "inkwell" +version = "0.2.0" +source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" +dependencies = [ + "either", + "inkwell_internals 0.8.0 (git+https://github.com/corbanvilla/inkwell.git?branch=v2.0)", "libc", "llvm-sys", "once_cell", @@ -1332,6 +1345,16 @@ dependencies = [ "syn 2.0.41", ] +[[package]] +name = "inkwell_internals" +version = "0.8.0" +source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" +dependencies = [ + "proc-macro2 1.0.70", + "quote 1.0.33", + "syn 2.0.41", +] + [[package]] name = "insta" version = "1.34.0" @@ -2277,9 +2300,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "base64", "bytes", @@ -2348,7 +2371,7 @@ dependencies = [ name = "rustc_llvm_coverage" version = "0.1.0" dependencies = [ - "inkwell", + "inkwell 0.2.0 (git+https://github.com/corbanvilla/inkwell.git?branch=v2.0)", "libc", "llvm-sys", "rustc_llvm", @@ -2391,7 +2414,7 @@ dependencies = [ "encoding_rs_io", "generational-arena", "indexmap 2.1.0", - "inkwell", + "inkwell 0.2.0 (git+https://github.com/HaithemLamri/inkwell.git?branch=master)", "insta", "itertools 0.11.0", "lazy_static", @@ -2972,18 +2995,18 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2 1.0.70", "quote 1.0.33", @@ -2992,9 +3015,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "powerfmt", @@ -3011,9 +3034,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] @@ -3035,9 +3058,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", diff --git a/compiler/rustc_llvm_coverage/Cargo.toml b/compiler/rustc_llvm_coverage/Cargo.toml index 4e81d32120f..73e24a4fd4a 100644 --- a/compiler/rustc_llvm_coverage/Cargo.toml +++ b/compiler/rustc_llvm_coverage/Cargo.toml @@ -9,6 +9,11 @@ edition = "2021" libc = "0.2.73" rustc_llvm = { path = "../rustc_llvm" } inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } +<<<<<<< HEAD # inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } +======= +#inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } +#inkwell = {path = "../../inkwell"} +>>>>>>> 3086373 (some minor changes) llvm-sys = { package = "llvm-sys", version = "140.0.2" } diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 782fc596d70..10314270d66 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -81,37 +81,30 @@ pub fn mapping_version() -> u32 { unsafe { ffi::LLVMRustCoverageMappingVersion() } } -/* == TODO - Refactor these helpers out */ -pub fn build_string(sr: &RustString) -> Result { - String::from_utf8(sr.bytes.borrow().clone()) -} -/* == END TODO */ - -pub fn save_cov_data_to_mod<'ctx>(module: &Module<'ctx>, cov_data_val: StructValue<'ctx>) { - let covmap_var_name = { - let mut s = RustString::new(); - unsafe { - ffi::LLVMRustCoverageWriteMappingVarNameToString(&mut s); - } - build_string(&mut s).expect("Rust Coverage Mapping var name failed UTF-8 conversion") +pub fn save_cov_data_to_mod<'ctx>(func: &FunctionValue<'ctx>, cov_data_val: &GlobalValue<'ctx>) { + //global_value + let covmap_var_nam = unsafe { + ffi::LLVMRustCoverageWriteMappingVarNameToString(s); }; - - let covmap_section_name = { - let mut s = RustString::new(); - unsafe { - ffi::LLVMRustCoverageWriteMapSectionNameToString(module.as_mut_ptr(), &mut s); - } - build_string(&mut s).expect("Rust Coverage Mapping section name failed UTF-8 conversion") - }; - - let llglobal = module.add_global(cov_data_val.get_type(), None, covmap_var_name.as_str()); - llglobal.set_initializer(&cov_data_val); - llglobal.set_constant(true); - llglobal.set_linkage(Linkage::Private); - llglobal.set_section(Some(&covmap_section_name)); - llglobal.set_alignment(VAR_ALIGN_BYTES); - // We will skip this for now... I don't think it's necessary (-Corban) - // cx.add_used_global(llglobal); + //.expect("Rust Coverage Mapping var name failed UTF-8 conversion"); + //debug!("covmap var name: {:?}", covmap_var_name); + //build_string not found + let covmap_section_name = unsafe { + ffi::LLVMRustCoverageWriteMapSectionNameToString(func.llmod(), s); + } ; + + // //.expect("Rust Coverage section name failed UTF-8 conversion"); + // //debug!("covmap section name: {:?}", covmap_section_name); + // // add_global not found in inkwell global value and func.llmod doesn't exist in func + // func.ty + let llglobal = Module::add_global(&func.llmod(), func.val_ty(cov_data_val.as_value_ref()), &covmap_var_name,func.get_name().to_str().unwrap()); + //GlobalValue::set_initializer(llglobal, cov_data_val); + // //set_global_cst not found in inkwell global value + // GlobalValue::set_constant(llglobal, true); + // GlobalValue::set_linkage(llglobal, GlobalValue::Linkage::PrivateLinkage); + // GlobalValue::set_section(llglobal, Some(&covmap_section_name)); + // GlobalValue::set_alignment(llglobal, VAR_ALIGN_BYTES); + //func.add_used_global(llglobal); } pub fn save_func_record_to_mod<'ctx>( diff --git a/inkwell b/inkwell new file mode 160000 index 00000000000..fa4597b5905 --- /dev/null +++ b/inkwell @@ -0,0 +1 @@ +Subproject commit fa4597b5905c64c49422f0fbae05451129b5d401 From 87ccb4c0605665a2ca99d8898eb79928460bfc16 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 20 Dec 2023 09:26:48 +0000 Subject: [PATCH 29/70] some changes --- inkwell | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inkwell b/inkwell index fa4597b5905..e02803fcf18 160000 --- a/inkwell +++ b/inkwell @@ -1 +1 @@ -Subproject commit fa4597b5905c64c49422f0fbae05451129b5d401 +Subproject commit e02803fcf18ccc1f389cc36aa035b46045a9d9c0 From 7fedb3fe4a9248211325d5010956e50b333ad869 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 20 Dec 2023 09:28:14 +0000 Subject: [PATCH 30/70] update toml --- compiler/rustc_llvm_coverage/Cargo.toml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_llvm_coverage/Cargo.toml b/compiler/rustc_llvm_coverage/Cargo.toml index 73e24a4fd4a..e01fa91ab0a 100644 --- a/compiler/rustc_llvm_coverage/Cargo.toml +++ b/compiler/rustc_llvm_coverage/Cargo.toml @@ -9,11 +9,9 @@ edition = "2021" libc = "0.2.73" rustc_llvm = { path = "../rustc_llvm" } inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } -<<<<<<< HEAD -# inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } -======= +# inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } #inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } #inkwell = {path = "../../inkwell"} ->>>>>>> 3086373 (some minor changes) + llvm-sys = { package = "llvm-sys", version = "140.0.2" } From e626910853f6bd5f175afec9584829a16e8641a1 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 20 Dec 2023 09:43:38 +0000 Subject: [PATCH 31/70] Revert "update toml" This reverts commit 7fedb3fe4a9248211325d5010956e50b333ad869. --- compiler/rustc_llvm_coverage/Cargo.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_llvm_coverage/Cargo.toml b/compiler/rustc_llvm_coverage/Cargo.toml index e01fa91ab0a..73e24a4fd4a 100644 --- a/compiler/rustc_llvm_coverage/Cargo.toml +++ b/compiler/rustc_llvm_coverage/Cargo.toml @@ -9,9 +9,11 @@ edition = "2021" libc = "0.2.73" rustc_llvm = { path = "../rustc_llvm" } inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } - +<<<<<<< HEAD # inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } + +======= #inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } #inkwell = {path = "../../inkwell"} - +>>>>>>> 3086373 (some minor changes) llvm-sys = { package = "llvm-sys", version = "140.0.2" } From 56664ec115dfb24c7225c576ccb5cf3843cdb0b9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 20 Dec 2023 09:45:50 +0000 Subject: [PATCH 32/70] update cargo.toml --- Cargo.lock | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73cb1d54f7b..8095d8116ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1315,36 +1315,13 @@ version = "0.2.0" source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" dependencies = [ "either", - "inkwell_internals 0.8.0 (git+https://github.com/HaithemLamri/inkwell.git?branch=master)", + "inkwell_internals", "libc", "llvm-sys", "once_cell", "parking_lot", ] -[[package]] -name = "inkwell" -version = "0.2.0" -source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" -dependencies = [ - "either", - "inkwell_internals 0.8.0 (git+https://github.com/corbanvilla/inkwell.git?branch=v2.0)", - "libc", - "llvm-sys", - "once_cell", - "parking_lot", -] - -[[package]] -name = "inkwell_internals" -version = "0.8.0" -source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" -dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.41", -] - [[package]] name = "inkwell_internals" version = "0.8.0" @@ -2371,7 +2348,7 @@ dependencies = [ name = "rustc_llvm_coverage" version = "0.1.0" dependencies = [ - "inkwell 0.2.0 (git+https://github.com/corbanvilla/inkwell.git?branch=v2.0)", + "inkwell", "libc", "llvm-sys", "rustc_llvm", @@ -2414,7 +2391,7 @@ dependencies = [ "encoding_rs_io", "generational-arena", "indexmap 2.1.0", - "inkwell 0.2.0 (git+https://github.com/HaithemLamri/inkwell.git?branch=master)", + "inkwell", "insta", "itertools 0.11.0", "lazy_static", From 1a64ecc678dc23af1970e724627d72b88d92de38 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 20 Dec 2023 16:57:06 +0000 Subject: [PATCH 33/70] finalize save_func_record_to_mod --- compiler/rustc_llvm_coverage/src/lib.rs | 26 +++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 10314270d66..5827ed86718 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -15,16 +15,19 @@ pub mod types; use types::*; use inkwell::{ - module::Linkage, + module::Linkage,GlobalVisibility, types::{AnyType, AsTypeRef, StructType}, values::{AsValueRef, FunctionValue, GlobalValue, StructValue}, }; +use inkwell::comdat::*; use libc::c_uint; use std::ffi::CString; use inkwell::module::Module; use inkwell::types::BasicType; +use llvm_sys::comdat::LLVMGetComdat; + /// Calls llvm::createPGOFuncNameVar() with the given function instance's /// mangled function name. The LLVM API returns an llvm::GlobalVariable @@ -138,13 +141,20 @@ pub fn save_func_record_to_mod<'ctx>( // llvm::set_initializer(llglobal, func_record_val); llglobal.set_initializer(&func_record_val); - // llvm::set_global_constant(llglobal, true); - // llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); - // llvm::set_visibility(llglobal, llvm::Visibility::Hidden); - // llvm::set_section(llglobal, &func_record_section_name); - // llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); - // Use https://thedan64.github.io/inkwell/inkwell/values/struct.GlobalValue.html#method.set_comdat - // llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); + llglobal.set_constant( true); + llglobal.set_linkage(Linkage::LinkOnceODR); + llglobal.set_visibility(GlobalVisibility::Hidden); + llglobal.set_section(Some(&func_record_section_name)); + llglobal.set_alignment(VAR_ALIGN_BYTES); + //Use https://thedan64.github.io/inkwell/inkwell/values/struct.GlobalValue.html#method.set_comdat + //create comdat for this value + assert!(llglobal.get_comdat().is_none()); + + let comdat = module.get_or_insert_comdat(llglobal.get_name().to_str().unwrap()); + + assert!(llglobal.get_comdat().is_none()); + + llglobal.set_comdat(comdat); // We will skip this for now... I don't think it's necessary (-Corban) // cx.add_used_global(llglobal); } From a8269193de2fce1c5da1e3eda1dc9a67beeca3ad Mon Sep 17 00:00:00 2001 From: HaithemLamri Date: Thu, 21 Dec 2023 10:09:15 +0000 Subject: [PATCH 34/70] test --- compiler/rustc_llvm_coverage/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 5827ed86718..4198b35c0e6 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -157,4 +157,5 @@ pub fn save_func_record_to_mod<'ctx>( llglobal.set_comdat(comdat); // We will skip this for now... I don't think it's necessary (-Corban) // cx.add_used_global(llglobal); + // } From 356604f68b875b864f4439644c054844a9a1d5dc Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Fri, 5 Jan 2024 07:29:52 +0000 Subject: [PATCH 35/70] add pass and cleanup functions --- Cargo.lock | 4 +- .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 1873 +---------------- .../llvm-wrapper/PassWrapper.cpp.bk | 1763 ++++++++++++++++ compiler/rustc_llvm_coverage/Cargo.toml | 7 - compiler/rustc_llvm_coverage/src/ffi.rs | 4 +- compiler/rustc_llvm_coverage/src/lib.rs | 68 +- src/codegen.rs | 35 +- 7 files changed, 1948 insertions(+), 1806 deletions(-) create mode 100644 compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp.bk diff --git a/Cargo.lock b/Cargo.lock index 8095d8116ae..8f3c20d724c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1312,7 +1312,7 @@ dependencies = [ [[package]] name = "inkwell" version = "0.2.0" -source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" +source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#583a6c20522cf8a743b9daf214f254424bf14eb2" dependencies = [ "either", "inkwell_internals", @@ -1325,7 +1325,7 @@ dependencies = [ [[package]] name = "inkwell_internals" version = "0.8.0" -source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#3cff0db5e9bd0bb3d16e65e1fc049feced71f220" +source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#583a6c20522cf8a743b9daf214f254424bf14eb2" dependencies = [ "proc-macro2 1.0.70", "quote 1.0.33", diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index c7634175414..05152aa0461 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -1,1763 +1,110 @@ -// #include - -// #include -// #include - -// #include "LLVMWrapper.h" - -// #include "llvm/Analysis/AliasAnalysis.h" -// #include "llvm/Analysis/TargetLibraryInfo.h" -// #include "llvm/Analysis/TargetTransformInfo.h" -// #include "llvm/CodeGen/TargetSubtargetInfo.h" -// #include "llvm/InitializePasses.h" -// #include "llvm/IR/AutoUpgrade.h" -// #include "llvm/IR/AssemblyAnnotationWriter.h" -// #include "llvm/IR/IntrinsicInst.h" -// #include "llvm/IR/Verifier.h" -// #include "llvm/Object/ObjectFile.h" -// #include "llvm/Object/IRObjectFile.h" -// #include "llvm/Passes/PassBuilder.h" -// #include "llvm/Passes/PassPlugin.h" -// #include "llvm/Passes/StandardInstrumentations.h" -// #include "llvm/Support/CBindingWrapping.h" -// #include "llvm/Support/FileSystem.h" -// #include "llvm/Support/Host.h" -// #if LLVM_VERSION_LT(14, 0) -// #include "llvm/Support/TargetRegistry.h" -// #else -// #include "llvm/MC/TargetRegistry.h" -// #endif -// #include "llvm/Target/TargetMachine.h" -// #include "llvm/Transforms/IPO/PassManagerBuilder.h" -// #include "llvm/Transforms/IPO/AlwaysInliner.h" -// #include "llvm/Transforms/IPO/FunctionImport.h" -// #include "llvm/Transforms/Utils/AddDiscriminators.h" -// #include "llvm/Transforms/Utils/FunctionImportUtils.h" -// #include "llvm/LTO/LTO.h" -// #include "llvm/Bitcode/BitcodeWriterPass.h" -// #include "llvm-c/Transforms/PassManagerBuilder.h" - -// #include "llvm/Transforms/Instrumentation.h" -// #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" -// #include "llvm/Support/TimeProfiler.h" -// #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" -// #include "llvm/Transforms/Instrumentation/InstrProfiling.h" -// #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" -// #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" -// #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" -// #include "llvm/Transforms/Utils/CanonicalizeAliases.h" -// #include "llvm/Transforms/Utils/NameAnonGlobals.h" -// #include "llvm/Transforms/Utils.h" - -// using namespace llvm; - -// typedef struct LLVMOpaquePass *LLVMPassRef; -// typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; - -// DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) -// DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) - -// extern "C" void LLVMInitializePasses() { -// PassRegistry &Registry = *PassRegistry::getPassRegistry(); -// initializeCore(Registry); -// initializeCodeGen(Registry); -// initializeScalarOpts(Registry); -// initializeVectorization(Registry); -// initializeIPO(Registry); -// initializeAnalysis(Registry); -// initializeTransformUtils(Registry); -// initializeInstCombine(Registry); -// initializeInstrumentation(Registry); -// initializeTarget(Registry); -// } - -// extern "C" void LLVMTimeTraceProfilerInitialize() { -// timeTraceProfilerInitialize( -// /* TimeTraceGranularity */ 0, -// /* ProcName */ "rustc"); -// } - -// extern "C" void LLVMTimeTraceProfilerFinishThread() { -// timeTraceProfilerFinishThread(); -// } - -// extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { -// StringRef FN(FileName); -// std::error_code EC; -// raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); - -// timeTraceProfilerWrite(OS); -// timeTraceProfilerCleanup(); -// } - -// extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { -// #if LLVM_VERSION_LT(15, 0) -// StringRef SR(PassName); -// PassRegistry *PR = PassRegistry::getPassRegistry(); - -// const PassInfo *PI = PR->getPassInfo(SR); -// if (PI) { -// return wrap(PI->createPass()); -// } -// return nullptr; -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) { -// #if LLVM_VERSION_LT(15, 0) -// const bool CompileKernel = false; -// const bool UseAfterScope = true; - -// return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope)); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) { -// #if LLVM_VERSION_LT(15, 0) -// const bool CompileKernel = false; - -// return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover)); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) { -// #if LLVM_VERSION_LT(15, 0) -// const bool CompileKernel = false; - -// return wrap(createMemorySanitizerLegacyPassPass( -// MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel})); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() { -// #if LLVM_VERSION_LT(15, 0) -// return wrap(createThreadSanitizerLegacyPassPass()); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) { -// #if LLVM_VERSION_LT(15, 0) -// const bool CompileKernel = false; - -// return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover)); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { -// #if LLVM_VERSION_LT(15, 0) -// assert(RustPass); -// Pass *Pass = unwrap(RustPass); -// PassManagerBase *PMB = unwrap(PMR); -// PMB->add(Pass); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() { -// #if LLVM_VERSION_LT(15, 0) -// return LLVMPassManagerBuilderCreate(); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) { -// #if LLVM_VERSION_LT(15, 0) -// LLVMPassManagerBuilderDispose(PMB); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager( -// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { -// #if LLVM_VERSION_LT(15, 0) -// LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager( -// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { -// #if LLVM_VERSION_LT(15, 0) -// LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager( -// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) { -// #if LLVM_VERSION_LT(15, 0) -// LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" -// void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( -// LLVMPassManagerBuilderRef PMBR, -// LLVMPassManagerRef PMR -// ) { -// #if LLVM_VERSION_LT(15, 0) -// unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold( -// LLVMPassManagerBuilderRef PMB, unsigned Threshold) { -// #if LLVM_VERSION_LT(15, 0) -// LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" -// void LLVMRustAddLastExtensionPasses( -// LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { -// #if LLVM_VERSION_LT(15, 0) -// auto AddExtensionPasses = [Passes, NumPasses]( -// const PassManagerBuilder &Builder, PassManagerBase &PM) { -// for (size_t I = 0; I < NumPasses; I++) { -// PM.add(unwrap(Passes[I])); -// } -// }; -// // Add the passes to both of the pre-finalization extension points, -// // so they are run for optimized and non-optimized builds. -// unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, -// AddExtensionPasses); -// unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, -// AddExtensionPasses); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// #ifdef LLVM_COMPONENT_X86 -// #define SUBTARGET_X86 SUBTARGET(X86) -// #else -// #define SUBTARGET_X86 -// #endif - -// #ifdef LLVM_COMPONENT_ARM -// #define SUBTARGET_ARM SUBTARGET(ARM) -// #else -// #define SUBTARGET_ARM -// #endif - -// #ifdef LLVM_COMPONENT_AARCH64 -// #define SUBTARGET_AARCH64 SUBTARGET(AArch64) -// #else -// #define SUBTARGET_AARCH64 -// #endif - -// #ifdef LLVM_COMPONENT_AVR -// #define SUBTARGET_AVR SUBTARGET(AVR) -// #else -// #define SUBTARGET_AVR -// #endif - -// #ifdef LLVM_COMPONENT_M68k -// #define SUBTARGET_M68K SUBTARGET(M68k) -// #else -// #define SUBTARGET_M68K -// #endif - -// #ifdef LLVM_COMPONENT_MIPS -// #define SUBTARGET_MIPS SUBTARGET(Mips) -// #else -// #define SUBTARGET_MIPS -// #endif - -// #ifdef LLVM_COMPONENT_POWERPC -// #define SUBTARGET_PPC SUBTARGET(PPC) -// #else -// #define SUBTARGET_PPC -// #endif - -// #ifdef LLVM_COMPONENT_SYSTEMZ -// #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) -// #else -// #define SUBTARGET_SYSTEMZ -// #endif - -// #ifdef LLVM_COMPONENT_MSP430 -// #define SUBTARGET_MSP430 SUBTARGET(MSP430) -// #else -// #define SUBTARGET_MSP430 -// #endif - -// #ifdef LLVM_COMPONENT_RISCV -// #define SUBTARGET_RISCV SUBTARGET(RISCV) -// #else -// #define SUBTARGET_RISCV -// #endif - -// #ifdef LLVM_COMPONENT_SPARC -// #define SUBTARGET_SPARC SUBTARGET(Sparc) -// #else -// #define SUBTARGET_SPARC -// #endif - -// #ifdef LLVM_COMPONENT_HEXAGON -// #define SUBTARGET_HEXAGON SUBTARGET(Hexagon) -// #else -// #define SUBTARGET_HEXAGON -// #endif - -// #define GEN_SUBTARGETS \ -// SUBTARGET_X86 \ -// SUBTARGET_ARM \ -// SUBTARGET_AARCH64 \ -// SUBTARGET_AVR \ -// SUBTARGET_M68K \ -// SUBTARGET_MIPS \ -// SUBTARGET_PPC \ -// SUBTARGET_SYSTEMZ \ -// SUBTARGET_MSP430 \ -// SUBTARGET_SPARC \ -// SUBTARGET_HEXAGON \ -// SUBTARGET_RISCV \ - -// #define SUBTARGET(x) \ -// namespace llvm { \ -// extern const SubtargetFeatureKV x##FeatureKV[]; \ -// extern const SubtargetFeatureKV x##SubTypeKV[]; \ -// } - -// GEN_SUBTARGETS -// #undef SUBTARGET - -// extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, -// const char *Feature) { -// TargetMachine *Target = unwrap(TM); -// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); -// return MCInfo->checkFeatures(std::string("+") + Feature); -// } - -// enum class LLVMRustCodeModel { -// Tiny, -// Small, -// Kernel, -// Medium, -// Large, -// None, -// }; - -// static Optional fromRust(LLVMRustCodeModel Model) { -// switch (Model) { -// case LLVMRustCodeModel::Tiny: -// return CodeModel::Tiny; -// case LLVMRustCodeModel::Small: -// return CodeModel::Small; -// case LLVMRustCodeModel::Kernel: -// return CodeModel::Kernel; -// case LLVMRustCodeModel::Medium: -// return CodeModel::Medium; -// case LLVMRustCodeModel::Large: -// return CodeModel::Large; -// case LLVMRustCodeModel::None: -// return None; -// default: -// report_fatal_error("Bad CodeModel."); -// } -// } - -// enum class LLVMRustCodeGenOptLevel { -// None, -// Less, -// Default, -// Aggressive, -// }; - -// static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { -// switch (Level) { -// case LLVMRustCodeGenOptLevel::None: -// return CodeGenOpt::None; -// case LLVMRustCodeGenOptLevel::Less: -// return CodeGenOpt::Less; -// case LLVMRustCodeGenOptLevel::Default: -// return CodeGenOpt::Default; -// case LLVMRustCodeGenOptLevel::Aggressive: -// return CodeGenOpt::Aggressive; -// default: -// report_fatal_error("Bad CodeGenOptLevel."); -// } -// } - -// enum class LLVMRustPassBuilderOptLevel { -// O0, -// O1, -// O2, -// O3, -// Os, -// Oz, -// }; - -// #if LLVM_VERSION_LT(14,0) -// using OptimizationLevel = PassBuilder::OptimizationLevel; -// #endif - -// static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { -// switch (Level) { -// case LLVMRustPassBuilderOptLevel::O0: -// return OptimizationLevel::O0; -// case LLVMRustPassBuilderOptLevel::O1: -// return OptimizationLevel::O1; -// case LLVMRustPassBuilderOptLevel::O2: -// return OptimizationLevel::O2; -// case LLVMRustPassBuilderOptLevel::O3: -// return OptimizationLevel::O3; -// case LLVMRustPassBuilderOptLevel::Os: -// return OptimizationLevel::Os; -// case LLVMRustPassBuilderOptLevel::Oz: -// return OptimizationLevel::Oz; -// default: -// report_fatal_error("Bad PassBuilderOptLevel."); -// } -// } - -// enum class LLVMRustRelocModel { -// Static, -// PIC, -// DynamicNoPic, -// ROPI, -// RWPI, -// ROPIRWPI, -// }; - -// static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { -// switch (RustReloc) { -// case LLVMRustRelocModel::Static: -// return Reloc::Static; -// case LLVMRustRelocModel::PIC: -// return Reloc::PIC_; -// case LLVMRustRelocModel::DynamicNoPic: -// return Reloc::DynamicNoPIC; -// case LLVMRustRelocModel::ROPI: -// return Reloc::ROPI; -// case LLVMRustRelocModel::RWPI: -// return Reloc::RWPI; -// case LLVMRustRelocModel::ROPIRWPI: -// return Reloc::ROPI_RWPI; -// } -// report_fatal_error("Bad RelocModel."); -// } - -// #ifdef LLVM_RUSTLLVM -// /// getLongestEntryLength - Return the length of the longest entry in the table. -// template -// static size_t getLongestEntryLength(ArrayRef Table) { -// size_t MaxLen = 0; -// for (auto &I : Table) -// MaxLen = std::max(MaxLen, std::strlen(I.Key)); -// return MaxLen; -// } - -// extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { -// const TargetMachine *Target = unwrap(TM); -// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); -// const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); -// const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); -// const ArrayRef CPUTable = MCInfo->getCPUTable(); -// unsigned MaxCPULen = getLongestEntryLength(CPUTable); - -// printf("Available CPUs for this target:\n"); -// if (HostArch == TargetArch) { -// const StringRef HostCPU = sys::getHostCPUName(); -// printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", -// MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); -// } -// for (auto &CPU : CPUTable) -// printf(" %-*s\n", MaxCPULen, CPU.Key); -// printf("\n"); -// } - -// extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { -// const TargetMachine *Target = unwrap(TM); -// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); -// const ArrayRef FeatTable = MCInfo->getFeatureTable(); -// return FeatTable.size(); -// } - -// extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, -// const char** Feature, const char** Desc) { -// const TargetMachine *Target = unwrap(TM); -// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); -// const ArrayRef FeatTable = MCInfo->getFeatureTable(); -// const SubtargetFeatureKV Feat = FeatTable[Index]; -// *Feature = Feat.Key; -// *Desc = Feat.Desc; -// } - -// #else - -// extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { -// printf("Target CPU help is not supported by this LLVM version.\n\n"); -// } - -// extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) { -// return 0; -// } - -// extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {} -// #endif - -// extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { -// StringRef Name = sys::getHostCPUName(); -// *len = Name.size(); -// return Name.data(); -// } - -// extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( -// const char *TripleStr, const char *CPU, const char *Feature, -// const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, -// LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, -// bool FunctionSections, -// bool DataSections, -// bool UniqueSectionNames, -// bool TrapUnreachable, -// bool Singlethread, -// bool AsmComments, -// bool EmitStackSizeSection, -// bool RelaxELFRelocations, -// bool UseInitArray, -// const char *SplitDwarfFile) { - -// auto OptLevel = fromRust(RustOptLevel); -// auto RM = fromRust(RustReloc); -// auto CM = fromRust(RustCM); - -// std::string Error; -// Triple Trip(Triple::normalize(TripleStr)); -// const llvm::Target *TheTarget = -// TargetRegistry::lookupTarget(Trip.getTriple(), Error); -// if (TheTarget == nullptr) { -// LLVMRustSetLastError(Error.c_str()); -// return nullptr; -// } - -// TargetOptions Options; - -// Options.FloatABIType = FloatABI::Default; -// if (UseSoftFloat) { -// Options.FloatABIType = FloatABI::Soft; -// } -// Options.DataSections = DataSections; -// Options.FunctionSections = FunctionSections; -// Options.UniqueSectionNames = UniqueSectionNames; -// Options.MCOptions.AsmVerbose = AsmComments; -// Options.MCOptions.PreserveAsmComments = AsmComments; -// Options.MCOptions.ABIName = ABIStr; -// if (SplitDwarfFile) { -// Options.MCOptions.SplitDwarfFile = SplitDwarfFile; -// } -// Options.RelaxELFRelocations = RelaxELFRelocations; -// Options.UseInitArray = UseInitArray; - -// if (TrapUnreachable) { -// // Tell LLVM to codegen `unreachable` into an explicit trap instruction. -// // This limits the extent of possible undefined behavior in some cases, as -// // it prevents control flow from "falling through" into whatever code -// // happens to be laid out next in memory. -// Options.TrapUnreachable = true; -// } - -// if (Singlethread) { -// Options.ThreadModel = ThreadModel::Single; -// } - -// Options.EmitStackSizeSection = EmitStackSizeSection; - -// TargetMachine *TM = TheTarget->createTargetMachine( -// Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); -// return wrap(TM); -// } - -// extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { -// delete unwrap(TM); -// } - -// extern "C" void LLVMRustConfigurePassManagerBuilder( -// LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, -// bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, -// const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath, -// int SizeLevel) { -// #if LLVM_VERSION_LT(15, 0) -// unwrap(PMBR)->MergeFunctions = MergeFunctions; -// unwrap(PMBR)->SLPVectorize = SLPVectorize; -// unwrap(PMBR)->OptLevel = fromRust(OptLevel); -// unwrap(PMBR)->LoopVectorize = LoopVectorize; -// unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; -// unwrap(PMBR)->SizeLevel = SizeLevel; -// unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0; - -// if (PGOGenPath) { -// assert(!PGOUsePath && !PGOSampleUsePath); -// unwrap(PMBR)->EnablePGOInstrGen = true; -// unwrap(PMBR)->PGOInstrGen = PGOGenPath; -// } else if (PGOUsePath) { -// assert(!PGOSampleUsePath); -// unwrap(PMBR)->PGOInstrUse = PGOUsePath; -// } else if (PGOSampleUsePath) { -// unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath; -// } -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` -// // field of a PassManagerBuilder, we expose our own method of doing so. -// extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, -// LLVMModuleRef M, -// bool DisableSimplifyLibCalls) { -// Triple TargetTriple(unwrap(M)->getTargetTriple()); -// TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); -// if (DisableSimplifyLibCalls) -// TLI->disableAllFunctions(); -// unwrap(PMBR)->LibraryInfo = TLI; -// } - -// // Unfortunately, the LLVM C API doesn't provide a way to create the -// // TargetLibraryInfo pass, so we use this method to do so. -// extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, -// bool DisableSimplifyLibCalls) { -// Triple TargetTriple(unwrap(M)->getTargetTriple()); -// TargetLibraryInfoImpl TLII(TargetTriple); -// if (DisableSimplifyLibCalls) -// TLII.disableAllFunctions(); -// unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); -// } - -// // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over -// // all the functions in a module, so we do that manually here. You'll find -// // similar code in clang's BackendUtil.cpp file. -// extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, -// LLVMModuleRef M) { -// llvm::legacy::FunctionPassManager *P = -// unwrap(PMR); -// P->doInitialization(); - -// // Upgrade all calls to old intrinsics first. -// for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) -// UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove - -// for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; -// ++I) -// if (!I->isDeclaration()) -// P->run(*I); - -// P->doFinalization(); -// } - -// extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { -// // Initializing the command-line options more than once is not allowed. So, -// // check if they've already been initialized. (This could happen if we're -// // being called from rustpkg, for example). If the arguments change, then -// // that's just kinda unfortunate. -// static bool Initialized = false; -// if (Initialized) -// return; -// Initialized = true; -// cl::ParseCommandLineOptions(Argc, Argv); -// } - -// enum class LLVMRustFileType { -// AssemblyFile, -// ObjectFile, -// }; - -// static CodeGenFileType fromRust(LLVMRustFileType Type) { -// switch (Type) { -// case LLVMRustFileType::AssemblyFile: -// return CGFT_AssemblyFile; -// case LLVMRustFileType::ObjectFile: -// return CGFT_ObjectFile; -// default: -// report_fatal_error("Bad FileType."); -// } -// } - -// extern "C" LLVMRustResult -// LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, -// LLVMModuleRef M, const char *Path, const char *DwoPath, -// LLVMRustFileType RustFileType) { -// llvm::legacy::PassManager *PM = unwrap(PMR); -// auto FileType = fromRust(RustFileType); - -// std::string ErrorInfo; -// std::error_code EC; -// raw_fd_ostream OS(Path, EC, sys::fs::OF_None); -// if (EC) -// ErrorInfo = EC.message(); -// if (ErrorInfo != "") { -// LLVMRustSetLastError(ErrorInfo.c_str()); -// return LLVMRustResult::Failure; -// } - -// buffer_ostream BOS(OS); -// if (DwoPath) { -// raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None); -// EC.clear(); -// if (EC) -// ErrorInfo = EC.message(); -// if (ErrorInfo != "") { -// LLVMRustSetLastError(ErrorInfo.c_str()); -// return LLVMRustResult::Failure; -// } -// buffer_ostream DBOS(DOS); -// unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); -// PM->run(*unwrap(M)); -// } else { -// unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); -// PM->run(*unwrap(M)); -// } - -// // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output -// // stream (OS), so the only real safe place to delete this is here? Don't we -// // wish this was written in Rust? -// LLVMDisposePassManager(PMR); -// return LLVMRustResult::Success; -// } - -// extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler -// const char*, // pass name -// const char*); // IR name -// extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler - -// std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { -// if (any_isa(WrappedIr)) -// return any_cast(WrappedIr)->getName().str(); -// if (any_isa(WrappedIr)) -// return any_cast(WrappedIr)->getName().str(); -// if (any_isa(WrappedIr)) -// return any_cast(WrappedIr)->getName().str(); -// if (any_isa(WrappedIr)) -// return any_cast(WrappedIr)->getName(); -// return ""; -// } - - -// void LLVMSelfProfileInitializeCallbacks( -// PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, -// LLVMRustSelfProfileBeforePassCallback BeforePassCallback, -// LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { -// PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( -// StringRef Pass, llvm::Any Ir) { -// std::string PassName = Pass.str(); -// std::string IrName = LLVMRustwrappedIrGetName(Ir); -// BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); -// }); - -// PIC.registerAfterPassCallback( -// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, -// const PreservedAnalyses &Preserved) { -// AfterPassCallback(LlvmSelfProfiler); -// }); - -// PIC.registerAfterPassInvalidatedCallback( -// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { -// AfterPassCallback(LlvmSelfProfiler); -// }); - -// PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( -// StringRef Pass, llvm::Any Ir) { -// std::string PassName = Pass.str(); -// std::string IrName = LLVMRustwrappedIrGetName(Ir); -// BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); -// }); - -// PIC.registerAfterAnalysisCallback( -// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { -// AfterPassCallback(LlvmSelfProfiler); -// }); -// } - -// enum class LLVMRustOptStage { -// PreLinkNoLTO, -// PreLinkThinLTO, -// PreLinkFatLTO, -// ThinLTO, -// FatLTO, -// }; - -// struct LLVMRustSanitizerOptions { -// bool SanitizeAddress; -// bool SanitizeAddressRecover; -// bool SanitizeMemory; -// bool SanitizeMemoryRecover; -// int SanitizeMemoryTrackOrigins; -// bool SanitizeThread; -// bool SanitizeHWAddress; -// bool SanitizeHWAddressRecover; -// }; - -// extern "C" LLVMRustResult -// LLVMRustOptimizeWithNewPassManager( -// LLVMModuleRef ModuleRef, -// LLVMTargetMachineRef TMRef, -// LLVMRustPassBuilderOptLevel OptLevelRust, -// LLVMRustOptStage OptStage, -// bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, -// bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, -// bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, -// LLVMRustSanitizerOptions *SanitizerOptions, -// const char *PGOGenPath, const char *PGOUsePath, -// bool InstrumentCoverage, bool InstrumentGCOV, -// const char *PGOSampleUsePath, bool DebugInfoForProfiling, -// void* LlvmSelfProfiler, -// LLVMRustSelfProfileBeforePassCallback BeforePassCallback, -// LLVMRustSelfProfileAfterPassCallback AfterPassCallback, -// const char *ExtraPasses, size_t ExtraPassesLen, -// const char *LLVMPlugins, size_t LLVMPluginsLen) { -// Module *TheModule = unwrap(ModuleRef); -// TargetMachine *TM = unwrap(TMRef); -// OptimizationLevel OptLevel = fromRust(OptLevelRust); - - -// PipelineTuningOptions PTO; -// PTO.LoopUnrolling = UnrollLoops; -// PTO.LoopInterleaving = UnrollLoops; -// PTO.LoopVectorization = LoopVectorize; -// PTO.SLPVectorization = SLPVectorize; -// PTO.MergeFunctions = MergeFunctions; - -// // FIXME: We may want to expose this as an option. -// bool DebugPassManager = false; - -// PassInstrumentationCallbacks PIC; -// StandardInstrumentations SI(DebugPassManager); -// SI.registerCallbacks(PIC); - -// if (LlvmSelfProfiler){ -// LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); -// } - -// Optional PGOOpt; -// if (PGOGenPath) { -// assert(!PGOUsePath && !PGOSampleUsePath); -// PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr, -// PGOOptions::NoCSAction, DebugInfoForProfiling); -// } else if (PGOUsePath) { -// assert(!PGOSampleUsePath); -// PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse, -// PGOOptions::NoCSAction, DebugInfoForProfiling); -// } else if (PGOSampleUsePath) { -// PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse, -// PGOOptions::NoCSAction, DebugInfoForProfiling); -// } else if (DebugInfoForProfiling) { -// PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction, -// PGOOptions::NoCSAction, DebugInfoForProfiling); -// } - -// #if LLVM_VERSION_GE(13, 0) -// PassBuilder PB(TM, PTO, PGOOpt, &PIC); -// LoopAnalysisManager LAM; -// FunctionAnalysisManager FAM; -// CGSCCAnalysisManager CGAM; -// ModuleAnalysisManager MAM; -// #else -// PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC); -// LoopAnalysisManager LAM(DebugPassManager); -// FunctionAnalysisManager FAM(DebugPassManager); -// CGSCCAnalysisManager CGAM(DebugPassManager); -// ModuleAnalysisManager MAM(DebugPassManager); -// #endif - -// FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); - -// Triple TargetTriple(TheModule->getTargetTriple()); -// std::unique_ptr TLII(new TargetLibraryInfoImpl(TargetTriple)); -// if (DisableSimplifyLibCalls) -// TLII->disableAllFunctions(); -// FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); - -// PB.registerModuleAnalyses(MAM); -// PB.registerCGSCCAnalyses(CGAM); -// PB.registerFunctionAnalyses(FAM); -// PB.registerLoopAnalyses(LAM); -// PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); - -// // We manually collect pipeline callbacks so we can apply them at O0, where the -// // PassBuilder does not create a pipeline. -// std::vector> -// PipelineStartEPCallbacks; -// std::vector> -// OptimizerLastEPCallbacks; - -// if (VerifyIR) { -// PipelineStartEPCallbacks.push_back( -// [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { -// MPM.addPass(VerifierPass()); -// } -// ); -// } - -// if (InstrumentGCOV) { -// PipelineStartEPCallbacks.push_back( -// [](ModulePassManager &MPM, OptimizationLevel Level) { -// MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); -// } -// ); -// } - -// if (InstrumentCoverage) { -// PipelineStartEPCallbacks.push_back( -// [](ModulePassManager &MPM, OptimizationLevel Level) { -// InstrProfOptions Options; -// MPM.addPass(InstrProfiling(Options, false)); -// } -// ); -// } - -// if (SanitizerOptions) { -// if (SanitizerOptions->SanitizeMemory) { -// MemorySanitizerOptions Options( -// SanitizerOptions->SanitizeMemoryTrackOrigins, -// SanitizerOptions->SanitizeMemoryRecover, -// /*CompileKernel=*/false); -// OptimizerLastEPCallbacks.push_back( -// [Options](ModulePassManager &MPM, OptimizationLevel Level) { -// #if LLVM_VERSION_GE(14, 0) -// MPM.addPass(ModuleMemorySanitizerPass(Options)); -// #else -// MPM.addPass(MemorySanitizerPass(Options)); -// #endif -// MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); -// } -// ); -// } - -// if (SanitizerOptions->SanitizeThread) { -// OptimizerLastEPCallbacks.push_back( -// [](ModulePassManager &MPM, OptimizationLevel Level) { -// #if LLVM_VERSION_GE(14, 0) -// MPM.addPass(ModuleThreadSanitizerPass()); -// #else -// MPM.addPass(ThreadSanitizerPass()); -// #endif -// MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); -// } -// ); -// } - -// if (SanitizerOptions->SanitizeAddress) { -// OptimizerLastEPCallbacks.push_back( -// [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { -// #if LLVM_VERSION_LT(15, 0) -// MPM.addPass(RequireAnalysisPass()); -// #endif -// #if LLVM_VERSION_GE(14, 0) -// AddressSanitizerOptions opts = AddressSanitizerOptions{ -// /*CompileKernel=*/false, -// SanitizerOptions->SanitizeAddressRecover, -// /*UseAfterScope=*/true, -// AsanDetectStackUseAfterReturnMode::Runtime, -// }; -// MPM.addPass(ModuleAddressSanitizerPass(opts)); -// #else -// MPM.addPass(ModuleAddressSanitizerPass( -// /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover)); -// MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( -// /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover, -// /*UseAfterScope=*/true))); -// #endif -// } -// ); -// } -// if (SanitizerOptions->SanitizeHWAddress) { -// OptimizerLastEPCallbacks.push_back( -// [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { -// #if LLVM_VERSION_GE(14, 0) -// HWAddressSanitizerOptions opts( -// /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, -// /*DisableOptimization=*/false); -// MPM.addPass(HWAddressSanitizerPass(opts)); -// #else -// MPM.addPass(HWAddressSanitizerPass( -// /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover)); -// #endif -// } -// ); -// } -// } - -// if (LLVMPluginsLen) { -// auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); -// SmallVector Plugins; -// PluginsStr.split(Plugins, ',', -1, false); -// for (auto PluginPath: Plugins) { -// auto Plugin = PassPlugin::Load(PluginPath.str()); -// if (!Plugin) { -// LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); -// continue; -// } -// Plugin->registerPassBuilderCallbacks(PB); -// } -// } - -// #if LLVM_VERSION_GE(13, 0) -// ModulePassManager MPM; -// #else -// ModulePassManager MPM(DebugPassManager); -// #endif -// bool NeedThinLTOBufferPasses = UseThinLTOBuffers; -// if (!NoPrepopulatePasses) { -// // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead. -// // At the same time, the LTO pipelines do support O0 and using them is required. -// bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; -// if (OptLevel == OptimizationLevel::O0 && !IsLTO) { -// for (const auto &C : PipelineStartEPCallbacks) -// PB.registerPipelineStartEPCallback(C); -// for (const auto &C : OptimizerLastEPCallbacks) -// PB.registerOptimizerLastEPCallback(C); - -// // Pass false as we manually schedule ThinLTOBufferPasses below. -// MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); -// } else { -// for (const auto &C : PipelineStartEPCallbacks) -// PB.registerPipelineStartEPCallback(C); -// if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { -// for (const auto &C : OptimizerLastEPCallbacks) -// PB.registerOptimizerLastEPCallback(C); -// } - -// switch (OptStage) { -// case LLVMRustOptStage::PreLinkNoLTO: -// MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); -// break; -// case LLVMRustOptStage::PreLinkThinLTO: -// MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); -// // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback -// // passes may still run afterwards. This means we need to run the buffer passes again. -// // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks -// // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks. -// if (OptimizerLastEPCallbacks.empty()) -// NeedThinLTOBufferPasses = false; -// for (const auto &C : OptimizerLastEPCallbacks) -// C(MPM, OptLevel); -// break; -// case LLVMRustOptStage::PreLinkFatLTO: -// MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); -// NeedThinLTOBufferPasses = false; -// break; -// case LLVMRustOptStage::ThinLTO: -// // FIXME: Does it make sense to pass the ModuleSummaryIndex? -// // It only seems to be needed for C++ specific optimizations. -// MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); -// break; -// case LLVMRustOptStage::FatLTO: -// MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); -// break; -// } -// } -// } else { -// // We're not building any of the default pipelines but we still want to -// // add the verifier, instrumentation, etc passes if they were requested -// for (const auto &C : PipelineStartEPCallbacks) -// C(MPM, OptLevel); -// for (const auto &C : OptimizerLastEPCallbacks) -// C(MPM, OptLevel); -// } - -// if (ExtraPassesLen) { -// if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { -// std::string ErrMsg = toString(std::move(Err)); -// LLVMRustSetLastError(ErrMsg.c_str()); -// return LLVMRustResult::Failure; -// } -// } - -// if (NeedThinLTOBufferPasses) { -// MPM.addPass(CanonicalizeAliasesPass()); -// MPM.addPass(NameAnonGlobalPass()); -// } - -// // Upgrade all calls to old intrinsics first. -// for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) -// UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove - -// MPM.run(*TheModule, MAM); -// return LLVMRustResult::Success; -// } - -// // Callback to demangle function name -// // Parameters: -// // * name to be demangled -// // * name len -// // * output buffer -// // * output buffer len -// // Returns len of demangled string, or 0 if demangle failed. -// typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); - - -// namespace { - -// class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { -// DemangleFn Demangle; -// std::vector Buf; - -// public: -// RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} - -// // Return empty string if demangle failed -// // or if name does not need to be demangled -// StringRef CallDemangle(StringRef name) { -// if (!Demangle) { -// return StringRef(); -// } - -// if (Buf.size() < name.size() * 2) { -// // Semangled name usually shorter than mangled, -// // but allocate twice as much memory just in case -// Buf.resize(name.size() * 2); -// } - -// auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); -// if (!R) { -// // Demangle failed. -// return StringRef(); -// } - -// auto Demangled = StringRef(Buf.data(), R); -// if (Demangled == name) { -// // Do not print anything if demangled name is equal to mangled. -// return StringRef(); -// } - -// return Demangled; -// } - -// void emitFunctionAnnot(const Function *F, -// formatted_raw_ostream &OS) override { -// StringRef Demangled = CallDemangle(F->getName()); -// if (Demangled.empty()) { -// return; -// } - -// OS << "; " << Demangled << "\n"; -// } - -// void emitInstructionAnnot(const Instruction *I, -// formatted_raw_ostream &OS) override { -// const char *Name; -// const Value *Value; -// if (const CallInst *CI = dyn_cast(I)) { -// Name = "call"; -// Value = CI->getCalledOperand(); -// } else if (const InvokeInst* II = dyn_cast(I)) { -// Name = "invoke"; -// Value = II->getCalledOperand(); -// } else { -// // Could demangle more operations, e. g. -// // `store %place, @function`. -// return; -// } - -// if (!Value->hasName()) { -// return; -// } - -// StringRef Demangled = CallDemangle(Value->getName()); -// if (Demangled.empty()) { -// return; -// } - -// OS << "; " << Name << " " << Demangled << "\n"; -// } -// }; - -// } // namespace - -// extern "C" LLVMRustResult -// LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { -// std::string ErrorInfo; -// std::error_code EC; -// raw_fd_ostream OS(Path, EC, sys::fs::OF_None); -// if (EC) -// ErrorInfo = EC.message(); -// if (ErrorInfo != "") { -// LLVMRustSetLastError(ErrorInfo.c_str()); -// return LLVMRustResult::Failure; -// } - -// RustAssemblyAnnotationWriter AAW(Demangle); -// formatted_raw_ostream FOS(OS); -// unwrap(M)->print(FOS, &AAW); - -// return LLVMRustResult::Success; -// } - -// extern "C" void LLVMRustPrintPasses() { -// LLVMInitializePasses(); -// struct MyListener : PassRegistrationListener { -// void passEnumerate(const PassInfo *Info) { -// StringRef PassArg = Info->getPassArgument(); -// StringRef PassName = Info->getPassName(); -// if (!PassArg.empty()) { -// // These unsigned->signed casts could theoretically overflow, but -// // realistically never will (and even if, the result is implementation -// // defined rather plain UB). -// printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), -// (int)PassName.size(), PassName.data()); -// } -// } -// } Listener; - -// PassRegistry *PR = PassRegistry::getPassRegistry(); -// PR->enumerateWith(&Listener); -// } - -// extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, -// bool AddLifetimes) { -// unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); -// } - -// extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, -// size_t Len) { -// llvm::legacy::PassManager passes; - -// auto PreserveFunctions = [=](const GlobalValue &GV) { -// for (size_t I = 0; I < Len; I++) { -// if (GV.getName() == Symbols[I]) { -// return true; -// } -// } -// return false; -// }; - -// passes.add(llvm::createInternalizePass(PreserveFunctions)); - -// passes.run(*unwrap(M)); -// } - -// extern "C" void -// LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, -// LLVMTargetMachineRef TMR) { -// TargetMachine *Target = unwrap(TMR); -// unwrap(Module)->setDataLayout(Target->createDataLayout()); -// } - -// extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { -// unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); -// } - -// extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { -// unwrap(M)->setPIELevel(PIELevel::Level::Large); -// } - -// extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, -// LLVMRustCodeModel Model) { -// auto CM = fromRust(Model); -// if (!CM.hasValue()) -// return; -// unwrap(M)->setCodeModel(*CM); -// } - -// // Here you'll find an implementation of ThinLTO as used by the Rust compiler -// // right now. This ThinLTO support is only enabled on "recent ish" versions of -// // LLVM, and otherwise it's just blanket rejected from other compilers. -// // -// // Most of this implementation is straight copied from LLVM. At the time of -// // this writing it wasn't *quite* suitable to reuse more code from upstream -// // for our purposes, but we should strive to upstream this support once it's -// // ready to go! I figure we may want a bit of testing locally first before -// // sending this upstream to LLVM. I hear though they're quite eager to receive -// // feedback like this! -// // -// // If you're reading this code and wondering "what in the world" or you're -// // working "good lord by LLVM upgrade is *still* failing due to these bindings" -// // then fear not! (ok maybe fear a little). All code here is mostly based -// // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. -// // -// // You'll find that the general layout here roughly corresponds to the `run` -// // method in that file as well as `ProcessThinLTOModule`. Functions are -// // specifically commented below as well, but if you're updating this code -// // or otherwise trying to understand it, the LLVM source will be useful in -// // interpreting the mysteries within. -// // -// // Otherwise I'll apologize in advance, it probably requires a relatively -// // significant investment on your part to "truly understand" what's going on -// // here. Not saying I do myself, but it took me awhile staring at LLVM's source -// // and various online resources about ThinLTO to make heads or tails of all -// // this. - -// // This is a shared data structure which *must* be threadsafe to share -// // read-only amongst threads. This also corresponds basically to the arguments -// // of the `ProcessThinLTOModule` function in the LLVM source. -// struct LLVMRustThinLTOData { -// // The combined index that is the global analysis over all modules we're -// // performing ThinLTO for. This is mostly managed by LLVM. -// ModuleSummaryIndex Index; - -// // All modules we may look at, stored as in-memory serialized versions. This -// // is later used when inlining to ensure we can extract any module to inline -// // from. -// StringMap ModuleMap; - -// // A set that we manage of everything we *don't* want internalized. Note that -// // this includes all transitive references right now as well, but it may not -// // always! -// DenseSet GUIDPreservedSymbols; - -// // Not 100% sure what these are, but they impact what's internalized and -// // what's inlined across modules, I believe. -// StringMap ImportLists; -// StringMap ExportLists; -// StringMap ModuleToDefinedGVSummaries; -// StringMap> ResolvedODR; - -// LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} -// }; - -// // Just an argument to the `LLVMRustCreateThinLTOData` function below. -// struct LLVMRustThinLTOModule { -// const char *identifier; -// const char *data; -// size_t len; -// }; - -// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it -// // does. -// static const GlobalValueSummary * -// getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { -// auto StrongDefForLinker = llvm::find_if( -// GVSummaryList, [](const std::unique_ptr &Summary) { -// auto Linkage = Summary->linkage(); -// return !GlobalValue::isAvailableExternallyLinkage(Linkage) && -// !GlobalValue::isWeakForLinker(Linkage); -// }); -// if (StrongDefForLinker != GVSummaryList.end()) -// return StrongDefForLinker->get(); - -// auto FirstDefForLinker = llvm::find_if( -// GVSummaryList, [](const std::unique_ptr &Summary) { -// auto Linkage = Summary->linkage(); -// return !GlobalValue::isAvailableExternallyLinkage(Linkage); -// }); -// if (FirstDefForLinker == GVSummaryList.end()) -// return nullptr; -// return FirstDefForLinker->get(); -// } - -// // The main entry point for creating the global ThinLTO analysis. The structure -// // here is basically the same as before threads are spawned in the `run` -// // function of `lib/LTO/ThinLTOCodeGenerator.cpp`. -// extern "C" LLVMRustThinLTOData* -// LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, -// int num_modules, -// const char **preserved_symbols, -// int num_symbols) { -// auto Ret = std::make_unique(); - -// // Load each module's summary and merge it into one combined index -// for (int i = 0; i < num_modules; i++) { -// auto module = &modules[i]; -// StringRef buffer(module->data, module->len); -// MemoryBufferRef mem_buffer(buffer, module->identifier); - -// Ret->ModuleMap[module->identifier] = mem_buffer; - -// if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { -// LLVMRustSetLastError(toString(std::move(Err)).c_str()); -// return nullptr; -// } -// } - -// // Collect for each module the list of function it defines (GUID -> Summary) -// Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); - -// // Convert the preserved symbols set from string to GUID, this is then needed -// // for internalization. -// for (int i = 0; i < num_symbols; i++) { -// auto GUID = GlobalValue::getGUID(preserved_symbols[i]); -// Ret->GUIDPreservedSymbols.insert(GUID); -// } - -// // Collect the import/export lists for all modules from the call-graph in the -// // combined index -// // -// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` -// auto deadIsPrevailing = [&](GlobalValue::GUID G) { -// return PrevailingType::Unknown; -// }; -// // We don't have a complete picture in our use of ThinLTO, just our immediate -// // crate, so we need `ImportEnabled = false` to limit internalization. -// // Otherwise, we sometimes lose `static` values -- see #60184. -// computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, -// deadIsPrevailing, /* ImportEnabled = */ false); -// ComputeCrossModuleImport( -// Ret->Index, -// Ret->ModuleToDefinedGVSummaries, -// Ret->ImportLists, -// Ret->ExportLists -// ); - -// // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it -// // impacts the caching. -// // -// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this -// // being lifted from `lib/LTO/LTO.cpp` as well -// DenseMap PrevailingCopy; -// for (auto &I : Ret->Index) { -// if (I.second.SummaryList.size() > 1) -// PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); -// } -// auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { -// const auto &Prevailing = PrevailingCopy.find(GUID); -// if (Prevailing == PrevailingCopy.end()) -// return true; -// return Prevailing->second == S; -// }; -// auto recordNewLinkage = [&](StringRef ModuleIdentifier, -// GlobalValue::GUID GUID, -// GlobalValue::LinkageTypes NewLinkage) { -// Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; -// }; - -// #if LLVM_VERSION_GE(13,0) -// // Uses FromPrevailing visibility scheme which works for many binary -// // formats. We probably could and should use ELF visibility scheme for many of -// // our targets, however. -// lto::Config conf; -// thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, -// Ret->GUIDPreservedSymbols); -// #else -// thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, -// Ret->GUIDPreservedSymbols); -// #endif -// // Here we calculate an `ExportedGUIDs` set for use in the `isExported` -// // callback below. This callback below will dictate the linkage for all -// // summaries in the index, and we basically just only want to ensure that dead -// // symbols are internalized. Otherwise everything that's already external -// // linkage will stay as external, and internal will stay as internal. -// std::set ExportedGUIDs; -// for (auto &List : Ret->Index) { -// for (auto &GVS: List.second.SummaryList) { -// if (GlobalValue::isLocalLinkage(GVS->linkage())) -// continue; -// auto GUID = GVS->getOriginalName(); -// if (GVS->flags().Live) -// ExportedGUIDs.insert(GUID); -// } -// } -// auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { -// const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); -// return (ExportList != Ret->ExportLists.end() && -// ExportList->second.count(VI)) || -// ExportedGUIDs.count(VI.getGUID()); -// }; -// thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); - -// return Ret.release(); -// } - -// extern "C" void -// LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { -// delete Data; -// } - -// // Below are the various passes that happen *per module* when doing ThinLTO. -// // -// // In other words, these are the functions that are all run concurrently -// // with one another, one per module. The passes here correspond to the analysis -// // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the -// // `ProcessThinLTOModule` function. Here they're split up into separate steps -// // so rustc can save off the intermediate bytecode between each step. - -// static bool -// clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { -// // When linking an ELF shared object, dso_local should be dropped. We -// // conservatively do this for -fpic. -// bool ClearDSOLocalOnDeclarations = -// TM.getTargetTriple().isOSBinFormatELF() && -// TM.getRelocationModel() != Reloc::Static && -// Mod.getPIELevel() == PIELevel::Default; -// return ClearDSOLocalOnDeclarations; -// } - -// extern "C" bool -// LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, -// LLVMTargetMachineRef TM) { -// Module &Mod = *unwrap(M); -// TargetMachine &Target = *unwrap(TM); - -// bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); -// bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); - -// if (error) { -// LLVMRustSetLastError("renameModuleForThinLTO failed"); -// return false; -// } -// return true; -// } - -// extern "C" bool -// LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { -// Module &Mod = *unwrap(M); -// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); -// #if LLVM_VERSION_GE(14, 0) -// thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); -// #else -// thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); -// #endif -// return true; -// } - -// extern "C" bool -// LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { -// Module &Mod = *unwrap(M); -// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); -// thinLTOInternalizeModule(Mod, DefinedGlobals); -// return true; -// } - -// extern "C" bool -// LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, -// LLVMTargetMachineRef TM) { -// Module &Mod = *unwrap(M); -// TargetMachine &Target = *unwrap(TM); - -// const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); -// auto Loader = [&](StringRef Identifier) { -// const auto &Memory = Data->ModuleMap.lookup(Identifier); -// auto &Context = Mod.getContext(); -// auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); - -// if (!MOrErr) -// return MOrErr; - -// // The rest of this closure is a workaround for -// // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports -// // we accidentally import wasm custom sections into different modules, -// // duplicating them by in the final output artifact. -// // -// // The issue is worked around here by manually removing the -// // `wasm.custom_sections` named metadata node from any imported module. This -// // we know isn't used by any optimization pass so there's no need for it to -// // be imported. -// // -// // Note that the metadata is currently lazily loaded, so we materialize it -// // here before looking up if there's metadata inside. The `FunctionImporter` -// // will immediately materialize metadata anyway after an import, so this -// // shouldn't be a perf hit. -// if (Error Err = (*MOrErr)->materializeMetadata()) { -// Expected> Ret(std::move(Err)); -// return Ret; -// } - -// auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); -// if (WasmCustomSections) -// WasmCustomSections->eraseFromParent(); - -// return MOrErr; -// }; -// bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); -// FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal); -// Expected Result = Importer.importFunctions(Mod, ImportList); -// if (!Result) { -// LLVMRustSetLastError(toString(Result.takeError()).c_str()); -// return false; -// } -// return true; -// } - -// // This struct and various functions are sort of a hack right now, but the -// // problem is that we've got in-memory LLVM modules after we generate and -// // optimize all codegen-units for one compilation in rustc. To be compatible -// // with the LTO support above we need to serialize the modules plus their -// // ThinLTO summary into memory. -// // -// // This structure is basically an owned version of a serialize module, with -// // a ThinLTO summary attached. -// struct LLVMRustThinLTOBuffer { -// std::string data; -// }; - -// extern "C" LLVMRustThinLTOBuffer* -// LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { -// auto Ret = std::make_unique(); -// { -// raw_string_ostream OS(Ret->data); -// { -// legacy::PassManager PM; -// if (is_thin) { -// PM.add(createWriteThinLTOBitcodePass(OS)); -// } else { -// PM.add(createBitcodeWriterPass(OS)); -// } -// PM.run(*unwrap(M)); -// } -// } -// return Ret.release(); -// } - -// extern "C" void -// LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { -// delete Buffer; -// } - -// extern "C" const void* -// LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { -// return Buffer->data.data(); -// } - -// extern "C" size_t -// LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { -// return Buffer->data.length(); -// } - -// // This is what we used to parse upstream bitcode for actual ThinLTO -// // processing. We'll call this once per module optimized through ThinLTO, and -// // it'll be called concurrently on many threads. -// extern "C" LLVMModuleRef -// LLVMRustParseBitcodeForLTO(LLVMContextRef Context, -// const char *data, -// size_t len, -// const char *identifier) { -// StringRef Data(data, len); -// MemoryBufferRef Buffer(Data, identifier); -// unwrap(Context)->enableDebugTypeODRUniquing(); -// Expected> SrcOrError = -// parseBitcodeFile(Buffer, *unwrap(Context)); -// if (!SrcOrError) { -// LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); -// return nullptr; -// } -// return wrap(std::move(*SrcOrError).release()); -// } - -// // Find the bitcode section in the object file data and return it as a slice. -// // Fail if the bitcode section is present but empty. -// // -// // On success, the return value is the pointer to the start of the slice and -// // `out_len` is filled with the (non-zero) length. On failure, the return value -// // is `nullptr` and `out_len` is set to zero. -// extern "C" const char* -// LLVMRustGetBitcodeSliceFromObjectData(const char *data, -// size_t len, -// size_t *out_len) { -// *out_len = 0; - -// StringRef Data(data, len); -// MemoryBufferRef Buffer(Data, ""); // The id is unused. - -// Expected BitcodeOrError = -// object::IRObjectFile::findBitcodeInMemBuffer(Buffer); -// if (!BitcodeOrError) { -// LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str()); -// return nullptr; -// } - -// *out_len = BitcodeOrError->getBufferSize(); -// return BitcodeOrError->getBufferStart(); -// } - -// // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See -// // the comment in `back/lto.rs` for why this exists. -// extern "C" void -// LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, -// DICompileUnit **A, -// DICompileUnit **B) { -// Module *M = unwrap(Mod); -// DICompileUnit **Cur = A; -// DICompileUnit **Next = B; -// for (DICompileUnit *CU : M->debug_compile_units()) { -// *Cur = CU; -// Cur = Next; -// Next = nullptr; -// if (Cur == nullptr) -// break; -// } -// } - -// // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See -// // the comment in `back/lto.rs` for why this exists. -// extern "C" void -// LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { -// Module *M = unwrap(Mod); - -// // If the original source module didn't have a `DICompileUnit` then try to -// // merge all the existing compile units. If there aren't actually any though -// // then there's not much for us to do so return. -// if (Unit == nullptr) { -// for (DICompileUnit *CU : M->debug_compile_units()) { -// Unit = CU; -// break; -// } -// if (Unit == nullptr) -// return; -// } - -// // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and -// // process it recursively. Note that we used to specifically iterate over -// // instructions to ensure we feed everything into it, but `processModule` -// // started doing this the same way in LLVM 7 (commit d769eb36ab2b8). -// DebugInfoFinder Finder; -// Finder.processModule(*M); - -// // After we've found all our debuginfo, rewrite all subprograms to point to -// // the same `DICompileUnit`. -// for (auto &F : Finder.subprograms()) { -// F->replaceUnit(Unit); -// } - -// // Erase any other references to other `DICompileUnit` instances, the verifier -// // will later ensure that we don't actually have any other stale references to -// // worry about. -// auto *MD = M->getNamedMetadata("llvm.dbg.cu"); -// MD->clearOperands(); -// MD->addOperand(Unit); -// } - -// // Computes the LTO cache key for the provided 'ModId' in the given 'Data', -// // storing the result in 'KeyOut'. -// // Currently, this cache key is a SHA-1 hash of anything that could affect -// // the result of optimizing this module (e.g. module imports, exports, liveness -// // of access globals, etc). -// // The precise details are determined by LLVM in `computeLTOCacheKey`, which is -// // used during the normal linker-plugin incremental thin-LTO process. -// extern "C" void -// LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) { -// SmallString<40> Key; -// llvm::lto::Config conf; -// const auto &ImportList = Data->ImportLists.lookup(ModId); -// const auto &ExportList = Data->ExportLists.lookup(ModId); -// const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); -// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); -// std::set CfiFunctionDefs; -// std::set CfiFunctionDecls; - -// // Based on the 'InProcessThinBackend' constructor in LLVM -// for (auto &Name : Data->Index.cfiFunctionDefs()) -// CfiFunctionDefs.insert( -// GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); -// for (auto &Name : Data->Index.cfiFunctionDecls()) -// CfiFunctionDecls.insert( -// GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); - -// llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, -// ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls -// ); - -// LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); -// } +#include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Instrumentation/InstrProfiling.h" +#include "llvm/Transforms/Instrumentation.h" + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/InitializePasses.h" +#include "llvm/IR/AutoUpgrade.h" +#include "llvm/IR/AssemblyAnnotationWriter.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/IRObjectFile.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Passes/PassPlugin.h" +#include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/Support/CBindingWrapping.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Host.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/Transforms/IPO/FunctionImport.h" +#include "llvm/Transforms/Utils/AddDiscriminators.h" +#include "llvm/Transforms/Utils/FunctionImportUtils.h" +#include "llvm/LTO/LTO.h" +#include "llvm/Bitcode/BitcodeWriterPass.h" +#include "llvm-c/Transforms/PassManagerBuilder.h" + +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" +#include "llvm/Support/TimeProfiler.h" +#include "llvm/Transforms/Instrumentation/GCOVProfiler.h" +#include "llvm/Transforms/Instrumentation/InstrProfiling.h" +#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" +#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" +#include "llvm/Transforms/Utils/CanonicalizeAliases.h" +#include "llvm/Transforms/Utils/NameAnonGlobals.h" +#include "llvm/Transforms/Utils.h" + +using namespace llvm; + +// ModulePass *createInstrumentationPass() { +// // Options - https://github.com/llvm-mirror/llvm/blob/2c4ca6832fa6b306ee6a7010bfb80a3f2596f824/include/llvm/Transforms/Instrumentation.h#L129C1-L148C1 +// InstrProfOptions Options; + +// Options.InstrProfileOutput = "rust.profraw"; +// Options.Atomic = true; + +// // No longer legacy once we updated to newer passes +// // return new InstrProfilingLegacyPass(Options, false); +// return createInstrProfilingLegacyPass(Options, false); +// } + +// extern "C" void LLVMRustAddInstrumentationPass(LLVMPassManagerRef PM) { +// InstrProfOptions Options; + +// Options.InstrProfileOutput = "rust.profraw"; +// Options.Atomic = true; + +// // unwrap(PM)->add(createInstrumentationPass()); +// unwrap(PM)->addPass(InstrProfiling(Options, false)); +// } + +extern "C" void LLVMRustRunInstrumentationPass(LLVMModuleRef M) { + // Options + InstrProfOptions Options; + Options.InstrProfileOutput = "rust.profraw"; + Options.Atomic = true; + + + + // Create the analysis managers. + // These must be declared in this order so that they are destroyed in the + // correct order due to inter-analysis-manager references. + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + + // Create the new pass manager builder. + // Take a look at the PassBuilder constructor parameters for more + // customization, e.g. specifying a TargetMachine or various debugging + // options. + PassBuilder PB; + + // Register all the basic analyses with the managers. + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + // Create the pass manager. + // This one corresponds to a typical -O2 optimization pipeline. + // ModulePassManager MPM = PB.buildPerModuleDefaultPipeline(OptimizationLevel::O0); + ModulePassManager MPM = PB.buildO0DefaultPipeline(OptimizationLevel::O0, /* PreLinkLTO */ false); + + MPM.addPass(InstrProfiling(Options, false)); + + // Optimize the IR! + // MPM.run(MyModule, MAM); + MPM.run(*unwrap(M), MAM); + + +} \ No newline at end of file diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp.bk b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp.bk new file mode 100644 index 00000000000..c7634175414 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp.bk @@ -0,0 +1,1763 @@ +// #include + +// #include +// #include + +// #include "LLVMWrapper.h" + +// #include "llvm/Analysis/AliasAnalysis.h" +// #include "llvm/Analysis/TargetLibraryInfo.h" +// #include "llvm/Analysis/TargetTransformInfo.h" +// #include "llvm/CodeGen/TargetSubtargetInfo.h" +// #include "llvm/InitializePasses.h" +// #include "llvm/IR/AutoUpgrade.h" +// #include "llvm/IR/AssemblyAnnotationWriter.h" +// #include "llvm/IR/IntrinsicInst.h" +// #include "llvm/IR/Verifier.h" +// #include "llvm/Object/ObjectFile.h" +// #include "llvm/Object/IRObjectFile.h" +// #include "llvm/Passes/PassBuilder.h" +// #include "llvm/Passes/PassPlugin.h" +// #include "llvm/Passes/StandardInstrumentations.h" +// #include "llvm/Support/CBindingWrapping.h" +// #include "llvm/Support/FileSystem.h" +// #include "llvm/Support/Host.h" +// #if LLVM_VERSION_LT(14, 0) +// #include "llvm/Support/TargetRegistry.h" +// #else +// #include "llvm/MC/TargetRegistry.h" +// #endif +// #include "llvm/Target/TargetMachine.h" +// #include "llvm/Transforms/IPO/PassManagerBuilder.h" +// #include "llvm/Transforms/IPO/AlwaysInliner.h" +// #include "llvm/Transforms/IPO/FunctionImport.h" +// #include "llvm/Transforms/Utils/AddDiscriminators.h" +// #include "llvm/Transforms/Utils/FunctionImportUtils.h" +// #include "llvm/LTO/LTO.h" +// #include "llvm/Bitcode/BitcodeWriterPass.h" +// #include "llvm-c/Transforms/PassManagerBuilder.h" + +// #include "llvm/Transforms/Instrumentation.h" +// #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" +// #include "llvm/Support/TimeProfiler.h" +// #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" +// #include "llvm/Transforms/Instrumentation/InstrProfiling.h" +// #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" +// #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +// #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" +// #include "llvm/Transforms/Utils/CanonicalizeAliases.h" +// #include "llvm/Transforms/Utils/NameAnonGlobals.h" +// #include "llvm/Transforms/Utils.h" + +// using namespace llvm; + +// typedef struct LLVMOpaquePass *LLVMPassRef; +// typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; + +// DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) +// DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + +// extern "C" void LLVMInitializePasses() { +// PassRegistry &Registry = *PassRegistry::getPassRegistry(); +// initializeCore(Registry); +// initializeCodeGen(Registry); +// initializeScalarOpts(Registry); +// initializeVectorization(Registry); +// initializeIPO(Registry); +// initializeAnalysis(Registry); +// initializeTransformUtils(Registry); +// initializeInstCombine(Registry); +// initializeInstrumentation(Registry); +// initializeTarget(Registry); +// } + +// extern "C" void LLVMTimeTraceProfilerInitialize() { +// timeTraceProfilerInitialize( +// /* TimeTraceGranularity */ 0, +// /* ProcName */ "rustc"); +// } + +// extern "C" void LLVMTimeTraceProfilerFinishThread() { +// timeTraceProfilerFinishThread(); +// } + +// extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { +// StringRef FN(FileName); +// std::error_code EC; +// raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); + +// timeTraceProfilerWrite(OS); +// timeTraceProfilerCleanup(); +// } + +// extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { +// #if LLVM_VERSION_LT(15, 0) +// StringRef SR(PassName); +// PassRegistry *PR = PassRegistry::getPassRegistry(); + +// const PassInfo *PI = PR->getPassInfo(SR); +// if (PI) { +// return wrap(PI->createPass()); +// } +// return nullptr; +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) { +// #if LLVM_VERSION_LT(15, 0) +// const bool CompileKernel = false; +// const bool UseAfterScope = true; + +// return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope)); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) { +// #if LLVM_VERSION_LT(15, 0) +// const bool CompileKernel = false; + +// return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover)); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) { +// #if LLVM_VERSION_LT(15, 0) +// const bool CompileKernel = false; + +// return wrap(createMemorySanitizerLegacyPassPass( +// MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel})); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() { +// #if LLVM_VERSION_LT(15, 0) +// return wrap(createThreadSanitizerLegacyPassPass()); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) { +// #if LLVM_VERSION_LT(15, 0) +// const bool CompileKernel = false; + +// return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover)); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { +// #if LLVM_VERSION_LT(15, 0) +// assert(RustPass); +// Pass *Pass = unwrap(RustPass); +// PassManagerBase *PMB = unwrap(PMR); +// PMB->add(Pass); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() { +// #if LLVM_VERSION_LT(15, 0) +// return LLVMPassManagerBuilderCreate(); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) { +// #if LLVM_VERSION_LT(15, 0) +// LLVMPassManagerBuilderDispose(PMB); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager( +// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { +// #if LLVM_VERSION_LT(15, 0) +// LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager( +// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { +// #if LLVM_VERSION_LT(15, 0) +// LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager( +// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) { +// #if LLVM_VERSION_LT(15, 0) +// LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" +// void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( +// LLVMPassManagerBuilderRef PMBR, +// LLVMPassManagerRef PMR +// ) { +// #if LLVM_VERSION_LT(15, 0) +// unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold( +// LLVMPassManagerBuilderRef PMB, unsigned Threshold) { +// #if LLVM_VERSION_LT(15, 0) +// LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// extern "C" +// void LLVMRustAddLastExtensionPasses( +// LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { +// #if LLVM_VERSION_LT(15, 0) +// auto AddExtensionPasses = [Passes, NumPasses]( +// const PassManagerBuilder &Builder, PassManagerBase &PM) { +// for (size_t I = 0; I < NumPasses; I++) { +// PM.add(unwrap(Passes[I])); +// } +// }; +// // Add the passes to both of the pre-finalization extension points, +// // so they are run for optimized and non-optimized builds. +// unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, +// AddExtensionPasses); +// unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, +// AddExtensionPasses); +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// #ifdef LLVM_COMPONENT_X86 +// #define SUBTARGET_X86 SUBTARGET(X86) +// #else +// #define SUBTARGET_X86 +// #endif + +// #ifdef LLVM_COMPONENT_ARM +// #define SUBTARGET_ARM SUBTARGET(ARM) +// #else +// #define SUBTARGET_ARM +// #endif + +// #ifdef LLVM_COMPONENT_AARCH64 +// #define SUBTARGET_AARCH64 SUBTARGET(AArch64) +// #else +// #define SUBTARGET_AARCH64 +// #endif + +// #ifdef LLVM_COMPONENT_AVR +// #define SUBTARGET_AVR SUBTARGET(AVR) +// #else +// #define SUBTARGET_AVR +// #endif + +// #ifdef LLVM_COMPONENT_M68k +// #define SUBTARGET_M68K SUBTARGET(M68k) +// #else +// #define SUBTARGET_M68K +// #endif + +// #ifdef LLVM_COMPONENT_MIPS +// #define SUBTARGET_MIPS SUBTARGET(Mips) +// #else +// #define SUBTARGET_MIPS +// #endif + +// #ifdef LLVM_COMPONENT_POWERPC +// #define SUBTARGET_PPC SUBTARGET(PPC) +// #else +// #define SUBTARGET_PPC +// #endif + +// #ifdef LLVM_COMPONENT_SYSTEMZ +// #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) +// #else +// #define SUBTARGET_SYSTEMZ +// #endif + +// #ifdef LLVM_COMPONENT_MSP430 +// #define SUBTARGET_MSP430 SUBTARGET(MSP430) +// #else +// #define SUBTARGET_MSP430 +// #endif + +// #ifdef LLVM_COMPONENT_RISCV +// #define SUBTARGET_RISCV SUBTARGET(RISCV) +// #else +// #define SUBTARGET_RISCV +// #endif + +// #ifdef LLVM_COMPONENT_SPARC +// #define SUBTARGET_SPARC SUBTARGET(Sparc) +// #else +// #define SUBTARGET_SPARC +// #endif + +// #ifdef LLVM_COMPONENT_HEXAGON +// #define SUBTARGET_HEXAGON SUBTARGET(Hexagon) +// #else +// #define SUBTARGET_HEXAGON +// #endif + +// #define GEN_SUBTARGETS \ +// SUBTARGET_X86 \ +// SUBTARGET_ARM \ +// SUBTARGET_AARCH64 \ +// SUBTARGET_AVR \ +// SUBTARGET_M68K \ +// SUBTARGET_MIPS \ +// SUBTARGET_PPC \ +// SUBTARGET_SYSTEMZ \ +// SUBTARGET_MSP430 \ +// SUBTARGET_SPARC \ +// SUBTARGET_HEXAGON \ +// SUBTARGET_RISCV \ + +// #define SUBTARGET(x) \ +// namespace llvm { \ +// extern const SubtargetFeatureKV x##FeatureKV[]; \ +// extern const SubtargetFeatureKV x##SubTypeKV[]; \ +// } + +// GEN_SUBTARGETS +// #undef SUBTARGET + +// extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, +// const char *Feature) { +// TargetMachine *Target = unwrap(TM); +// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); +// return MCInfo->checkFeatures(std::string("+") + Feature); +// } + +// enum class LLVMRustCodeModel { +// Tiny, +// Small, +// Kernel, +// Medium, +// Large, +// None, +// }; + +// static Optional fromRust(LLVMRustCodeModel Model) { +// switch (Model) { +// case LLVMRustCodeModel::Tiny: +// return CodeModel::Tiny; +// case LLVMRustCodeModel::Small: +// return CodeModel::Small; +// case LLVMRustCodeModel::Kernel: +// return CodeModel::Kernel; +// case LLVMRustCodeModel::Medium: +// return CodeModel::Medium; +// case LLVMRustCodeModel::Large: +// return CodeModel::Large; +// case LLVMRustCodeModel::None: +// return None; +// default: +// report_fatal_error("Bad CodeModel."); +// } +// } + +// enum class LLVMRustCodeGenOptLevel { +// None, +// Less, +// Default, +// Aggressive, +// }; + +// static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { +// switch (Level) { +// case LLVMRustCodeGenOptLevel::None: +// return CodeGenOpt::None; +// case LLVMRustCodeGenOptLevel::Less: +// return CodeGenOpt::Less; +// case LLVMRustCodeGenOptLevel::Default: +// return CodeGenOpt::Default; +// case LLVMRustCodeGenOptLevel::Aggressive: +// return CodeGenOpt::Aggressive; +// default: +// report_fatal_error("Bad CodeGenOptLevel."); +// } +// } + +// enum class LLVMRustPassBuilderOptLevel { +// O0, +// O1, +// O2, +// O3, +// Os, +// Oz, +// }; + +// #if LLVM_VERSION_LT(14,0) +// using OptimizationLevel = PassBuilder::OptimizationLevel; +// #endif + +// static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { +// switch (Level) { +// case LLVMRustPassBuilderOptLevel::O0: +// return OptimizationLevel::O0; +// case LLVMRustPassBuilderOptLevel::O1: +// return OptimizationLevel::O1; +// case LLVMRustPassBuilderOptLevel::O2: +// return OptimizationLevel::O2; +// case LLVMRustPassBuilderOptLevel::O3: +// return OptimizationLevel::O3; +// case LLVMRustPassBuilderOptLevel::Os: +// return OptimizationLevel::Os; +// case LLVMRustPassBuilderOptLevel::Oz: +// return OptimizationLevel::Oz; +// default: +// report_fatal_error("Bad PassBuilderOptLevel."); +// } +// } + +// enum class LLVMRustRelocModel { +// Static, +// PIC, +// DynamicNoPic, +// ROPI, +// RWPI, +// ROPIRWPI, +// }; + +// static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { +// switch (RustReloc) { +// case LLVMRustRelocModel::Static: +// return Reloc::Static; +// case LLVMRustRelocModel::PIC: +// return Reloc::PIC_; +// case LLVMRustRelocModel::DynamicNoPic: +// return Reloc::DynamicNoPIC; +// case LLVMRustRelocModel::ROPI: +// return Reloc::ROPI; +// case LLVMRustRelocModel::RWPI: +// return Reloc::RWPI; +// case LLVMRustRelocModel::ROPIRWPI: +// return Reloc::ROPI_RWPI; +// } +// report_fatal_error("Bad RelocModel."); +// } + +// #ifdef LLVM_RUSTLLVM +// /// getLongestEntryLength - Return the length of the longest entry in the table. +// template +// static size_t getLongestEntryLength(ArrayRef Table) { +// size_t MaxLen = 0; +// for (auto &I : Table) +// MaxLen = std::max(MaxLen, std::strlen(I.Key)); +// return MaxLen; +// } + +// extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { +// const TargetMachine *Target = unwrap(TM); +// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); +// const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); +// const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); +// const ArrayRef CPUTable = MCInfo->getCPUTable(); +// unsigned MaxCPULen = getLongestEntryLength(CPUTable); + +// printf("Available CPUs for this target:\n"); +// if (HostArch == TargetArch) { +// const StringRef HostCPU = sys::getHostCPUName(); +// printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", +// MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); +// } +// for (auto &CPU : CPUTable) +// printf(" %-*s\n", MaxCPULen, CPU.Key); +// printf("\n"); +// } + +// extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { +// const TargetMachine *Target = unwrap(TM); +// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); +// const ArrayRef FeatTable = MCInfo->getFeatureTable(); +// return FeatTable.size(); +// } + +// extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, +// const char** Feature, const char** Desc) { +// const TargetMachine *Target = unwrap(TM); +// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); +// const ArrayRef FeatTable = MCInfo->getFeatureTable(); +// const SubtargetFeatureKV Feat = FeatTable[Index]; +// *Feature = Feat.Key; +// *Desc = Feat.Desc; +// } + +// #else + +// extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { +// printf("Target CPU help is not supported by this LLVM version.\n\n"); +// } + +// extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) { +// return 0; +// } + +// extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {} +// #endif + +// extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { +// StringRef Name = sys::getHostCPUName(); +// *len = Name.size(); +// return Name.data(); +// } + +// extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( +// const char *TripleStr, const char *CPU, const char *Feature, +// const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, +// LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, +// bool FunctionSections, +// bool DataSections, +// bool UniqueSectionNames, +// bool TrapUnreachable, +// bool Singlethread, +// bool AsmComments, +// bool EmitStackSizeSection, +// bool RelaxELFRelocations, +// bool UseInitArray, +// const char *SplitDwarfFile) { + +// auto OptLevel = fromRust(RustOptLevel); +// auto RM = fromRust(RustReloc); +// auto CM = fromRust(RustCM); + +// std::string Error; +// Triple Trip(Triple::normalize(TripleStr)); +// const llvm::Target *TheTarget = +// TargetRegistry::lookupTarget(Trip.getTriple(), Error); +// if (TheTarget == nullptr) { +// LLVMRustSetLastError(Error.c_str()); +// return nullptr; +// } + +// TargetOptions Options; + +// Options.FloatABIType = FloatABI::Default; +// if (UseSoftFloat) { +// Options.FloatABIType = FloatABI::Soft; +// } +// Options.DataSections = DataSections; +// Options.FunctionSections = FunctionSections; +// Options.UniqueSectionNames = UniqueSectionNames; +// Options.MCOptions.AsmVerbose = AsmComments; +// Options.MCOptions.PreserveAsmComments = AsmComments; +// Options.MCOptions.ABIName = ABIStr; +// if (SplitDwarfFile) { +// Options.MCOptions.SplitDwarfFile = SplitDwarfFile; +// } +// Options.RelaxELFRelocations = RelaxELFRelocations; +// Options.UseInitArray = UseInitArray; + +// if (TrapUnreachable) { +// // Tell LLVM to codegen `unreachable` into an explicit trap instruction. +// // This limits the extent of possible undefined behavior in some cases, as +// // it prevents control flow from "falling through" into whatever code +// // happens to be laid out next in memory. +// Options.TrapUnreachable = true; +// } + +// if (Singlethread) { +// Options.ThreadModel = ThreadModel::Single; +// } + +// Options.EmitStackSizeSection = EmitStackSizeSection; + +// TargetMachine *TM = TheTarget->createTargetMachine( +// Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); +// return wrap(TM); +// } + +// extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { +// delete unwrap(TM); +// } + +// extern "C" void LLVMRustConfigurePassManagerBuilder( +// LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, +// bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, +// const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath, +// int SizeLevel) { +// #if LLVM_VERSION_LT(15, 0) +// unwrap(PMBR)->MergeFunctions = MergeFunctions; +// unwrap(PMBR)->SLPVectorize = SLPVectorize; +// unwrap(PMBR)->OptLevel = fromRust(OptLevel); +// unwrap(PMBR)->LoopVectorize = LoopVectorize; +// unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; +// unwrap(PMBR)->SizeLevel = SizeLevel; +// unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0; + +// if (PGOGenPath) { +// assert(!PGOUsePath && !PGOSampleUsePath); +// unwrap(PMBR)->EnablePGOInstrGen = true; +// unwrap(PMBR)->PGOInstrGen = PGOGenPath; +// } else if (PGOUsePath) { +// assert(!PGOSampleUsePath); +// unwrap(PMBR)->PGOInstrUse = PGOUsePath; +// } else if (PGOSampleUsePath) { +// unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath; +// } +// #else +// report_fatal_error("Legacy PM not supported with LLVM 15"); +// #endif +// } + +// // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` +// // field of a PassManagerBuilder, we expose our own method of doing so. +// extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, +// LLVMModuleRef M, +// bool DisableSimplifyLibCalls) { +// Triple TargetTriple(unwrap(M)->getTargetTriple()); +// TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); +// if (DisableSimplifyLibCalls) +// TLI->disableAllFunctions(); +// unwrap(PMBR)->LibraryInfo = TLI; +// } + +// // Unfortunately, the LLVM C API doesn't provide a way to create the +// // TargetLibraryInfo pass, so we use this method to do so. +// extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, +// bool DisableSimplifyLibCalls) { +// Triple TargetTriple(unwrap(M)->getTargetTriple()); +// TargetLibraryInfoImpl TLII(TargetTriple); +// if (DisableSimplifyLibCalls) +// TLII.disableAllFunctions(); +// unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); +// } + +// // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over +// // all the functions in a module, so we do that manually here. You'll find +// // similar code in clang's BackendUtil.cpp file. +// extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, +// LLVMModuleRef M) { +// llvm::legacy::FunctionPassManager *P = +// unwrap(PMR); +// P->doInitialization(); + +// // Upgrade all calls to old intrinsics first. +// for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) +// UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove + +// for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; +// ++I) +// if (!I->isDeclaration()) +// P->run(*I); + +// P->doFinalization(); +// } + +// extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { +// // Initializing the command-line options more than once is not allowed. So, +// // check if they've already been initialized. (This could happen if we're +// // being called from rustpkg, for example). If the arguments change, then +// // that's just kinda unfortunate. +// static bool Initialized = false; +// if (Initialized) +// return; +// Initialized = true; +// cl::ParseCommandLineOptions(Argc, Argv); +// } + +// enum class LLVMRustFileType { +// AssemblyFile, +// ObjectFile, +// }; + +// static CodeGenFileType fromRust(LLVMRustFileType Type) { +// switch (Type) { +// case LLVMRustFileType::AssemblyFile: +// return CGFT_AssemblyFile; +// case LLVMRustFileType::ObjectFile: +// return CGFT_ObjectFile; +// default: +// report_fatal_error("Bad FileType."); +// } +// } + +// extern "C" LLVMRustResult +// LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, +// LLVMModuleRef M, const char *Path, const char *DwoPath, +// LLVMRustFileType RustFileType) { +// llvm::legacy::PassManager *PM = unwrap(PMR); +// auto FileType = fromRust(RustFileType); + +// std::string ErrorInfo; +// std::error_code EC; +// raw_fd_ostream OS(Path, EC, sys::fs::OF_None); +// if (EC) +// ErrorInfo = EC.message(); +// if (ErrorInfo != "") { +// LLVMRustSetLastError(ErrorInfo.c_str()); +// return LLVMRustResult::Failure; +// } + +// buffer_ostream BOS(OS); +// if (DwoPath) { +// raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None); +// EC.clear(); +// if (EC) +// ErrorInfo = EC.message(); +// if (ErrorInfo != "") { +// LLVMRustSetLastError(ErrorInfo.c_str()); +// return LLVMRustResult::Failure; +// } +// buffer_ostream DBOS(DOS); +// unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); +// PM->run(*unwrap(M)); +// } else { +// unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); +// PM->run(*unwrap(M)); +// } + +// // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output +// // stream (OS), so the only real safe place to delete this is here? Don't we +// // wish this was written in Rust? +// LLVMDisposePassManager(PMR); +// return LLVMRustResult::Success; +// } + +// extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler +// const char*, // pass name +// const char*); // IR name +// extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler + +// std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { +// if (any_isa(WrappedIr)) +// return any_cast(WrappedIr)->getName().str(); +// if (any_isa(WrappedIr)) +// return any_cast(WrappedIr)->getName().str(); +// if (any_isa(WrappedIr)) +// return any_cast(WrappedIr)->getName().str(); +// if (any_isa(WrappedIr)) +// return any_cast(WrappedIr)->getName(); +// return ""; +// } + + +// void LLVMSelfProfileInitializeCallbacks( +// PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, +// LLVMRustSelfProfileBeforePassCallback BeforePassCallback, +// LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { +// PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( +// StringRef Pass, llvm::Any Ir) { +// std::string PassName = Pass.str(); +// std::string IrName = LLVMRustwrappedIrGetName(Ir); +// BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); +// }); + +// PIC.registerAfterPassCallback( +// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, +// const PreservedAnalyses &Preserved) { +// AfterPassCallback(LlvmSelfProfiler); +// }); + +// PIC.registerAfterPassInvalidatedCallback( +// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { +// AfterPassCallback(LlvmSelfProfiler); +// }); + +// PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( +// StringRef Pass, llvm::Any Ir) { +// std::string PassName = Pass.str(); +// std::string IrName = LLVMRustwrappedIrGetName(Ir); +// BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); +// }); + +// PIC.registerAfterAnalysisCallback( +// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { +// AfterPassCallback(LlvmSelfProfiler); +// }); +// } + +// enum class LLVMRustOptStage { +// PreLinkNoLTO, +// PreLinkThinLTO, +// PreLinkFatLTO, +// ThinLTO, +// FatLTO, +// }; + +// struct LLVMRustSanitizerOptions { +// bool SanitizeAddress; +// bool SanitizeAddressRecover; +// bool SanitizeMemory; +// bool SanitizeMemoryRecover; +// int SanitizeMemoryTrackOrigins; +// bool SanitizeThread; +// bool SanitizeHWAddress; +// bool SanitizeHWAddressRecover; +// }; + +// extern "C" LLVMRustResult +// LLVMRustOptimizeWithNewPassManager( +// LLVMModuleRef ModuleRef, +// LLVMTargetMachineRef TMRef, +// LLVMRustPassBuilderOptLevel OptLevelRust, +// LLVMRustOptStage OptStage, +// bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, +// bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, +// bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, +// LLVMRustSanitizerOptions *SanitizerOptions, +// const char *PGOGenPath, const char *PGOUsePath, +// bool InstrumentCoverage, bool InstrumentGCOV, +// const char *PGOSampleUsePath, bool DebugInfoForProfiling, +// void* LlvmSelfProfiler, +// LLVMRustSelfProfileBeforePassCallback BeforePassCallback, +// LLVMRustSelfProfileAfterPassCallback AfterPassCallback, +// const char *ExtraPasses, size_t ExtraPassesLen, +// const char *LLVMPlugins, size_t LLVMPluginsLen) { +// Module *TheModule = unwrap(ModuleRef); +// TargetMachine *TM = unwrap(TMRef); +// OptimizationLevel OptLevel = fromRust(OptLevelRust); + + +// PipelineTuningOptions PTO; +// PTO.LoopUnrolling = UnrollLoops; +// PTO.LoopInterleaving = UnrollLoops; +// PTO.LoopVectorization = LoopVectorize; +// PTO.SLPVectorization = SLPVectorize; +// PTO.MergeFunctions = MergeFunctions; + +// // FIXME: We may want to expose this as an option. +// bool DebugPassManager = false; + +// PassInstrumentationCallbacks PIC; +// StandardInstrumentations SI(DebugPassManager); +// SI.registerCallbacks(PIC); + +// if (LlvmSelfProfiler){ +// LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); +// } + +// Optional PGOOpt; +// if (PGOGenPath) { +// assert(!PGOUsePath && !PGOSampleUsePath); +// PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr, +// PGOOptions::NoCSAction, DebugInfoForProfiling); +// } else if (PGOUsePath) { +// assert(!PGOSampleUsePath); +// PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse, +// PGOOptions::NoCSAction, DebugInfoForProfiling); +// } else if (PGOSampleUsePath) { +// PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse, +// PGOOptions::NoCSAction, DebugInfoForProfiling); +// } else if (DebugInfoForProfiling) { +// PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction, +// PGOOptions::NoCSAction, DebugInfoForProfiling); +// } + +// #if LLVM_VERSION_GE(13, 0) +// PassBuilder PB(TM, PTO, PGOOpt, &PIC); +// LoopAnalysisManager LAM; +// FunctionAnalysisManager FAM; +// CGSCCAnalysisManager CGAM; +// ModuleAnalysisManager MAM; +// #else +// PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC); +// LoopAnalysisManager LAM(DebugPassManager); +// FunctionAnalysisManager FAM(DebugPassManager); +// CGSCCAnalysisManager CGAM(DebugPassManager); +// ModuleAnalysisManager MAM(DebugPassManager); +// #endif + +// FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); + +// Triple TargetTriple(TheModule->getTargetTriple()); +// std::unique_ptr TLII(new TargetLibraryInfoImpl(TargetTriple)); +// if (DisableSimplifyLibCalls) +// TLII->disableAllFunctions(); +// FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); + +// PB.registerModuleAnalyses(MAM); +// PB.registerCGSCCAnalyses(CGAM); +// PB.registerFunctionAnalyses(FAM); +// PB.registerLoopAnalyses(LAM); +// PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + +// // We manually collect pipeline callbacks so we can apply them at O0, where the +// // PassBuilder does not create a pipeline. +// std::vector> +// PipelineStartEPCallbacks; +// std::vector> +// OptimizerLastEPCallbacks; + +// if (VerifyIR) { +// PipelineStartEPCallbacks.push_back( +// [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { +// MPM.addPass(VerifierPass()); +// } +// ); +// } + +// if (InstrumentGCOV) { +// PipelineStartEPCallbacks.push_back( +// [](ModulePassManager &MPM, OptimizationLevel Level) { +// MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); +// } +// ); +// } + +// if (InstrumentCoverage) { +// PipelineStartEPCallbacks.push_back( +// [](ModulePassManager &MPM, OptimizationLevel Level) { +// InstrProfOptions Options; +// MPM.addPass(InstrProfiling(Options, false)); +// } +// ); +// } + +// if (SanitizerOptions) { +// if (SanitizerOptions->SanitizeMemory) { +// MemorySanitizerOptions Options( +// SanitizerOptions->SanitizeMemoryTrackOrigins, +// SanitizerOptions->SanitizeMemoryRecover, +// /*CompileKernel=*/false); +// OptimizerLastEPCallbacks.push_back( +// [Options](ModulePassManager &MPM, OptimizationLevel Level) { +// #if LLVM_VERSION_GE(14, 0) +// MPM.addPass(ModuleMemorySanitizerPass(Options)); +// #else +// MPM.addPass(MemorySanitizerPass(Options)); +// #endif +// MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); +// } +// ); +// } + +// if (SanitizerOptions->SanitizeThread) { +// OptimizerLastEPCallbacks.push_back( +// [](ModulePassManager &MPM, OptimizationLevel Level) { +// #if LLVM_VERSION_GE(14, 0) +// MPM.addPass(ModuleThreadSanitizerPass()); +// #else +// MPM.addPass(ThreadSanitizerPass()); +// #endif +// MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); +// } +// ); +// } + +// if (SanitizerOptions->SanitizeAddress) { +// OptimizerLastEPCallbacks.push_back( +// [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +// #if LLVM_VERSION_LT(15, 0) +// MPM.addPass(RequireAnalysisPass()); +// #endif +// #if LLVM_VERSION_GE(14, 0) +// AddressSanitizerOptions opts = AddressSanitizerOptions{ +// /*CompileKernel=*/false, +// SanitizerOptions->SanitizeAddressRecover, +// /*UseAfterScope=*/true, +// AsanDetectStackUseAfterReturnMode::Runtime, +// }; +// MPM.addPass(ModuleAddressSanitizerPass(opts)); +// #else +// MPM.addPass(ModuleAddressSanitizerPass( +// /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover)); +// MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( +// /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover, +// /*UseAfterScope=*/true))); +// #endif +// } +// ); +// } +// if (SanitizerOptions->SanitizeHWAddress) { +// OptimizerLastEPCallbacks.push_back( +// [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +// #if LLVM_VERSION_GE(14, 0) +// HWAddressSanitizerOptions opts( +// /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, +// /*DisableOptimization=*/false); +// MPM.addPass(HWAddressSanitizerPass(opts)); +// #else +// MPM.addPass(HWAddressSanitizerPass( +// /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover)); +// #endif +// } +// ); +// } +// } + +// if (LLVMPluginsLen) { +// auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); +// SmallVector Plugins; +// PluginsStr.split(Plugins, ',', -1, false); +// for (auto PluginPath: Plugins) { +// auto Plugin = PassPlugin::Load(PluginPath.str()); +// if (!Plugin) { +// LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); +// continue; +// } +// Plugin->registerPassBuilderCallbacks(PB); +// } +// } + +// #if LLVM_VERSION_GE(13, 0) +// ModulePassManager MPM; +// #else +// ModulePassManager MPM(DebugPassManager); +// #endif +// bool NeedThinLTOBufferPasses = UseThinLTOBuffers; +// if (!NoPrepopulatePasses) { +// // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead. +// // At the same time, the LTO pipelines do support O0 and using them is required. +// bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; +// if (OptLevel == OptimizationLevel::O0 && !IsLTO) { +// for (const auto &C : PipelineStartEPCallbacks) +// PB.registerPipelineStartEPCallback(C); +// for (const auto &C : OptimizerLastEPCallbacks) +// PB.registerOptimizerLastEPCallback(C); + +// // Pass false as we manually schedule ThinLTOBufferPasses below. +// MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); +// } else { +// for (const auto &C : PipelineStartEPCallbacks) +// PB.registerPipelineStartEPCallback(C); +// if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { +// for (const auto &C : OptimizerLastEPCallbacks) +// PB.registerOptimizerLastEPCallback(C); +// } + +// switch (OptStage) { +// case LLVMRustOptStage::PreLinkNoLTO: +// MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); +// break; +// case LLVMRustOptStage::PreLinkThinLTO: +// MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); +// // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback +// // passes may still run afterwards. This means we need to run the buffer passes again. +// // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks +// // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks. +// if (OptimizerLastEPCallbacks.empty()) +// NeedThinLTOBufferPasses = false; +// for (const auto &C : OptimizerLastEPCallbacks) +// C(MPM, OptLevel); +// break; +// case LLVMRustOptStage::PreLinkFatLTO: +// MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); +// NeedThinLTOBufferPasses = false; +// break; +// case LLVMRustOptStage::ThinLTO: +// // FIXME: Does it make sense to pass the ModuleSummaryIndex? +// // It only seems to be needed for C++ specific optimizations. +// MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); +// break; +// case LLVMRustOptStage::FatLTO: +// MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); +// break; +// } +// } +// } else { +// // We're not building any of the default pipelines but we still want to +// // add the verifier, instrumentation, etc passes if they were requested +// for (const auto &C : PipelineStartEPCallbacks) +// C(MPM, OptLevel); +// for (const auto &C : OptimizerLastEPCallbacks) +// C(MPM, OptLevel); +// } + +// if (ExtraPassesLen) { +// if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { +// std::string ErrMsg = toString(std::move(Err)); +// LLVMRustSetLastError(ErrMsg.c_str()); +// return LLVMRustResult::Failure; +// } +// } + +// if (NeedThinLTOBufferPasses) { +// MPM.addPass(CanonicalizeAliasesPass()); +// MPM.addPass(NameAnonGlobalPass()); +// } + +// // Upgrade all calls to old intrinsics first. +// for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) +// UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove + +// MPM.run(*TheModule, MAM); +// return LLVMRustResult::Success; +// } + +// // Callback to demangle function name +// // Parameters: +// // * name to be demangled +// // * name len +// // * output buffer +// // * output buffer len +// // Returns len of demangled string, or 0 if demangle failed. +// typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); + + +// namespace { + +// class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { +// DemangleFn Demangle; +// std::vector Buf; + +// public: +// RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} + +// // Return empty string if demangle failed +// // or if name does not need to be demangled +// StringRef CallDemangle(StringRef name) { +// if (!Demangle) { +// return StringRef(); +// } + +// if (Buf.size() < name.size() * 2) { +// // Semangled name usually shorter than mangled, +// // but allocate twice as much memory just in case +// Buf.resize(name.size() * 2); +// } + +// auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); +// if (!R) { +// // Demangle failed. +// return StringRef(); +// } + +// auto Demangled = StringRef(Buf.data(), R); +// if (Demangled == name) { +// // Do not print anything if demangled name is equal to mangled. +// return StringRef(); +// } + +// return Demangled; +// } + +// void emitFunctionAnnot(const Function *F, +// formatted_raw_ostream &OS) override { +// StringRef Demangled = CallDemangle(F->getName()); +// if (Demangled.empty()) { +// return; +// } + +// OS << "; " << Demangled << "\n"; +// } + +// void emitInstructionAnnot(const Instruction *I, +// formatted_raw_ostream &OS) override { +// const char *Name; +// const Value *Value; +// if (const CallInst *CI = dyn_cast(I)) { +// Name = "call"; +// Value = CI->getCalledOperand(); +// } else if (const InvokeInst* II = dyn_cast(I)) { +// Name = "invoke"; +// Value = II->getCalledOperand(); +// } else { +// // Could demangle more operations, e. g. +// // `store %place, @function`. +// return; +// } + +// if (!Value->hasName()) { +// return; +// } + +// StringRef Demangled = CallDemangle(Value->getName()); +// if (Demangled.empty()) { +// return; +// } + +// OS << "; " << Name << " " << Demangled << "\n"; +// } +// }; + +// } // namespace + +// extern "C" LLVMRustResult +// LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { +// std::string ErrorInfo; +// std::error_code EC; +// raw_fd_ostream OS(Path, EC, sys::fs::OF_None); +// if (EC) +// ErrorInfo = EC.message(); +// if (ErrorInfo != "") { +// LLVMRustSetLastError(ErrorInfo.c_str()); +// return LLVMRustResult::Failure; +// } + +// RustAssemblyAnnotationWriter AAW(Demangle); +// formatted_raw_ostream FOS(OS); +// unwrap(M)->print(FOS, &AAW); + +// return LLVMRustResult::Success; +// } + +// extern "C" void LLVMRustPrintPasses() { +// LLVMInitializePasses(); +// struct MyListener : PassRegistrationListener { +// void passEnumerate(const PassInfo *Info) { +// StringRef PassArg = Info->getPassArgument(); +// StringRef PassName = Info->getPassName(); +// if (!PassArg.empty()) { +// // These unsigned->signed casts could theoretically overflow, but +// // realistically never will (and even if, the result is implementation +// // defined rather plain UB). +// printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), +// (int)PassName.size(), PassName.data()); +// } +// } +// } Listener; + +// PassRegistry *PR = PassRegistry::getPassRegistry(); +// PR->enumerateWith(&Listener); +// } + +// extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, +// bool AddLifetimes) { +// unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); +// } + +// extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, +// size_t Len) { +// llvm::legacy::PassManager passes; + +// auto PreserveFunctions = [=](const GlobalValue &GV) { +// for (size_t I = 0; I < Len; I++) { +// if (GV.getName() == Symbols[I]) { +// return true; +// } +// } +// return false; +// }; + +// passes.add(llvm::createInternalizePass(PreserveFunctions)); + +// passes.run(*unwrap(M)); +// } + +// extern "C" void +// LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, +// LLVMTargetMachineRef TMR) { +// TargetMachine *Target = unwrap(TMR); +// unwrap(Module)->setDataLayout(Target->createDataLayout()); +// } + +// extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { +// unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); +// } + +// extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { +// unwrap(M)->setPIELevel(PIELevel::Level::Large); +// } + +// extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, +// LLVMRustCodeModel Model) { +// auto CM = fromRust(Model); +// if (!CM.hasValue()) +// return; +// unwrap(M)->setCodeModel(*CM); +// } + +// // Here you'll find an implementation of ThinLTO as used by the Rust compiler +// // right now. This ThinLTO support is only enabled on "recent ish" versions of +// // LLVM, and otherwise it's just blanket rejected from other compilers. +// // +// // Most of this implementation is straight copied from LLVM. At the time of +// // this writing it wasn't *quite* suitable to reuse more code from upstream +// // for our purposes, but we should strive to upstream this support once it's +// // ready to go! I figure we may want a bit of testing locally first before +// // sending this upstream to LLVM. I hear though they're quite eager to receive +// // feedback like this! +// // +// // If you're reading this code and wondering "what in the world" or you're +// // working "good lord by LLVM upgrade is *still* failing due to these bindings" +// // then fear not! (ok maybe fear a little). All code here is mostly based +// // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. +// // +// // You'll find that the general layout here roughly corresponds to the `run` +// // method in that file as well as `ProcessThinLTOModule`. Functions are +// // specifically commented below as well, but if you're updating this code +// // or otherwise trying to understand it, the LLVM source will be useful in +// // interpreting the mysteries within. +// // +// // Otherwise I'll apologize in advance, it probably requires a relatively +// // significant investment on your part to "truly understand" what's going on +// // here. Not saying I do myself, but it took me awhile staring at LLVM's source +// // and various online resources about ThinLTO to make heads or tails of all +// // this. + +// // This is a shared data structure which *must* be threadsafe to share +// // read-only amongst threads. This also corresponds basically to the arguments +// // of the `ProcessThinLTOModule` function in the LLVM source. +// struct LLVMRustThinLTOData { +// // The combined index that is the global analysis over all modules we're +// // performing ThinLTO for. This is mostly managed by LLVM. +// ModuleSummaryIndex Index; + +// // All modules we may look at, stored as in-memory serialized versions. This +// // is later used when inlining to ensure we can extract any module to inline +// // from. +// StringMap ModuleMap; + +// // A set that we manage of everything we *don't* want internalized. Note that +// // this includes all transitive references right now as well, but it may not +// // always! +// DenseSet GUIDPreservedSymbols; + +// // Not 100% sure what these are, but they impact what's internalized and +// // what's inlined across modules, I believe. +// StringMap ImportLists; +// StringMap ExportLists; +// StringMap ModuleToDefinedGVSummaries; +// StringMap> ResolvedODR; + +// LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} +// }; + +// // Just an argument to the `LLVMRustCreateThinLTOData` function below. +// struct LLVMRustThinLTOModule { +// const char *identifier; +// const char *data; +// size_t len; +// }; + +// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it +// // does. +// static const GlobalValueSummary * +// getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { +// auto StrongDefForLinker = llvm::find_if( +// GVSummaryList, [](const std::unique_ptr &Summary) { +// auto Linkage = Summary->linkage(); +// return !GlobalValue::isAvailableExternallyLinkage(Linkage) && +// !GlobalValue::isWeakForLinker(Linkage); +// }); +// if (StrongDefForLinker != GVSummaryList.end()) +// return StrongDefForLinker->get(); + +// auto FirstDefForLinker = llvm::find_if( +// GVSummaryList, [](const std::unique_ptr &Summary) { +// auto Linkage = Summary->linkage(); +// return !GlobalValue::isAvailableExternallyLinkage(Linkage); +// }); +// if (FirstDefForLinker == GVSummaryList.end()) +// return nullptr; +// return FirstDefForLinker->get(); +// } + +// // The main entry point for creating the global ThinLTO analysis. The structure +// // here is basically the same as before threads are spawned in the `run` +// // function of `lib/LTO/ThinLTOCodeGenerator.cpp`. +// extern "C" LLVMRustThinLTOData* +// LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, +// int num_modules, +// const char **preserved_symbols, +// int num_symbols) { +// auto Ret = std::make_unique(); + +// // Load each module's summary and merge it into one combined index +// for (int i = 0; i < num_modules; i++) { +// auto module = &modules[i]; +// StringRef buffer(module->data, module->len); +// MemoryBufferRef mem_buffer(buffer, module->identifier); + +// Ret->ModuleMap[module->identifier] = mem_buffer; + +// if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { +// LLVMRustSetLastError(toString(std::move(Err)).c_str()); +// return nullptr; +// } +// } + +// // Collect for each module the list of function it defines (GUID -> Summary) +// Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); + +// // Convert the preserved symbols set from string to GUID, this is then needed +// // for internalization. +// for (int i = 0; i < num_symbols; i++) { +// auto GUID = GlobalValue::getGUID(preserved_symbols[i]); +// Ret->GUIDPreservedSymbols.insert(GUID); +// } + +// // Collect the import/export lists for all modules from the call-graph in the +// // combined index +// // +// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` +// auto deadIsPrevailing = [&](GlobalValue::GUID G) { +// return PrevailingType::Unknown; +// }; +// // We don't have a complete picture in our use of ThinLTO, just our immediate +// // crate, so we need `ImportEnabled = false` to limit internalization. +// // Otherwise, we sometimes lose `static` values -- see #60184. +// computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, +// deadIsPrevailing, /* ImportEnabled = */ false); +// ComputeCrossModuleImport( +// Ret->Index, +// Ret->ModuleToDefinedGVSummaries, +// Ret->ImportLists, +// Ret->ExportLists +// ); + +// // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it +// // impacts the caching. +// // +// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this +// // being lifted from `lib/LTO/LTO.cpp` as well +// DenseMap PrevailingCopy; +// for (auto &I : Ret->Index) { +// if (I.second.SummaryList.size() > 1) +// PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); +// } +// auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { +// const auto &Prevailing = PrevailingCopy.find(GUID); +// if (Prevailing == PrevailingCopy.end()) +// return true; +// return Prevailing->second == S; +// }; +// auto recordNewLinkage = [&](StringRef ModuleIdentifier, +// GlobalValue::GUID GUID, +// GlobalValue::LinkageTypes NewLinkage) { +// Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; +// }; + +// #if LLVM_VERSION_GE(13,0) +// // Uses FromPrevailing visibility scheme which works for many binary +// // formats. We probably could and should use ELF visibility scheme for many of +// // our targets, however. +// lto::Config conf; +// thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, +// Ret->GUIDPreservedSymbols); +// #else +// thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, +// Ret->GUIDPreservedSymbols); +// #endif +// // Here we calculate an `ExportedGUIDs` set for use in the `isExported` +// // callback below. This callback below will dictate the linkage for all +// // summaries in the index, and we basically just only want to ensure that dead +// // symbols are internalized. Otherwise everything that's already external +// // linkage will stay as external, and internal will stay as internal. +// std::set ExportedGUIDs; +// for (auto &List : Ret->Index) { +// for (auto &GVS: List.second.SummaryList) { +// if (GlobalValue::isLocalLinkage(GVS->linkage())) +// continue; +// auto GUID = GVS->getOriginalName(); +// if (GVS->flags().Live) +// ExportedGUIDs.insert(GUID); +// } +// } +// auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { +// const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); +// return (ExportList != Ret->ExportLists.end() && +// ExportList->second.count(VI)) || +// ExportedGUIDs.count(VI.getGUID()); +// }; +// thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); + +// return Ret.release(); +// } + +// extern "C" void +// LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { +// delete Data; +// } + +// // Below are the various passes that happen *per module* when doing ThinLTO. +// // +// // In other words, these are the functions that are all run concurrently +// // with one another, one per module. The passes here correspond to the analysis +// // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the +// // `ProcessThinLTOModule` function. Here they're split up into separate steps +// // so rustc can save off the intermediate bytecode between each step. + +// static bool +// clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { +// // When linking an ELF shared object, dso_local should be dropped. We +// // conservatively do this for -fpic. +// bool ClearDSOLocalOnDeclarations = +// TM.getTargetTriple().isOSBinFormatELF() && +// TM.getRelocationModel() != Reloc::Static && +// Mod.getPIELevel() == PIELevel::Default; +// return ClearDSOLocalOnDeclarations; +// } + +// extern "C" bool +// LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, +// LLVMTargetMachineRef TM) { +// Module &Mod = *unwrap(M); +// TargetMachine &Target = *unwrap(TM); + +// bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); +// bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); + +// if (error) { +// LLVMRustSetLastError("renameModuleForThinLTO failed"); +// return false; +// } +// return true; +// } + +// extern "C" bool +// LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { +// Module &Mod = *unwrap(M); +// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); +// #if LLVM_VERSION_GE(14, 0) +// thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); +// #else +// thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); +// #endif +// return true; +// } + +// extern "C" bool +// LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { +// Module &Mod = *unwrap(M); +// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); +// thinLTOInternalizeModule(Mod, DefinedGlobals); +// return true; +// } + +// extern "C" bool +// LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, +// LLVMTargetMachineRef TM) { +// Module &Mod = *unwrap(M); +// TargetMachine &Target = *unwrap(TM); + +// const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); +// auto Loader = [&](StringRef Identifier) { +// const auto &Memory = Data->ModuleMap.lookup(Identifier); +// auto &Context = Mod.getContext(); +// auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); + +// if (!MOrErr) +// return MOrErr; + +// // The rest of this closure is a workaround for +// // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports +// // we accidentally import wasm custom sections into different modules, +// // duplicating them by in the final output artifact. +// // +// // The issue is worked around here by manually removing the +// // `wasm.custom_sections` named metadata node from any imported module. This +// // we know isn't used by any optimization pass so there's no need for it to +// // be imported. +// // +// // Note that the metadata is currently lazily loaded, so we materialize it +// // here before looking up if there's metadata inside. The `FunctionImporter` +// // will immediately materialize metadata anyway after an import, so this +// // shouldn't be a perf hit. +// if (Error Err = (*MOrErr)->materializeMetadata()) { +// Expected> Ret(std::move(Err)); +// return Ret; +// } + +// auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); +// if (WasmCustomSections) +// WasmCustomSections->eraseFromParent(); + +// return MOrErr; +// }; +// bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); +// FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal); +// Expected Result = Importer.importFunctions(Mod, ImportList); +// if (!Result) { +// LLVMRustSetLastError(toString(Result.takeError()).c_str()); +// return false; +// } +// return true; +// } + +// // This struct and various functions are sort of a hack right now, but the +// // problem is that we've got in-memory LLVM modules after we generate and +// // optimize all codegen-units for one compilation in rustc. To be compatible +// // with the LTO support above we need to serialize the modules plus their +// // ThinLTO summary into memory. +// // +// // This structure is basically an owned version of a serialize module, with +// // a ThinLTO summary attached. +// struct LLVMRustThinLTOBuffer { +// std::string data; +// }; + +// extern "C" LLVMRustThinLTOBuffer* +// LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { +// auto Ret = std::make_unique(); +// { +// raw_string_ostream OS(Ret->data); +// { +// legacy::PassManager PM; +// if (is_thin) { +// PM.add(createWriteThinLTOBitcodePass(OS)); +// } else { +// PM.add(createBitcodeWriterPass(OS)); +// } +// PM.run(*unwrap(M)); +// } +// } +// return Ret.release(); +// } + +// extern "C" void +// LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { +// delete Buffer; +// } + +// extern "C" const void* +// LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { +// return Buffer->data.data(); +// } + +// extern "C" size_t +// LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { +// return Buffer->data.length(); +// } + +// // This is what we used to parse upstream bitcode for actual ThinLTO +// // processing. We'll call this once per module optimized through ThinLTO, and +// // it'll be called concurrently on many threads. +// extern "C" LLVMModuleRef +// LLVMRustParseBitcodeForLTO(LLVMContextRef Context, +// const char *data, +// size_t len, +// const char *identifier) { +// StringRef Data(data, len); +// MemoryBufferRef Buffer(Data, identifier); +// unwrap(Context)->enableDebugTypeODRUniquing(); +// Expected> SrcOrError = +// parseBitcodeFile(Buffer, *unwrap(Context)); +// if (!SrcOrError) { +// LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); +// return nullptr; +// } +// return wrap(std::move(*SrcOrError).release()); +// } + +// // Find the bitcode section in the object file data and return it as a slice. +// // Fail if the bitcode section is present but empty. +// // +// // On success, the return value is the pointer to the start of the slice and +// // `out_len` is filled with the (non-zero) length. On failure, the return value +// // is `nullptr` and `out_len` is set to zero. +// extern "C" const char* +// LLVMRustGetBitcodeSliceFromObjectData(const char *data, +// size_t len, +// size_t *out_len) { +// *out_len = 0; + +// StringRef Data(data, len); +// MemoryBufferRef Buffer(Data, ""); // The id is unused. + +// Expected BitcodeOrError = +// object::IRObjectFile::findBitcodeInMemBuffer(Buffer); +// if (!BitcodeOrError) { +// LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str()); +// return nullptr; +// } + +// *out_len = BitcodeOrError->getBufferSize(); +// return BitcodeOrError->getBufferStart(); +// } + +// // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See +// // the comment in `back/lto.rs` for why this exists. +// extern "C" void +// LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, +// DICompileUnit **A, +// DICompileUnit **B) { +// Module *M = unwrap(Mod); +// DICompileUnit **Cur = A; +// DICompileUnit **Next = B; +// for (DICompileUnit *CU : M->debug_compile_units()) { +// *Cur = CU; +// Cur = Next; +// Next = nullptr; +// if (Cur == nullptr) +// break; +// } +// } + +// // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See +// // the comment in `back/lto.rs` for why this exists. +// extern "C" void +// LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { +// Module *M = unwrap(Mod); + +// // If the original source module didn't have a `DICompileUnit` then try to +// // merge all the existing compile units. If there aren't actually any though +// // then there's not much for us to do so return. +// if (Unit == nullptr) { +// for (DICompileUnit *CU : M->debug_compile_units()) { +// Unit = CU; +// break; +// } +// if (Unit == nullptr) +// return; +// } + +// // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and +// // process it recursively. Note that we used to specifically iterate over +// // instructions to ensure we feed everything into it, but `processModule` +// // started doing this the same way in LLVM 7 (commit d769eb36ab2b8). +// DebugInfoFinder Finder; +// Finder.processModule(*M); + +// // After we've found all our debuginfo, rewrite all subprograms to point to +// // the same `DICompileUnit`. +// for (auto &F : Finder.subprograms()) { +// F->replaceUnit(Unit); +// } + +// // Erase any other references to other `DICompileUnit` instances, the verifier +// // will later ensure that we don't actually have any other stale references to +// // worry about. +// auto *MD = M->getNamedMetadata("llvm.dbg.cu"); +// MD->clearOperands(); +// MD->addOperand(Unit); +// } + +// // Computes the LTO cache key for the provided 'ModId' in the given 'Data', +// // storing the result in 'KeyOut'. +// // Currently, this cache key is a SHA-1 hash of anything that could affect +// // the result of optimizing this module (e.g. module imports, exports, liveness +// // of access globals, etc). +// // The precise details are determined by LLVM in `computeLTOCacheKey`, which is +// // used during the normal linker-plugin incremental thin-LTO process. +// extern "C" void +// LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) { +// SmallString<40> Key; +// llvm::lto::Config conf; +// const auto &ImportList = Data->ImportLists.lookup(ModId); +// const auto &ExportList = Data->ExportLists.lookup(ModId); +// const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); +// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); +// std::set CfiFunctionDefs; +// std::set CfiFunctionDecls; + +// // Based on the 'InProcessThinBackend' constructor in LLVM +// for (auto &Name : Data->Index.cfiFunctionDefs()) +// CfiFunctionDefs.insert( +// GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); +// for (auto &Name : Data->Index.cfiFunctionDecls()) +// CfiFunctionDecls.insert( +// GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); + +// llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, +// ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls +// ); + +// LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); +// } diff --git a/compiler/rustc_llvm_coverage/Cargo.toml b/compiler/rustc_llvm_coverage/Cargo.toml index 73e24a4fd4a..0def989f990 100644 --- a/compiler/rustc_llvm_coverage/Cargo.toml +++ b/compiler/rustc_llvm_coverage/Cargo.toml @@ -9,11 +9,4 @@ edition = "2021" libc = "0.2.73" rustc_llvm = { path = "../rustc_llvm" } inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } -<<<<<<< HEAD -# inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } - -======= -#inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } -#inkwell = {path = "../../inkwell"} ->>>>>>> 3086373 (some minor changes) llvm-sys = { package = "llvm-sys", version = "140.0.2" } diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index 800a9ec82e7..d564b49f85c 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -8,7 +8,7 @@ use libc::{c_char, c_uint, c_void, size_t}; use std::slice; use super::types::*; -use llvm_sys::prelude::{LLVMModuleRef, LLVMValueRef}; +use llvm_sys::prelude::{LLVMModuleRef, LLVMPassManagerRef, LLVMValueRef}; /// Appending to a Rust string -- used by RawRustStringOstream. #[no_mangle] @@ -53,4 +53,6 @@ extern "C" { pub fn LLVMRustCoverageMappingVersion() -> u32; + // pub fn LLVMRustAddInstrumentationPass(PM: LLVMPassManagerRef); + pub fn LLVMRustRunInstrumentationPass(M: LLVMModuleRef); } diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 4198b35c0e6..9b9245620f9 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -11,15 +11,17 @@ use std::string::FromUtf8Error; mod ffi; pub mod types; +pub use ffi::LLVMRustRunInstrumentationPass; use types::*; +use inkwell::comdat::*; use inkwell::{ - module::Linkage,GlobalVisibility, + module::Linkage, types::{AnyType, AsTypeRef, StructType}, values::{AsValueRef, FunctionValue, GlobalValue, StructValue}, + GlobalVisibility, }; -use inkwell::comdat::*; use libc::c_uint; use std::ffi::CString; @@ -28,6 +30,11 @@ use inkwell::module::Module; use inkwell::types::BasicType; use llvm_sys::comdat::LLVMGetComdat; +/* == TODO - Refactor these helpers out */ +pub fn build_string(sr: &RustString) -> Result { + String::from_utf8(sr.bytes.borrow().clone()) +} +/* == END TODO */ /// Calls llvm::createPGOFuncNameVar() with the given function instance's /// mangled function name. The LLVM API returns an llvm::GlobalVariable @@ -84,30 +91,31 @@ pub fn mapping_version() -> u32 { unsafe { ffi::LLVMRustCoverageMappingVersion() } } -pub fn save_cov_data_to_mod<'ctx>(func: &FunctionValue<'ctx>, cov_data_val: &GlobalValue<'ctx>) { - //global_value - let covmap_var_nam = unsafe { - ffi::LLVMRustCoverageWriteMappingVarNameToString(s); +pub fn save_cov_data_to_mod<'ctx>(module: &Module<'ctx>, cov_data_val: StructValue<'ctx>) { + let covmap_var_name = { + let mut s = RustString::new(); + unsafe { + ffi::LLVMRustCoverageWriteMappingVarNameToString(&mut s); + } + build_string(&mut s).expect("Rust Coverage Mapping var name failed UTF-8 conversion") + }; + + let covmap_section_name = { + let mut s = RustString::new(); + unsafe { + ffi::LLVMRustCoverageWriteMapSectionNameToString(module.as_mut_ptr(), &mut s); + } + build_string(&mut s).expect("Rust Coverage Mapping section name failed UTF-8 conversion") }; - //.expect("Rust Coverage Mapping var name failed UTF-8 conversion"); - //debug!("covmap var name: {:?}", covmap_var_name); - //build_string not found - let covmap_section_name = unsafe { - ffi::LLVMRustCoverageWriteMapSectionNameToString(func.llmod(), s); - } ; - - // //.expect("Rust Coverage section name failed UTF-8 conversion"); - // //debug!("covmap section name: {:?}", covmap_section_name); - // // add_global not found in inkwell global value and func.llmod doesn't exist in func - // func.ty - let llglobal = Module::add_global(&func.llmod(), func.val_ty(cov_data_val.as_value_ref()), &covmap_var_name,func.get_name().to_str().unwrap()); - //GlobalValue::set_initializer(llglobal, cov_data_val); - // //set_global_cst not found in inkwell global value - // GlobalValue::set_constant(llglobal, true); - // GlobalValue::set_linkage(llglobal, GlobalValue::Linkage::PrivateLinkage); - // GlobalValue::set_section(llglobal, Some(&covmap_section_name)); - // GlobalValue::set_alignment(llglobal, VAR_ALIGN_BYTES); - //func.add_used_global(llglobal); + + let llglobal = module.add_global(cov_data_val.get_type(), None, covmap_var_name.as_str()); + llglobal.set_initializer(&cov_data_val); + llglobal.set_constant(true); + llglobal.set_linkage(Linkage::Private); + llglobal.set_section(Some(&covmap_section_name)); + llglobal.set_alignment(VAR_ALIGN_BYTES); + // We will skip this for now... I don't think it's necessary (-Corban) + // cx.add_used_global(llglobal); } pub fn save_func_record_to_mod<'ctx>( @@ -141,21 +149,17 @@ pub fn save_func_record_to_mod<'ctx>( // llvm::set_initializer(llglobal, func_record_val); llglobal.set_initializer(&func_record_val); - llglobal.set_constant( true); + llglobal.set_constant(true); llglobal.set_linkage(Linkage::LinkOnceODR); llglobal.set_visibility(GlobalVisibility::Hidden); llglobal.set_section(Some(&func_record_section_name)); llglobal.set_alignment(VAR_ALIGN_BYTES); - //Use https://thedan64.github.io/inkwell/inkwell/values/struct.GlobalValue.html#method.set_comdat - //create comdat for this value + // TODO - verify this in the IR assert!(llglobal.get_comdat().is_none()); - let comdat = module.get_or_insert_comdat(llglobal.get_name().to_str().unwrap()); - assert!(llglobal.get_comdat().is_none()); - llglobal.set_comdat(comdat); + // We will skip this for now... I don't think it's necessary (-Corban) // cx.add_used_global(llglobal); - // } diff --git a/src/codegen.rs b/src/codegen.rs index a07a36d4a2d..20fe472f709 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -33,7 +33,7 @@ use inkwell::{ }; use inkwell::{ module::Module, - passes::PassBuilderOptions, + passes::{PassBuilderOptions, PassManager, PassManagerBuilder}, targets::{CodeModel, FileType, InitializationConfig, RelocMode}, }; use plc_ast::ast::{CompilationUnit, LinkageType}; @@ -101,6 +101,20 @@ impl<'ink> CodeGen<'ink> { let module = context.create_module(module_location); module.set_source_file_name(module_location); let debug = debug::DebugBuilderEnum::new(context, &module, root, optimization_level, debug_level); + + // let mut pm = PassManager::create(()); + + // let pass_manager_builder = PassManagerBuilder::create(); + // pass_manager_builder.populate_module_pass_manager(&pm); + + // unsafe { + // rustc_llvm_coverage::LLVMRustAddInstrumentationPass(pm.as_mut_ptr()); + // } + // let did_init = pm.initialize(); + // println!("Did init: {}", did_init); + // let did_finalize = pm.finalize(); + // println!("Did finalize: {:?}", did_finalize); + CodeGen { module, debug, module_location: module_location.to_string() } } @@ -429,6 +443,8 @@ impl<'ink> GeneratedModule<'ink> { if let Some(parent) = output.parent() { std::fs::create_dir_all(parent)?; } + // Log passes + println!("Optimization level: {:?}", optimization_level.opt_params()); ////Run the passes machine .and_then(|it| { @@ -522,6 +538,23 @@ impl<'ink> GeneratedModule<'ink> { log::debug!("Output location: {}", output.to_string_lossy()); log::debug!("{}", self.persist_to_string()); + println!("Writing to IR"); + + // let pm = PassManager::create(()); + // unsafe { + // rustc_llvm_coverage::LLVMRustAddInstrumentationPass(pm.as_mut_ptr()); + // } + // let did_run = pm.run_on(&self.module); + // println!("Did run: {}", did_run); + // let did_init = pm.initialize(); + // println!("Did init: {}", did_init); + // let did_finalize = pm.finalize(); + // println!("Did finalize: {:?}", did_finalize); + + unsafe { + rustc_llvm_coverage::LLVMRustRunInstrumentationPass(self.module.as_mut_ptr()); + } + self.module .print_to_file(&output) .map_err(|err| { From 96f2012805ed94faf635fcabd06f52f17ab065d6 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Thu, 11 Jan 2024 11:34:26 +0000 Subject: [PATCH 36/70] working cov map header --- compiler/rustc_llvm_coverage/src/interface.rs | 69 +++++++++++++++++++ compiler/rustc_llvm_coverage/src/lib.rs | 6 +- 2 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 compiler/rustc_llvm_coverage/src/interface.rs diff --git a/compiler/rustc_llvm_coverage/src/interface.rs b/compiler/rustc_llvm_coverage/src/interface.rs new file mode 100644 index 00000000000..e3cba233a4f --- /dev/null +++ b/compiler/rustc_llvm_coverage/src/interface.rs @@ -0,0 +1,69 @@ +use crate::write_filenames_section_to_buffer; + +use super::*; +use inkwell::context::Context; +use inkwell::module::Module; +use std::ffi::CString; + +pub struct FunctionRecord { + name: String, + structural_hash: u64, + filenames_hash: u64, + virtual_file_mapping: Vec, + expressions: Vec, + mapping_regions: Vec, +} + +pub fn write_coverage_mapping_header<'ctx>(module: &Module<'ctx>, filenames: Vec) { + // Get context + let context = module.get_context(); + + // Convert filenames to CStrings + let filenames = filenames.into_iter().map(|f| CString::new(f).unwrap()).collect::>(); + let mut encoded_filename_buffer = RustString::new(); + write_filenames_section_to_buffer(&filenames, &mut encoded_filename_buffer); + + // Get values + let cov_mapping_version = mapping_version(); + let encoded_filenames_len = encoded_filename_buffer.len(); + + // Create mapping header types + let i32_type = context.i32_type(); + let i32_zero = i32_type.const_int(0, false); + let i32_cov_mapping_version = i32_type.const_int(cov_mapping_version.into(), false); + let i32_filenames_len = i32_type.const_int(encoded_filenames_len as u64, false); + + // See LLVM Code Coverage Specification for details on this data structure + let cov_mapping_header = context.const_struct( + &[ + // Value 1 : Always zero + i32_zero.into(), + // Value 2 : Len(encoded_filenames) + i32_filenames_len.into(), + // Value 3 : Always zero + i32_zero.into(), + // Value 4 : Mapping version + i32_cov_mapping_version.into(), + ], + false, + ); + + // Create filename value types + let i8_type = context.i8_type(); + let i8_filename_array = i8_type.const_array( + &encoded_filename_buffer + .bytes + .borrow() + .iter() + .map(|byte| i8_type.const_int(*byte as u64, false)) + .collect::>(), + ); + + // Create structure + let coverage_struct = context.const_struct(&[cov_mapping_header.into(), i8_filename_array.into()], false); + + // Write to module + save_cov_data_to_mod(module, coverage_struct); +} + +pub fn write_function_record() {} diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 9b9245620f9..4276082c505 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -10,6 +10,7 @@ const VAR_ALIGN_BYTES: u32 = 8; use std::string::FromUtf8Error; mod ffi; +pub mod interface; pub mod types; pub use ffi::LLVMRustRunInstrumentationPass; @@ -133,7 +134,6 @@ pub fn save_func_record_to_mod<'ctx>( // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. let func_record_var_name = format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); - println!("function record var name: {:?}", func_record_var_name); let func_record_section_name = { let mut s = RustString::new(); @@ -142,18 +142,16 @@ pub fn save_func_record_to_mod<'ctx>( } build_string(&mut s).expect("Rust Coverage function record section name failed UTF-8 conversion") }; - println!("function record section name: {:?}", func_record_section_name); let llglobal = module.add_global(func_record_val.get_type(), None, func_record_var_name.as_str()); - // llvm::set_initializer(llglobal, func_record_val); llglobal.set_initializer(&func_record_val); - llglobal.set_constant(true); llglobal.set_linkage(Linkage::LinkOnceODR); llglobal.set_visibility(GlobalVisibility::Hidden); llglobal.set_section(Some(&func_record_section_name)); llglobal.set_alignment(VAR_ALIGN_BYTES); + // TODO - verify this in the IR assert!(llglobal.get_comdat().is_none()); let comdat = module.get_or_insert_comdat(llglobal.get_name().to_str().unwrap()); From 33619f067a68b483d7a3b6f0361b4fc7074eace5 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Thu, 11 Jan 2024 11:35:03 +0000 Subject: [PATCH 37/70] steps towards rusty integration --- src/codegen.rs | 200 +++++++++++++----------- src/codegen/generators/pou_generator.rs | 2 + src/codegen/instrument.rs | 19 +++ 3 files changed, 126 insertions(+), 95 deletions(-) create mode 100644 src/codegen/instrument.rs diff --git a/src/codegen.rs b/src/codegen.rs index 20fe472f709..ebb56ba64f9 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -15,6 +15,7 @@ use self::{ pou_generator::{self, PouGenerator}, variable_generator::VariableGenerator, }, + instrument::InstrumentBuilder, llvm_index::LlvmTypedIndex, }; use crate::{ @@ -46,6 +47,7 @@ use rustc_llvm_coverage::{ mod debug; pub(crate) mod generators; +mod instrument; mod llvm_index; mod llvm_typesystem; #[cfg(test)] @@ -77,7 +79,8 @@ pub struct CodeGen<'ink> { pub module: Module<'ink>, /// the debugging module creates debug information at appropriate locations pub debug: DebugBuilderEnum<'ink>, - + /// the instrumentation builder (to be later hoisted out of the codegen struct) + // pub instrument: Option< pub module_location: String, } @@ -233,100 +236,107 @@ impl<'ink> CodeGen<'ink> { self.debug.finalize(); log::debug!("{}", self.module.to_string()); - println!("Done generating POUs"); - println!("Cov mapping version: {}", rustc_llvm_coverage::mapping_version()); - println!("Hash string: {:#?}", rustc_llvm_coverage::hash_str("asdf")); - let byte_string: Vec = vec![0x01, 0x02, 0x03, 0x04]; - println!("Hash bytes: {:#?}", rustc_llvm_coverage::hash_bytes(byte_string)); - - let rust_string = rustc_llvm_coverage::types::RustString { bytes: RefCell::new(Vec::new()) }; - let filenames = vec![CString::new("test.c").unwrap()]; - rustc_llvm_coverage::write_filenames_section_to_buffer(&filenames, &rust_string); - // print buffer - // println!("Filenames: {:#?}", rust_string.bytes.borrow()); - // print buffer as hex string - println!( - "Filenames: {:#?}", - rust_string - .bytes - .borrow() - .iter() - .map(|it| format!("{:02x}", it)) - .collect::>() - .join("") - ); - - // println!("{:#?}", llvm_index); - // let prg_func: inkwell::values::FunctionValue<'_> = - // llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); - let prg_func = self.module.get_function("prg").expect("Unable to get prg"); - println!("prg: {:#?}", prg_func); - let global_func_var = rustc_llvm_coverage::create_pgo_func_name_var(&prg_func); - println!("global_func_var: {:#?}", global_func_var); - - let global_vars: Vec<_> = self.module.get_globals().collect(); - println!("global_vars: {:#?}", global_vars); - //create write_mapping_to_buffer params - //virtual_file_mapping - let virtual_file_mapping: Vec = vec![0x0]; - - let counter1 = Counter::counter_value_reference(CounterId::new(1)); - let counter2 = Counter::expression(ExpressionId::new(2)); - let counter3 = Counter::counter_value_reference(CounterId::new(3)); - - // Creating a vector of CounterExpression instances - let expressions: Vec = vec![ - CounterExpression::new(counter1, ExprKind::Add, counter2), - CounterExpression::new(counter2, ExprKind::Subtract, counter3), - // Add more CounterExpression instances as needed - ]; - //mapping_regions - let mapping_regions: Vec = vec![ - CounterMappingRegion::code_region(counter1, 0, 0, 0, 3, 10), - CounterMappingRegion::code_region(counter3, 0, 4, 0, 9, 10), - ]; - //buffer - let buffer = rustc_llvm_coverage::types::RustString { bytes: RefCell::new(Vec::new()) }; - write_mapping_to_buffer(virtual_file_mapping, expressions, mapping_regions, &buffer); - //print the buffer - //print the buffer - println!( - "Buffer: {:#?}", - buffer.bytes.borrow().iter().map(|it| format!("{:02x}", it)).collect::>().join("") - ); - - // cov mappping - // let struct_type = context.opaque_struct_type("my_struct"); - let i32_type = context.i32_type(); - let i32_zero = i32_type.const_int(0, false); - // TODO - generate this dynamically - let i32_42 = i32_type.const_int(42, false); - // TODO - replace this w/ cov mapping version - let i32_5 = i32_type.const_int(5, false); - - let cov_data_header = - context.const_struct(&[i32_zero.into(), i32_42.into(), i32_zero.into(), i32_5.into()], false); - - // TODO - generate this dynamically from cov mapping data - let i8_type = context.i8_type(); - let i8_zero = i8_type.const_int(0, false); - let cov_mapping_data = i8_type.const_array(&[i8_zero, i8_zero, i8_zero]); - - let cov_data_val = context.const_struct(&[cov_data_header.into(), cov_mapping_data.into()], false); - - rustc_llvm_coverage::save_cov_data_to_mod(&self.module, cov_data_val); - - // func record - - let i64_type = context.i64_type(); - let i64_zero = i64_type.const_int(0, false); - let i64_one = i64_type.const_int(1, false); - let func_mapping_data = i8_type.const_array(&[i8_zero, i8_zero, i8_zero]); - let func_data_val = context.const_struct( - &[i64_one.into(), i32_zero.into(), i64_zero.into(), i64_zero.into(), func_mapping_data.into()], - false, - ); - rustc_llvm_coverage::save_func_record_to_mod(&self.module, 0x1234, func_data_val, true); + let filenames = vec![self.module_location]; + rustc_llvm_coverage::interface::write_coverage_mapping_header(&self.module, filenames); + + // println!("AST annotations: {:#?}", annotations); + // println!("AST unit: {:#?}", unit); + // println!("Module location: {}", self.module_location); + + // println!("Done generating POUs"); + // println!("Cov mapping version: {}", rustc_llvm_coverage::mapping_version()); + // println!("Hash string: {:#?}", rustc_llvm_coverage::hash_str("asdf")); + // let byte_string: Vec = vec![0x01, 0x02, 0x03, 0x04]; + // println!("Hash bytes: {:#?}", rustc_llvm_coverage::hash_bytes(byte_string)); + + // let rust_string = rustc_llvm_coverage::types::RustString { bytes: RefCell::new(Vec::new()) }; + // let filenames = vec![CString::new("test.c").unwrap()]; + // rustc_llvm_coverage::write_filenames_section_to_buffer(&filenames, &rust_string); + // // print buffer + // // println!("Filenames: {:#?}", rust_string.bytes.borrow()); + // // print buffer as hex string + // println!( + // "Filenames: {:#?}", + // rust_string + // .bytes + // .borrow() + // .iter() + // .map(|it| format!("{:02x}", it)) + // .collect::>() + // .join("") + // ); + + // // println!("{:#?}", llvm_index); + // // let prg_func: inkwell::values::FunctionValue<'_> = + // // llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); + // let prg_func = self.module.get_function("prg").expect("Unable to get prg"); + // println!("prg: {:#?}", prg_func); + // let global_func_var = rustc_llvm_coverage::create_pgo_func_name_var(&prg_func); + // println!("global_func_var: {:#?}", global_func_var); + + // let global_vars: Vec<_> = self.module.get_globals().collect(); + // println!("global_vars: {:#?}", global_vars); + // //create write_mapping_to_buffer params + // //virtual_file_mapping + // let virtual_file_mapping: Vec = vec![0x0]; + + // let counter1 = Counter::counter_value_reference(CounterId::new(1)); + // let counter2 = Counter::expression(ExpressionId::new(2)); + // let counter3 = Counter::counter_value_reference(CounterId::new(3)); + + // // Creating a vector of CounterExpression instances + // let expressions: Vec = vec![ + // CounterExpression::new(counter1, ExprKind::Add, counter2), + // CounterExpression::new(counter2, ExprKind::Subtract, counter3), + // // Add more CounterExpression instances as needed + // ]; + // //mapping_regions + // let mapping_regions: Vec = vec![ + // CounterMappingRegion::code_region(counter1, 0, 0, 0, 3, 10), + // CounterMappingRegion::code_region(counter3, 0, 4, 0, 9, 10), + // ]; + // //buffer + // let buffer = rustc_llvm_coverage::types::RustString { bytes: RefCell::new(Vec::new()) }; + // write_mapping_to_buffer(virtual_file_mapping, expressions, mapping_regions, &buffer); + // //print the buffer + // //print the buffer + // println!( + // "Buffer: {:#?}", + // buffer.bytes.borrow().iter().map(|it| format!("{:02x}", it)).collect::>().join("") + // ); + + // // cov mappping + // // let struct_type = context.opaque_struct_type("my_struct"); + // let i32_type = context.i32_type(); + // let i32_zero = i32_type.const_int(0, false); + // // TODO - generate this dynamically + // let i32_42 = i32_type.const_int(42, false); + // // TODO - replace this w/ cov mapping version + // let i32_5 = i32_type.const_int(5, false); + + // let cov_data_header = + // context.const_struct(&[i32_zero.into(), i32_42.into(), i32_zero.into(), i32_5.into()], false); + + // // TODO - generate this dynamically from cov mapping data + // let i8_type = context.i8_type(); + // let i8_zero = i8_type.const_int(0, false); + // let cov_mapping_data = i8_type.const_array(&[i8_zero, i8_zero, i8_zero]); + + // let cov_data_val = context.const_struct(&[cov_data_header.into(), cov_mapping_data.into()], false); + + // rustc_llvm_coverage::save_cov_data_to_mod(&self.module, cov_data_val); + + // // func record + + // let i64_type = context.i64_type(); + // let i64_zero = i64_type.const_int(0, false); + // let i64_one = i64_type.const_int(1, false); + // let func_mapping_data = i8_type.const_array(&[i8_zero, i8_zero, i8_zero]); + // let func_data_val = context.const_struct( + // &[i64_one.into(), i32_zero.into(), i64_zero.into(), i64_zero.into(), func_mapping_data.into()], + // false, + // ); + // rustc_llvm_coverage::save_func_record_to_mod(&self.module, 0x1234, func_data_val, true); #[cfg(feature = "verify")] { diff --git a/src/codegen/generators/pou_generator.rs b/src/codegen/generators/pou_generator.rs index e977f45acec..9258daeec5f 100644 --- a/src/codegen/generators/pou_generator.rs +++ b/src/codegen/generators/pou_generator.rs @@ -294,6 +294,8 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { ) })?; + // TODO - this is where we should possibly add counter increments + let (line, column) = implementation .statements .first() diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs new file mode 100644 index 00000000000..bd3b3b95a8d --- /dev/null +++ b/src/codegen/instrument.rs @@ -0,0 +1,19 @@ +use inkwell::context::Context; +use rustc_llvm_coverage; +use std::ffi::CString; + +pub struct InstrumentBuilder<'ink> { + context: &'ink Context, + files: Vec<&'ink str>, +} + +impl<'ink> InstrumentBuilder<'ink> { + pub fn new(context: &'ink Context, file_name: &'ink str) -> Self { + Self { context, files: vec![file_name] } + } + + pub fn write_header(&self) { + // Filenames + let cstring_filenames = self.files.iter().map(|f| CString::new(*f).unwrap()).collect::>(); + } +} From 3d8488c7bd6452d6c0ef374292c37f09f8e4741b Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Fri, 12 Jan 2024 11:50:20 +0000 Subject: [PATCH 38/70] inkwell-level interface working --- compiler/rustc_llvm_coverage/src/interface.rs | 134 ++++++++++++++++-- compiler/rustc_llvm_coverage/src/lib.rs | 3 +- src/codegen.rs | 42 +++--- 3 files changed, 143 insertions(+), 36 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/interface.rs b/compiler/rustc_llvm_coverage/src/interface.rs index e3cba233a4f..50d451ba5cd 100644 --- a/compiler/rustc_llvm_coverage/src/interface.rs +++ b/compiler/rustc_llvm_coverage/src/interface.rs @@ -5,32 +5,36 @@ use inkwell::context::Context; use inkwell::module::Module; use std::ffi::CString; -pub struct FunctionRecord { - name: String, - structural_hash: u64, - filenames_hash: u64, - virtual_file_mapping: Vec, - expressions: Vec, - mapping_regions: Vec, +/// This represents a coverage mapping header that has been written to a module. +/// It is returned for debugging purposes and use with write_function_record. +pub struct WrittenCoverageMappingHeader { + pub mapping_version: u32, + pub filenames: Vec, + pub filenames_hash: u64, } -pub fn write_coverage_mapping_header<'ctx>(module: &Module<'ctx>, filenames: Vec) { +/// filenames: In Coverage Mapping Version > 6, first filename must be the compilation directory +pub fn write_coverage_mapping_header<'ctx>( + module: &Module<'ctx>, + filenames: Vec, +) -> WrittenCoverageMappingHeader { // Get context let context = module.get_context(); // Convert filenames to CStrings - let filenames = filenames.into_iter().map(|f| CString::new(f).unwrap()).collect::>(); + let filenames_cstr = filenames.clone().into_iter().map(|f| CString::new(f).unwrap()).collect::>(); let mut encoded_filename_buffer = RustString::new(); - write_filenames_section_to_buffer(&filenames, &mut encoded_filename_buffer); + write_filenames_section_to_buffer(&filenames_cstr, &mut encoded_filename_buffer); + let filenames_hash = hash_bytes(encoded_filename_buffer.bytes.borrow().to_vec()); // Get values - let cov_mapping_version = mapping_version(); + let mapping_version = mapping_version(); let encoded_filenames_len = encoded_filename_buffer.len(); // Create mapping header types let i32_type = context.i32_type(); let i32_zero = i32_type.const_int(0, false); - let i32_cov_mapping_version = i32_type.const_int(cov_mapping_version.into(), false); + let i32_cov_mapping_version = i32_type.const_int(mapping_version.into(), false); let i32_filenames_len = i32_type.const_int(encoded_filenames_len as u64, false); // See LLVM Code Coverage Specification for details on this data structure @@ -64,6 +68,110 @@ pub fn write_coverage_mapping_header<'ctx>(module: &Module<'ctx>, filenames: Vec // Write to module save_cov_data_to_mod(module, coverage_struct); + + // Return header + WrittenCoverageMappingHeader { mapping_version, filenames, filenames_hash } } -pub fn write_function_record() {} +pub struct FunctionRecord { + name: String, + name_md5_hash: u64, + structural_hash: u64, + virtual_file_mapping: Vec, + expressions: Vec, + mapping_regions: Vec, + mapping_buffer: RustString, + + // A.k.a. hash of all filenames in module + translation_unit_hash: u64, + is_used: bool, +} + +impl FunctionRecord { + pub fn new( + name: String, + structural_hash: u64, + // TODO - better names for these + function_filenames: Vec, + expressions: Vec, + mapping_regions: Vec, + is_used: bool, + + written_mapping_header: &WrittenCoverageMappingHeader, + ) -> Self { + let name_md5_hash = hash_str(&name); + + // Get indexes of function filenames in module file list + let mut virtual_file_mapping = Vec::new(); + for filename in function_filenames { + let filename_idx = written_mapping_header + .filenames + .iter() + .position(|f| f == &filename) + .expect("Unable to find function filename in module files"); + virtual_file_mapping.push(filename_idx.try_into().unwrap()); + } + + // Write mapping to buffer + let mut mapping_buffer = RustString::new(); + write_mapping_to_buffer( + virtual_file_mapping.clone(), + expressions.clone(), + mapping_regions.clone(), + &mut mapping_buffer, + ); + + FunctionRecord { + name, + name_md5_hash, + structural_hash, + virtual_file_mapping, + expressions, + is_used, + mapping_regions, + mapping_buffer, + translation_unit_hash: written_mapping_header.filenames_hash, + } + } + + pub fn write_to_module<'ctx>(&self, module: &Module<'ctx>) { + // Get context + let context = module.get_context(); + + // Create types + let i64_type = context.i64_type(); + let i32_type = context.i32_type(); + let i8_type = context.i8_type(); + + // Create values + let i64_name_md5_hash = i64_type.const_int(self.name_md5_hash, false); + let i32_mapping_len = i32_type.const_int(self.mapping_buffer.len() as u64, false); + let i64_structural_hash = i64_type.const_int(self.structural_hash, false); + let i64_translation_unit_hash = i64_type.const_int(self.translation_unit_hash, false); + + // Build mapping array + let i8_mapping_array = i8_type.const_array( + &self + .mapping_buffer + .bytes + .borrow() + .iter() + .map(|byte| i8_type.const_int(*byte as u64, false)) + .collect::>(), + ); + + // Create structure + let function_record_struct = context.const_struct( + &[ + i64_name_md5_hash.into(), + i32_mapping_len.into(), + i64_structural_hash.into(), + i64_translation_unit_hash.into(), + i8_mapping_array.into(), + ], + false, + ); + + save_func_record_to_mod(&module, self.name_md5_hash, function_record_struct, self.is_used); + } +} diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 4276082c505..b53eb9ae855 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -64,7 +64,7 @@ pub fn write_mapping_to_buffer( virtual_file_mapping: Vec, expressions: Vec, mapping_regions: Vec, - buffer: &RustString, + buffer: &mut RustString, ) { unsafe { ffi::LLVMRustCoverageWriteMappingToBuffer( @@ -133,6 +133,7 @@ pub fn save_func_record_to_mod<'ctx>( // in a Crate can be replaced by full description provided by a different Crate. The two kinds // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. + // TODO - investigate removing this (-Corban) let func_record_var_name = format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); let func_record_section_name = { diff --git a/src/codegen.rs b/src/codegen.rs index ebb56ba64f9..81646e0ed5b 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -236,8 +236,23 @@ impl<'ink> CodeGen<'ink> { self.debug.finalize(); log::debug!("{}", self.module.to_string()); - let filenames = vec![self.module_location]; - rustc_llvm_coverage::interface::write_coverage_mapping_header(&self.module, filenames); + let filenames = vec!["cwd".to_string(), self.module_location]; + let cov_header = + rustc_llvm_coverage::interface::write_coverage_mapping_header(&self.module, filenames); + + let prg_func = self.module.get_function("prg").expect("Unable to get prg"); + + let func_record = rustc_llvm_coverage::interface::FunctionRecord::new( + "main".to_string(), + 1, + vec!["cwd".to_string()], + Vec::new(), + Vec::new(), + true, + &cov_header, + ); + + func_record.write_to_module(&self.module); // println!("AST annotations: {:#?}", annotations); // println!("AST unit: {:#?}", unit); @@ -249,26 +264,9 @@ impl<'ink> CodeGen<'ink> { // let byte_string: Vec = vec![0x01, 0x02, 0x03, 0x04]; // println!("Hash bytes: {:#?}", rustc_llvm_coverage::hash_bytes(byte_string)); - // let rust_string = rustc_llvm_coverage::types::RustString { bytes: RefCell::new(Vec::new()) }; - // let filenames = vec![CString::new("test.c").unwrap()]; - // rustc_llvm_coverage::write_filenames_section_to_buffer(&filenames, &rust_string); - // // print buffer - // // println!("Filenames: {:#?}", rust_string.bytes.borrow()); - // // print buffer as hex string - // println!( - // "Filenames: {:#?}", - // rust_string - // .bytes - // .borrow() - // .iter() - // .map(|it| format!("{:02x}", it)) - // .collect::>() - // .join("") - // ); - - // // println!("{:#?}", llvm_index); - // // let prg_func: inkwell::values::FunctionValue<'_> = - // // llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); + // println!("{:#?}", llvm_index); + // let prg_func: inkwell::values::FunctionValue<'_> = + // llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); // let prg_func = self.module.get_function("prg").expect("Unable to get prg"); // println!("prg: {:#?}", prg_func); // let global_func_var = rustc_llvm_coverage::create_pgo_func_name_var(&prg_func); From 0b00d2059cd5b087270ddd82acad3819a84a0c08 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Fri, 12 Jan 2024 12:02:15 +0000 Subject: [PATCH 39/70] cleanup test features --- src/codegen.rs | 88 +++----------------------------------------------- 1 file changed, 5 insertions(+), 83 deletions(-) diff --git a/src/codegen.rs b/src/codegen.rs index 81646e0ed5b..63495b00cfa 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -242,100 +242,22 @@ impl<'ink> CodeGen<'ink> { let prg_func = self.module.get_function("prg").expect("Unable to get prg"); + let counter1 = Counter::counter_value_reference(CounterId::new(1)); + let mapping_regions: Vec = + vec![CounterMappingRegion::code_region(counter1, 0, 0, 0, 3, 10)]; + let func_record = rustc_llvm_coverage::interface::FunctionRecord::new( "main".to_string(), 1, vec!["cwd".to_string()], Vec::new(), - Vec::new(), + mapping_regions, true, &cov_header, ); func_record.write_to_module(&self.module); - // println!("AST annotations: {:#?}", annotations); - // println!("AST unit: {:#?}", unit); - // println!("Module location: {}", self.module_location); - - // println!("Done generating POUs"); - // println!("Cov mapping version: {}", rustc_llvm_coverage::mapping_version()); - // println!("Hash string: {:#?}", rustc_llvm_coverage::hash_str("asdf")); - // let byte_string: Vec = vec![0x01, 0x02, 0x03, 0x04]; - // println!("Hash bytes: {:#?}", rustc_llvm_coverage::hash_bytes(byte_string)); - - // println!("{:#?}", llvm_index); - // let prg_func: inkwell::values::FunctionValue<'_> = - // llvm_index.find_associated_implementation("prg").expect("Unable to get prg"); - // let prg_func = self.module.get_function("prg").expect("Unable to get prg"); - // println!("prg: {:#?}", prg_func); - // let global_func_var = rustc_llvm_coverage::create_pgo_func_name_var(&prg_func); - // println!("global_func_var: {:#?}", global_func_var); - - // let global_vars: Vec<_> = self.module.get_globals().collect(); - // println!("global_vars: {:#?}", global_vars); - // //create write_mapping_to_buffer params - // //virtual_file_mapping - // let virtual_file_mapping: Vec = vec![0x0]; - - // let counter1 = Counter::counter_value_reference(CounterId::new(1)); - // let counter2 = Counter::expression(ExpressionId::new(2)); - // let counter3 = Counter::counter_value_reference(CounterId::new(3)); - - // // Creating a vector of CounterExpression instances - // let expressions: Vec = vec![ - // CounterExpression::new(counter1, ExprKind::Add, counter2), - // CounterExpression::new(counter2, ExprKind::Subtract, counter3), - // // Add more CounterExpression instances as needed - // ]; - // //mapping_regions - // let mapping_regions: Vec = vec![ - // CounterMappingRegion::code_region(counter1, 0, 0, 0, 3, 10), - // CounterMappingRegion::code_region(counter3, 0, 4, 0, 9, 10), - // ]; - // //buffer - // let buffer = rustc_llvm_coverage::types::RustString { bytes: RefCell::new(Vec::new()) }; - // write_mapping_to_buffer(virtual_file_mapping, expressions, mapping_regions, &buffer); - // //print the buffer - // //print the buffer - // println!( - // "Buffer: {:#?}", - // buffer.bytes.borrow().iter().map(|it| format!("{:02x}", it)).collect::>().join("") - // ); - - // // cov mappping - // // let struct_type = context.opaque_struct_type("my_struct"); - // let i32_type = context.i32_type(); - // let i32_zero = i32_type.const_int(0, false); - // // TODO - generate this dynamically - // let i32_42 = i32_type.const_int(42, false); - // // TODO - replace this w/ cov mapping version - // let i32_5 = i32_type.const_int(5, false); - - // let cov_data_header = - // context.const_struct(&[i32_zero.into(), i32_42.into(), i32_zero.into(), i32_5.into()], false); - - // // TODO - generate this dynamically from cov mapping data - // let i8_type = context.i8_type(); - // let i8_zero = i8_type.const_int(0, false); - // let cov_mapping_data = i8_type.const_array(&[i8_zero, i8_zero, i8_zero]); - - // let cov_data_val = context.const_struct(&[cov_data_header.into(), cov_mapping_data.into()], false); - - // rustc_llvm_coverage::save_cov_data_to_mod(&self.module, cov_data_val); - - // // func record - - // let i64_type = context.i64_type(); - // let i64_zero = i64_type.const_int(0, false); - // let i64_one = i64_type.const_int(1, false); - // let func_mapping_data = i8_type.const_array(&[i8_zero, i8_zero, i8_zero]); - // let func_data_val = context.const_struct( - // &[i64_one.into(), i32_zero.into(), i64_zero.into(), i64_zero.into(), func_mapping_data.into()], - // false, - // ); - // rustc_llvm_coverage::save_func_record_to_mod(&self.module, 0x1234, func_data_val, true); - #[cfg(feature = "verify")] { self.module From 25694e1986f584eb762fd204243bad145300cdd5 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Sat, 13 Jan 2024 07:24:24 +0000 Subject: [PATCH 40/70] pass runs successfully --- .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 96 +++++++++---------- compiler/rustc_llvm_coverage/src/ffi.rs | 4 +- compiler/rustc_llvm_coverage/src/interface.rs | 39 +++++++- compiler/rustc_llvm_coverage/src/lib.rs | 2 +- src/codegen.rs | 14 ++- src/codegen/generators/pou_generator.rs | 31 +++++- 6 files changed, 124 insertions(+), 62 deletions(-) diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 05152aa0461..09ca633c644 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -44,67 +44,65 @@ using namespace llvm; -// ModulePass *createInstrumentationPass() { -// // Options - https://github.com/llvm-mirror/llvm/blob/2c4ca6832fa6b306ee6a7010bfb80a3f2596f824/include/llvm/Transforms/Instrumentation.h#L129C1-L148C1 -// InstrProfOptions Options; - -// Options.InstrProfileOutput = "rust.profraw"; -// Options.Atomic = true; +ModulePass *createInstrumentationPass() { + // Options - https://github.com/llvm-mirror/llvm/blob/2c4ca6832fa6b306ee6a7010bfb80a3f2596f824/include/llvm/Transforms/Instrumentation.h#L129C1-L148C1 + InstrProfOptions Options; -// // No longer legacy once we updated to newer passes -// // return new InstrProfilingLegacyPass(Options, false); -// return createInstrProfilingLegacyPass(Options, false); -// } + // Options.InstrProfileOutput = "rust.profraw"; + // Options.Atomic = true; -// extern "C" void LLVMRustAddInstrumentationPass(LLVMPassManagerRef PM) { -// InstrProfOptions Options; + // No longer legacy once we updated to newer passes + // return new InstrProfilingLegacyPass(Options, false); + return createInstrProfilingLegacyPass(Options, false); +} -// Options.InstrProfileOutput = "rust.profraw"; -// Options.Atomic = true; +extern "C" void LLVMRustAddInstrumentationPass(LLVMPassManagerRef PM) { -// // unwrap(PM)->add(createInstrumentationPass()); -// unwrap(PM)->addPass(InstrProfiling(Options, false)); -// } + unwrap(PM)->add(createInstrumentationPass()); + // unwrap(PM)->addPass(InstrProfiling(Options, false)); +} -extern "C" void LLVMRustRunInstrumentationPass(LLVMModuleRef M) { - // Options - InstrProfOptions Options; - Options.InstrProfileOutput = "rust.profraw"; - Options.Atomic = true; +// // https://github.com/rust-lang/rust/blob/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp#L923-L930 +// extern "C" void LLVMRustRunInstrumentationPass(LLVMModuleRef M) { +// // Options +// InstrProfOptions Options; +// // Options.InstrProfileOutput = "rust.profraw"; +// // Options.Atomic = true; - // Create the analysis managers. - // These must be declared in this order so that they are destroyed in the - // correct order due to inter-analysis-manager references. - LoopAnalysisManager LAM; - FunctionAnalysisManager FAM; - CGSCCAnalysisManager CGAM; - ModuleAnalysisManager MAM; +// // Create the analysis managers. +// // These must be declared in this order so that they are destroyed in the +// // correct order due to inter-analysis-manager references. +// LoopAnalysisManager LAM; +// FunctionAnalysisManager FAM; +// CGSCCAnalysisManager CGAM; +// ModuleAnalysisManager MAM; - // Create the new pass manager builder. - // Take a look at the PassBuilder constructor parameters for more - // customization, e.g. specifying a TargetMachine or various debugging - // options. - PassBuilder PB; +// // Create the new pass manager builder. +// // Take a look at the PassBuilder constructor parameters for more +// // customization, e.g. specifying a TargetMachine or various debugging +// // options. +// PassBuilder PB; - // Register all the basic analyses with the managers. - PB.registerModuleAnalyses(MAM); - PB.registerCGSCCAnalyses(CGAM); - PB.registerFunctionAnalyses(FAM); - PB.registerLoopAnalyses(LAM); - PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); +// // Register all the basic analyses with the managers. +// PB.registerModuleAnalyses(MAM); +// PB.registerCGSCCAnalyses(CGAM); +// PB.registerFunctionAnalyses(FAM); +// PB.registerLoopAnalyses(LAM); +// PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); - // Create the pass manager. - // This one corresponds to a typical -O2 optimization pipeline. - // ModulePassManager MPM = PB.buildPerModuleDefaultPipeline(OptimizationLevel::O0); - ModulePassManager MPM = PB.buildO0DefaultPipeline(OptimizationLevel::O0, /* PreLinkLTO */ false); +// // Create the pass manager. +// // This one corresponds to a typical -O2 optimization pipeline. +// // ModulePassManager MPM = PB.buildPerModuleDefaultPipeline(OptimizationLevel::O0); +// ModulePassManager MPM = PB.buildO0DefaultPipeline(OptimizationLevel::O0, /* PreLinkLTO */ false); - MPM.addPass(InstrProfiling(Options, false)); +// // TODO - this needed to be updated in LLVM >=18 +// MPM.addPass(InstrProfiling(Options, false)); - // Optimize the IR! - // MPM.run(MyModule, MAM); - MPM.run(*unwrap(M), MAM); +// // Optimize the IR! +// // MPM.run(MyModule, MAM); +// MPM.run(*unwrap(M), MAM); -} \ No newline at end of file +// } \ No newline at end of file diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index d564b49f85c..1796891347d 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -53,6 +53,6 @@ extern "C" { pub fn LLVMRustCoverageMappingVersion() -> u32; - // pub fn LLVMRustAddInstrumentationPass(PM: LLVMPassManagerRef); - pub fn LLVMRustRunInstrumentationPass(M: LLVMModuleRef); + pub fn LLVMRustAddInstrumentationPass(PM: LLVMPassManagerRef); + // pub fn LLVMRustRunInstrumentationPass(M: LLVMModuleRef); } diff --git a/compiler/rustc_llvm_coverage/src/interface.rs b/compiler/rustc_llvm_coverage/src/interface.rs index 50d451ba5cd..6c132ec5164 100644 --- a/compiler/rustc_llvm_coverage/src/interface.rs +++ b/compiler/rustc_llvm_coverage/src/interface.rs @@ -2,7 +2,9 @@ use crate::write_filenames_section_to_buffer; use super::*; use inkwell::context::Context; +use inkwell::intrinsics::Intrinsic; use inkwell::module::Module; +use inkwell::passes::PassManager; use std::ffi::CString; /// This represents a coverage mapping header that has been written to a module. @@ -28,7 +30,8 @@ pub fn write_coverage_mapping_header<'ctx>( let filenames_hash = hash_bytes(encoded_filename_buffer.bytes.borrow().to_vec()); // Get values - let mapping_version = mapping_version(); + let mapping_version = mapping_version(); // versions are zero-indexed + assert_eq!(mapping_version, 5, "Only mapping version 6 is supported"); let encoded_filenames_len = encoded_filename_buffer.len(); // Create mapping header types @@ -49,6 +52,7 @@ pub fn write_coverage_mapping_header<'ctx>( // Value 4 : Mapping version i32_cov_mapping_version.into(), ], + // https://github.com/rust-lang/rust/blob/e6707df0de337976dce7577e68fc57adcd5e4842/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs#L301 false, ); @@ -102,6 +106,7 @@ impl FunctionRecord { let name_md5_hash = hash_str(&name); // Get indexes of function filenames in module file list + // TODO - hoist this into rusty let mut virtual_file_mapping = Vec::new(); for filename in function_filenames { let filename_idx = written_mapping_header @@ -169,9 +174,39 @@ impl FunctionRecord { i64_translation_unit_hash.into(), i8_mapping_array.into(), ], - false, + // https://github.com/rust-lang/rust/blob/e6707df0de337976dce7577e68fc57adcd5e4842/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs#L311 + true, ); save_func_record_to_mod(&module, self.name_md5_hash, function_record_struct, self.is_used); } } +/// Why the pass isn't working yet: +/// +pub fn run_legacy_coverage_pass<'ctx>(module: &Module<'ctx>) { + // Add the intrinsic + let context = module.get_context(); + // let increment_intrinsic = Intrinsic::find("llvm.instrprof.increment").unwrap(); + // println!("intrinsic: {:?}", increment_intrinsic); + // let definition = increment_intrinsic.get_declaration(&module, &[]).unwrap(); + // println!("definition: {:?}", definition); + + // Run the pass + let pm = PassManager::create(()); + unsafe { + LLVMRustAddInstrumentationPass(pm.as_mut_ptr()); + } + let did_run = pm.run_on(module); + println!("Did run: {}", did_run); + // let did_init = pm.initialize(); + // println!("Did init: {}", did_init); + // let did_finalize = pm.finalize(); + // println!("Did finalize: {:?}", did_finalize); +} + +// TODO +// - why the pass isn't working yet: https://github.com/llvm/llvm-project/blob/f28c006a5895fc0e329fe15fead81e37457cb1d1/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp#L539-L549 +// - pass pgo func var to incr (after creating) - https://github.com/rust-lang/rust/blob/174e73a3f6df6f96ab453493796e461164dea94a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs#L59-L74 +// - this pass is what generates increment calls - https://github.com/rust-lang/rust/blob/1.64.0/compiler/rustc_mir_transform/src/coverage/mod.rs +// - call invoking in normal inkwell pipeline, through -instrprof +// - investigate codegen diffs for function/function blocks/programs diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index b53eb9ae855..dea6392ad90 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -12,7 +12,7 @@ use std::string::FromUtf8Error; mod ffi; pub mod interface; pub mod types; -pub use ffi::LLVMRustRunInstrumentationPass; +pub use ffi::LLVMRustAddInstrumentationPass; use types::*; diff --git a/src/codegen.rs b/src/codegen.rs index 63495b00cfa..c5568245e95 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -228,7 +228,7 @@ impl<'ink> CodeGen<'ink> { //Don't generate external or generic functions if let Some(entry) = global_index.find_pou(implementation.name.as_str()) { if !entry.is_generic() && entry.get_linkage() != &LinkageType::External { - pou_generator.generate_implementation(implementation, &self.debug)?; + pou_generator.generate_implementation(implementation, &self.debug, &self.module)?; } } } @@ -258,6 +258,10 @@ impl<'ink> CodeGen<'ink> { func_record.write_to_module(&self.module); + // unsafe { + // rustc_llvm_coverage::LLVMRustRunInstrumentationPass(self.module.as_mut_ptr()); + // } + #[cfg(feature = "verify")] { self.module @@ -470,6 +474,8 @@ impl<'ink> GeneratedModule<'ink> { println!("Writing to IR"); + rustc_llvm_coverage::interface::run_legacy_coverage_pass(&self.module); + // let pm = PassManager::create(()); // unsafe { // rustc_llvm_coverage::LLVMRustAddInstrumentationPass(pm.as_mut_ptr()); @@ -481,9 +487,9 @@ impl<'ink> GeneratedModule<'ink> { // let did_finalize = pm.finalize(); // println!("Did finalize: {:?}", did_finalize); - unsafe { - rustc_llvm_coverage::LLVMRustRunInstrumentationPass(self.module.as_mut_ptr()); - } + // unsafe { + // rustc_llvm_coverage::llvmrustruninstrumentationpass(self.module.as_mut_ptr()); + // } self.module .print_to_file(&output) diff --git a/src/codegen/generators/pou_generator.rs b/src/codegen/generators/pou_generator.rs index 9258daeec5f..c53736e55b8 100644 --- a/src/codegen/generators/pou_generator.rs +++ b/src/codegen/generators/pou_generator.rs @@ -28,6 +28,7 @@ use crate::index::{ImplementationIndexEntry, VariableIndexEntry}; use crate::index::Index; use indexmap::{IndexMap, IndexSet}; use inkwell::{ + intrinsics::Intrinsic, module::Module, types::{BasicMetadataTypeEnum, BasicTypeEnum, FunctionType}, values::{BasicValue, BasicValueEnum, FunctionValue}, @@ -277,15 +278,17 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { } /// generates a function for the given pou - pub fn generate_implementation( + pub fn generate_implementation<'ctx>( &self, implementation: &Implementation, debug: &DebugBuilderEnum<'ink>, + module: &Module<'ctx>, ) -> Result<(), Diagnostic> { let context = self.llvm.context; let mut local_index = LlvmTypedIndex::create_child(self.llvm_index); let pou_name = &implementation.name; + println!("Generating implementation for {}", pou_name); let current_function = self.llvm_index.find_associated_implementation(pou_name).ok_or_else(|| { Diagnostic::codegen_error( @@ -294,8 +297,6 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { ) })?; - // TODO - this is where we should possibly add counter increments - let (line, column) = implementation .statements .first() @@ -307,7 +308,7 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { .unwrap(); debug.set_debug_location(&self.llvm, ¤t_function, line, column); - //generate the body + //generate the body - "entry" block marks the beginning of the function let block = context.append_basic_block(current_function, "entry"); //Create all labels this function will have @@ -389,6 +390,28 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { &function_context, debug, ); + // This is where we generate the actual statements, but AFTER we initialize locals + // TODO - add increments + { + let increment_intrinsic = Intrinsic::find("llvm.instrprof.increment").unwrap(); + let increment_intrinsic_func = increment_intrinsic.get_declaration(module, &[]).unwrap(); + let pgo_func_var = rustc_llvm_coverage::create_pgo_func_name_var(¤t_function); + + // Create types + let i64_type = self.llvm.context.i64_type(); + let i32_type = self.llvm.context.i32_type(); + + let i8_name_ptr = pgo_func_var.as_pointer_value(); + let i64_hash = i64_type.const_int(1, false); + let i32_num_counters = i32_type.const_int(1, false); + let i64_counter_idx = i64_type.const_int(0, false); + + self.llvm.builder.build_call( + increment_intrinsic_func, + &[i8_name_ptr.into(), i64_hash.into(), i32_num_counters.into(), i64_counter_idx.into()], + "increment_call", + ); + } statement_gen.generate_body(&implementation.statements)?; statement_gen.generate_return_statement()?; } From ecabe92aa54a3d714583b6451f7fb76000ac6ea8 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 15 Jan 2024 10:46:39 +0000 Subject: [PATCH 41/70] pass lowering works, with issue --- .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 88 +++++++++---------- compiler/rustc_llvm_coverage/src/ffi.rs | 2 +- compiler/rustc_llvm_coverage/src/interface.rs | 2 + compiler/rustc_llvm_coverage/src/lib.rs | 2 +- src/codegen.rs | 14 ++- 5 files changed, 53 insertions(+), 55 deletions(-) diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 09ca633c644..18329a520bf 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -48,7 +48,7 @@ ModulePass *createInstrumentationPass() { // Options - https://github.com/llvm-mirror/llvm/blob/2c4ca6832fa6b306ee6a7010bfb80a3f2596f824/include/llvm/Transforms/Instrumentation.h#L129C1-L148C1 InstrProfOptions Options; - // Options.InstrProfileOutput = "rust.profraw"; + Options.InstrProfileOutput = "rust.profraw"; // Options.Atomic = true; // No longer legacy once we updated to newer passes @@ -62,47 +62,45 @@ extern "C" void LLVMRustAddInstrumentationPass(LLVMPassManagerRef PM) { // unwrap(PM)->addPass(InstrProfiling(Options, false)); } -// // https://github.com/rust-lang/rust/blob/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp#L923-L930 -// extern "C" void LLVMRustRunInstrumentationPass(LLVMModuleRef M) { -// // Options -// InstrProfOptions Options; -// // Options.InstrProfileOutput = "rust.profraw"; -// // Options.Atomic = true; - - - -// // Create the analysis managers. -// // These must be declared in this order so that they are destroyed in the -// // correct order due to inter-analysis-manager references. -// LoopAnalysisManager LAM; -// FunctionAnalysisManager FAM; -// CGSCCAnalysisManager CGAM; -// ModuleAnalysisManager MAM; - -// // Create the new pass manager builder. -// // Take a look at the PassBuilder constructor parameters for more -// // customization, e.g. specifying a TargetMachine or various debugging -// // options. -// PassBuilder PB; - -// // Register all the basic analyses with the managers. -// PB.registerModuleAnalyses(MAM); -// PB.registerCGSCCAnalyses(CGAM); -// PB.registerFunctionAnalyses(FAM); -// PB.registerLoopAnalyses(LAM); -// PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); - -// // Create the pass manager. -// // This one corresponds to a typical -O2 optimization pipeline. -// // ModulePassManager MPM = PB.buildPerModuleDefaultPipeline(OptimizationLevel::O0); -// ModulePassManager MPM = PB.buildO0DefaultPipeline(OptimizationLevel::O0, /* PreLinkLTO */ false); - -// // TODO - this needed to be updated in LLVM >=18 -// MPM.addPass(InstrProfiling(Options, false)); - -// // Optimize the IR! -// // MPM.run(MyModule, MAM); -// MPM.run(*unwrap(M), MAM); - - -// } \ No newline at end of file +// https://github.com/rust-lang/rust/blob/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp#L923-L930 +extern "C" void LLVMRustRunInstrumentationPass(LLVMModuleRef M) { + // Options + InstrProfOptions Options; + Options.InstrProfileOutput = "rust.profraw"; + Options.Atomic = true; + + // Create the analysis managers. + // These must be declared in this order so that they are destroyed in the + // correct order due to inter-analysis-manager references. + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + + // Create the new pass manager builder. + // Take a look at the PassBuilder constructor parameters for more + // customization, e.g. specifying a TargetMachine or various debugging + // options. + PassBuilder PB; + + // Register all the basic analyses with the managers. + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + // Create the pass manager. + // This one corresponds to a typical -O2 optimization pipeline. + // ModulePassManager MPM = PB.buildPerModuleDefaultPipeline(OptimizationLevel::O0); + ModulePassManager MPM = PB.buildO0DefaultPipeline(OptimizationLevel::O0, /* PreLinkLTO */ false); + + // TODO - this needed to be updated in LLVM >=18 + MPM.addPass(InstrProfiling(Options, false)); + + // Optimize the IR! + // MPM.run(MyModule, MAM); + MPM.run(*unwrap(M), MAM); + + +} \ No newline at end of file diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index 1796891347d..7163884e5d6 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -54,5 +54,5 @@ extern "C" { pub fn LLVMRustCoverageMappingVersion() -> u32; pub fn LLVMRustAddInstrumentationPass(PM: LLVMPassManagerRef); - // pub fn LLVMRustRunInstrumentationPass(M: LLVMModuleRef); + pub fn LLVMRustRunInstrumentationPass(M: LLVMModuleRef); } diff --git a/compiler/rustc_llvm_coverage/src/interface.rs b/compiler/rustc_llvm_coverage/src/interface.rs index 6c132ec5164..7e1ad943def 100644 --- a/compiler/rustc_llvm_coverage/src/interface.rs +++ b/compiler/rustc_llvm_coverage/src/interface.rs @@ -194,7 +194,9 @@ pub fn run_legacy_coverage_pass<'ctx>(module: &Module<'ctx>) { // Run the pass let pm = PassManager::create(()); unsafe { + // Switch between new and legacy pass managers with the following: LLVMRustAddInstrumentationPass(pm.as_mut_ptr()); + // LLVMRustRunInstrumentationPass(module.as_mut_ptr()); } let did_run = pm.run_on(module); println!("Did run: {}", did_run); diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index dea6392ad90..6831d0ca9a6 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -12,7 +12,7 @@ use std::string::FromUtf8Error; mod ffi; pub mod interface; pub mod types; -pub use ffi::LLVMRustAddInstrumentationPass; +pub use ffi::{LLVMRustAddInstrumentationPass, LLVMRustRunInstrumentationPass}; use types::*; diff --git a/src/codegen.rs b/src/codegen.rs index c5568245e95..8432ee9c663 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -236,20 +236,20 @@ impl<'ink> CodeGen<'ink> { self.debug.finalize(); log::debug!("{}", self.module.to_string()); - let filenames = vec!["cwd".to_string(), self.module_location]; + let filenames = vec!["/home/animcogn/corbanvilla_rusty".to_string(), self.module_location.clone()]; let cov_header = rustc_llvm_coverage::interface::write_coverage_mapping_header(&self.module, filenames); - let prg_func = self.module.get_function("prg").expect("Unable to get prg"); + let prg_func = self.module.get_function("main").expect("Unable to get prg"); let counter1 = Counter::counter_value_reference(CounterId::new(1)); let mapping_regions: Vec = - vec![CounterMappingRegion::code_region(counter1, 0, 0, 0, 3, 10)]; + vec![CounterMappingRegion::code_region(counter1, 1, 8, 1, 10, 10)]; let func_record = rustc_llvm_coverage::interface::FunctionRecord::new( "main".to_string(), 1, - vec!["cwd".to_string()], + vec![self.module_location.clone()], Vec::new(), mapping_regions, true, @@ -258,9 +258,7 @@ impl<'ink> CodeGen<'ink> { func_record.write_to_module(&self.module); - // unsafe { - // rustc_llvm_coverage::LLVMRustRunInstrumentationPass(self.module.as_mut_ptr()); - // } + rustc_llvm_coverage::interface::run_legacy_coverage_pass(&self.module); #[cfg(feature = "verify")] { @@ -474,7 +472,7 @@ impl<'ink> GeneratedModule<'ink> { println!("Writing to IR"); - rustc_llvm_coverage::interface::run_legacy_coverage_pass(&self.module); + // rustc_llvm_coverage::interface::run_legacy_coverage_pass(&self.module); // let pm = PassManager::create(()); // unsafe { From a1980ca9f77460b805081c9432aba9df992499cf Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 15 Jan 2024 12:24:10 +0000 Subject: [PATCH 42/70] working pass, before cleanup --- compiler/rustc_llvm_coverage/src/interface.rs | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/interface.rs b/compiler/rustc_llvm_coverage/src/interface.rs index 7e1ad943def..f20c3b762d1 100644 --- a/compiler/rustc_llvm_coverage/src/interface.rs +++ b/compiler/rustc_llvm_coverage/src/interface.rs @@ -4,7 +4,9 @@ use super::*; use inkwell::context::Context; use inkwell::intrinsics::Intrinsic; use inkwell::module::Module; -use inkwell::passes::PassManager; +use inkwell::passes::PassBuilderOptions; +use inkwell::targets::{CodeModel, InitializationConfig, RelocMode, Target, TargetMachine, TargetTriple}; +use inkwell::OptimizationLevel; use std::ffi::CString; /// This represents a coverage mapping header that has been written to a module. @@ -181,29 +183,30 @@ impl FunctionRecord { save_func_record_to_mod(&module, self.name_md5_hash, function_record_struct, self.is_used); } } -/// Why the pass isn't working yet: -/// + pub fn run_legacy_coverage_pass<'ctx>(module: &Module<'ctx>) { - // Add the intrinsic + // Setup let context = module.get_context(); - // let increment_intrinsic = Intrinsic::find("llvm.instrprof.increment").unwrap(); - // println!("intrinsic: {:?}", increment_intrinsic); - // let definition = increment_intrinsic.get_declaration(&module, &[]).unwrap(); - // println!("definition: {:?}", definition); - - // Run the pass - let pm = PassManager::create(()); - unsafe { - // Switch between new and legacy pass managers with the following: - LLVMRustAddInstrumentationPass(pm.as_mut_ptr()); - // LLVMRustRunInstrumentationPass(module.as_mut_ptr()); - } - let did_run = pm.run_on(module); - println!("Did run: {}", did_run); - // let did_init = pm.initialize(); - // println!("Did init: {}", did_init); - // let did_finalize = pm.finalize(); - // println!("Did finalize: {:?}", did_finalize); + let initialization_config = &InitializationConfig::default(); + inkwell::targets::Target::initialize_all(initialization_config); + + // Architecture Specifics + let triple = TargetTriple::create("x86_64-unknown-linux-gnu"); + module.set_triple(&triple); + let target = Target::from_triple(&triple).unwrap(); + let machine = target + .create_target_machine( + &triple, + "generic", + "", + OptimizationLevel::None, + RelocMode::Default, + CodeModel::Default, + ) + .unwrap(); + + // Run pass (uses new pass manager) + module.run_passes("instrprof", &machine, PassBuilderOptions::create()); } // TODO From 0d79c1ed1c5282eab9d47bb247fd929faae615bd Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 15 Jan 2024 13:06:11 +0000 Subject: [PATCH 43/70] slight cleanup and rename --- compiler/rustc_llvm_coverage/src/ffi.rs | 4 ++-- compiler/rustc_llvm_coverage/src/interface.rs | 8 +++----- compiler/rustc_llvm_coverage/src/lib.rs | 1 - 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index 7163884e5d6..bb68c040b76 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -53,6 +53,6 @@ extern "C" { pub fn LLVMRustCoverageMappingVersion() -> u32; - pub fn LLVMRustAddInstrumentationPass(PM: LLVMPassManagerRef); - pub fn LLVMRustRunInstrumentationPass(M: LLVMModuleRef); + // pub fn LLVMRustAddInstrumentationPass(PM: LLVMPassManagerRef); + // pub fn LLVMRustRunInstrumentationPass(M: LLVMModuleRef); } diff --git a/compiler/rustc_llvm_coverage/src/interface.rs b/compiler/rustc_llvm_coverage/src/interface.rs index f20c3b762d1..b3d08ecb670 100644 --- a/compiler/rustc_llvm_coverage/src/interface.rs +++ b/compiler/rustc_llvm_coverage/src/interface.rs @@ -184,7 +184,9 @@ impl FunctionRecord { } } -pub fn run_legacy_coverage_pass<'ctx>(module: &Module<'ctx>) { +/// This pass will not operate unless the module already has intrinsic calls. +/// See [here](https://github.com/llvm/llvm-project/blob/f28c006a5895fc0e329fe15fead81e37457cb1d1/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp#L539-L549) for why. +pub fn run_instrumentation_lowering_pass<'ctx>(module: &Module<'ctx>) { // Setup let context = module.get_context(); let initialization_config = &InitializationConfig::default(); @@ -210,8 +212,4 @@ pub fn run_legacy_coverage_pass<'ctx>(module: &Module<'ctx>) { } // TODO -// - why the pass isn't working yet: https://github.com/llvm/llvm-project/blob/f28c006a5895fc0e329fe15fead81e37457cb1d1/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp#L539-L549 -// - pass pgo func var to incr (after creating) - https://github.com/rust-lang/rust/blob/174e73a3f6df6f96ab453493796e461164dea94a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs#L59-L74 -// - this pass is what generates increment calls - https://github.com/rust-lang/rust/blob/1.64.0/compiler/rustc_mir_transform/src/coverage/mod.rs -// - call invoking in normal inkwell pipeline, through -instrprof // - investigate codegen diffs for function/function blocks/programs diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 6831d0ca9a6..0830351e57c 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -12,7 +12,6 @@ use std::string::FromUtf8Error; mod ffi; pub mod interface; pub mod types; -pub use ffi::{LLVMRustAddInstrumentationPass, LLVMRustRunInstrumentationPass}; use types::*; From ff439d1fbdbf85d5b10b34255ab81f4bf6b29607 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 15 Jan 2024 13:06:37 +0000 Subject: [PATCH 44/70] fix rename --- src/codegen.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegen.rs b/src/codegen.rs index 8432ee9c663..92e1f1e26b8 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -258,7 +258,7 @@ impl<'ink> CodeGen<'ink> { func_record.write_to_module(&self.module); - rustc_llvm_coverage::interface::run_legacy_coverage_pass(&self.module); + rustc_llvm_coverage::interface::run_instrumentation_lowering_pass(&self.module); #[cfg(feature = "verify")] { From 1c0c35ea537f13d7ffa56136bb4be450df5c6382 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 15 Jan 2024 17:24:45 +0000 Subject: [PATCH 45/70] add python debugger scripts --- scripts/debug/decode_filenames.py | 37 ++++++++++++++++++++++++++++++ scripts/debug/decode_pgo.py | 35 ++++++++++++++++++++++++++++ scripts/debug/parse.py | 38 +++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100755 scripts/debug/decode_filenames.py create mode 100755 scripts/debug/decode_pgo.py create mode 100644 scripts/debug/parse.py diff --git a/scripts/debug/decode_filenames.py b/scripts/debug/decode_filenames.py new file mode 100755 index 00000000000..c9cef59d48f --- /dev/null +++ b/scripts/debug/decode_filenames.py @@ -0,0 +1,37 @@ +#!/usr/bin/python3 + +""" +Useful for decoding filenames in generated IR data. + +WARNING: This script assumes all LEB128 values are one byte only! +""" + +import zlib +import argparse + +from parse import parse_llvm_bytestring, parse_llvm_string_to_list, parse_hex_string + +# This should be the encoded data under `__llvm_prf_nm` +parser = argparse.ArgumentParser() +parser.add_argument("encoded", help="encoded data under `__llvm_prf_nm`") +args = parser.parse_args() + +encoded = args.encoded +decoded = parse_llvm_bytestring(encoded) + +# Take off the headers +num_files = decoded.pop(0) +len_uncompressed = decoded.pop(0) +len_compressed = decoded.pop(0) +assert(len_compressed == len(decoded)) + +# Decompress and separate the filenames +decoded_filenames = zlib.decompress(bytes(decoded)) +assert(len(decoded_filenames) == len_uncompressed) +filenames = parse_llvm_string_to_list(decoded_filenames) + +# Display +print(f'Files: {num_files}') +print(f'Len(uncompressed): {len_uncompressed}') +print(f'Len(compressed): {len_compressed}') +print(f'Filenames: {", ".join([filename.decode() for filename in filenames])}') diff --git a/scripts/debug/decode_pgo.py b/scripts/debug/decode_pgo.py new file mode 100755 index 00000000000..a73b848bc35 --- /dev/null +++ b/scripts/debug/decode_pgo.py @@ -0,0 +1,35 @@ +#!/usr/bin/python3 + +""" +Useful for decoding function names in generated IR data. + +WARNING: This script assumes all LEB128 values are one byte only! +""" + +import zlib +import argparse + +from parse import parse_llvm_bytestring + +# This should be the encoded data under `__llvm_coverage_mapping` +parser = argparse.ArgumentParser() +parser.add_argument("encoded", help="encoded data under `__llvm_coverage_mapping`") +args = parser.parse_args() + +encoded = args.encoded +decoded = parse_llvm_bytestring(encoded) + +# Take off the headers +len_uncompressed = decoded.pop(0) +len_compressed = decoded.pop(0) +assert(len_compressed == len(decoded)) + +# Decompress and separate the function names +decompressed_function_names = zlib.decompress(bytes(decoded)) +assert(len_uncompressed == len(decompressed_function_names)) +function_names = decompressed_function_names.split(b"\x01") + +# Display +print(f'Len(uncompressed): {len_uncompressed}') +print(f'Len(compressed): {len_compressed}') +print(f'Function names: {", ".join([function_name.decode() for function_name in function_names])}') diff --git a/scripts/debug/parse.py b/scripts/debug/parse.py new file mode 100644 index 00000000000..b8233216ac7 --- /dev/null +++ b/scripts/debug/parse.py @@ -0,0 +1,38 @@ +def parse_llvm_bytestring(encoded: str): + """ + Parse strings formatted like: + "\04\0Cx\DA\CBM\CC\CC\03\00\04\1B\01\A6" + """ + decoded = [] + while(encoded): + # \ indicates next two chars are hex + if encoded[0] == '\\': + decoded.append(int(encoded[1:3], 16)) + encoded = encoded[3:] # skip the / and the two hex letters + + # ASCII letter has the value + else: + decoded.append(ord(encoded[0])) + encoded = encoded[1:] + + return decoded + +def parse_hex_string(hex_string): + """ + Parse strings formatted like: + 0011223344.. + """ + return [int(hex_string[i:i+2], 16) for i in range(0, len(hex_string), 2)] + +def parse_llvm_string_to_list(packed_string): + """ + Unpack multiple strings formatted like: + ... + """ + values = [] + while (packed_string): + next_string_length = packed_string[0] + values.append(packed_string[1:next_string_length+1]) + packed_string = packed_string[next_string_length+1:] + + return values From 872dd14327ce1247246aea6cd3cb6a2690f12660 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 15 Jan 2024 17:32:21 +0000 Subject: [PATCH 46/70] comments and fix source map --- compiler/rustc_llvm_coverage/src/interface.rs | 7 +++++-- src/codegen.rs | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/interface.rs b/compiler/rustc_llvm_coverage/src/interface.rs index b3d08ecb670..5fc5376899d 100644 --- a/compiler/rustc_llvm_coverage/src/interface.rs +++ b/compiler/rustc_llvm_coverage/src/interface.rs @@ -193,7 +193,10 @@ pub fn run_instrumentation_lowering_pass<'ctx>(module: &Module<'ctx>) { inkwell::targets::Target::initialize_all(initialization_config); // Architecture Specifics - let triple = TargetTriple::create("x86_64-unknown-linux-gnu"); + // Module.set_triple() is required because the pass needs to know it's compiling + // to ELF [here](https://github.com/llvm/llvm-project/blob/cfa30fa4852275eed0c59b81b5d8088d3e55f778/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp#L1191-L1199). + // TODO - pass this as a param + let triple = TargetTriple::create("x86_64-pc-linux-gnu"); module.set_triple(&triple); let target = Target::from_triple(&triple).unwrap(); let machine = target @@ -208,7 +211,7 @@ pub fn run_instrumentation_lowering_pass<'ctx>(module: &Module<'ctx>) { .unwrap(); // Run pass (uses new pass manager) - module.run_passes("instrprof", &machine, PassBuilderOptions::create()); + let _ = module.run_passes("instrprof", &machine, PassBuilderOptions::create()); } // TODO diff --git a/src/codegen.rs b/src/codegen.rs index 92e1f1e26b8..98da201a883 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -236,15 +236,15 @@ impl<'ink> CodeGen<'ink> { self.debug.finalize(); log::debug!("{}", self.module.to_string()); - let filenames = vec!["/home/animcogn/corbanvilla_rusty".to_string(), self.module_location.clone()]; + let filenames = vec!["/workspaces/corbanvilla_rusty".to_string(), self.module_location.clone()]; let cov_header = rustc_llvm_coverage::interface::write_coverage_mapping_header(&self.module, filenames); let prg_func = self.module.get_function("main").expect("Unable to get prg"); - let counter1 = Counter::counter_value_reference(CounterId::new(1)); + let counter1 = Counter::counter_value_reference(CounterId::new(0)); let mapping_regions: Vec = - vec![CounterMappingRegion::code_region(counter1, 1, 8, 1, 10, 10)]; + vec![CounterMappingRegion::code_region(counter1, 1, 1, 1, 2, 3)]; let func_record = rustc_llvm_coverage::interface::FunctionRecord::new( "main".to_string(), From c505c8fd0005d6f9827998c9727114f27d9fe478 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 15 Jan 2024 18:47:45 +0000 Subject: [PATCH 47/70] add used var calls --- .../llvm-wrapper/CoverageMappingWrapper.cpp | 5 ++++ compiler/rustc_llvm_coverage/src/ffi.rs | 4 +++ compiler/rustc_llvm_coverage/src/lib.rs | 27 ++++++++++++------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp index 154f554d607..8ff8b1d079c 100644 --- a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp @@ -3,6 +3,7 @@ #include "llvm/ProfileData/Coverage/CoverageMappingWriter.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" #include @@ -115,3 +116,7 @@ extern "C" uint32_t LLVMRustCoverageMappingVersion() { return coverage::CovMapVersion::Version5; #endif } + +extern "C" void LLVMRustAppendToUsed(LLVMModuleRef M, GlobalValue *G) { + appendToUsed(*unwrap(M), makeArrayRef(G)); +} diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index bb68c040b76..a1ab38da4b3 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -8,6 +8,7 @@ use libc::{c_char, c_uint, c_void, size_t}; use std::slice; use super::types::*; +use inkwell::values::PointerValue; use llvm_sys::prelude::{LLVMModuleRef, LLVMPassManagerRef, LLVMValueRef}; /// Appending to a Rust string -- used by RawRustStringOstream. @@ -55,4 +56,7 @@ extern "C" { // pub fn LLVMRustAddInstrumentationPass(PM: LLVMPassManagerRef); // pub fn LLVMRustRunInstrumentationPass(M: LLVMModuleRef); + + pub fn LLVMRustAppendToUsed(M: LLVMModuleRef, V: PointerValue); + } diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 0830351e57c..ef7df01d4b4 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -3,6 +3,9 @@ * `rustc` implementation of LLVM code coverage. * * https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs#L220-L221 + * + * TODO - Consider updating functions to reflect configurations in latest Rust (not 1.64) + * https://github.com/rust-lang/rust/blob/master/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs */ const VAR_ALIGN_BYTES: u32 = 8; @@ -15,8 +18,9 @@ pub mod types; use types::*; -use inkwell::comdat::*; use inkwell::{ + comdat::*, + intrinsics::Intrinsic, module::Linkage, types::{AnyType, AsTypeRef, StructType}, values::{AsValueRef, FunctionValue, GlobalValue, StructValue}, @@ -114,8 +118,11 @@ pub fn save_cov_data_to_mod<'ctx>(module: &Module<'ctx>, cov_data_val: StructVal llglobal.set_linkage(Linkage::Private); llglobal.set_section(Some(&covmap_section_name)); llglobal.set_alignment(VAR_ALIGN_BYTES); - // We will skip this for now... I don't think it's necessary (-Corban) - // cx.add_used_global(llglobal); + + // Mark as used to prevent removal by LLVM optimizations + unsafe { + ffi::LLVMRustAppendToUsed(module.as_mut_ptr(), llglobal.as_pointer_value()); + } } pub fn save_func_record_to_mod<'ctx>( @@ -143,21 +150,21 @@ pub fn save_func_record_to_mod<'ctx>( build_string(&mut s).expect("Rust Coverage function record section name failed UTF-8 conversion") }; + // Create types let llglobal = module.add_global(func_record_val.get_type(), None, func_record_var_name.as_str()); + let comdat = module.get_or_insert_comdat(&func_record_var_name); + // Assign llglobal.set_initializer(&func_record_val); llglobal.set_constant(true); llglobal.set_linkage(Linkage::LinkOnceODR); llglobal.set_visibility(GlobalVisibility::Hidden); llglobal.set_section(Some(&func_record_section_name)); llglobal.set_alignment(VAR_ALIGN_BYTES); - - // TODO - verify this in the IR - assert!(llglobal.get_comdat().is_none()); - let comdat = module.get_or_insert_comdat(llglobal.get_name().to_str().unwrap()); - assert!(llglobal.get_comdat().is_none()); llglobal.set_comdat(comdat); - // We will skip this for now... I don't think it's necessary (-Corban) - // cx.add_used_global(llglobal); + // Mark as used to prevent removal by LLVM optimizations + unsafe { + ffi::LLVMRustAppendToUsed(module.as_mut_ptr(), llglobal.as_pointer_value()); + } } From a0732ec697da6eb66ae56f8cc546b8233982bc10 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Tue, 16 Jan 2024 09:12:27 +0000 Subject: [PATCH 48/70] add profiling files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 4bce7875f7b..66512eb0505 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ *.a *.elf *.out +default.profraw +default.profexport From 235e33512209b13ba1d45daf055da7cd47f4fc68 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Tue, 16 Jan 2024 10:03:02 +0000 Subject: [PATCH 49/70] cleanup build files --- compiler/rustc_llvm/build.rs | 6 +- .../llvm-wrapper/ArchiveWrapper.cpp | 222 -- compiler/rustc_llvm/llvm-wrapper/Linker.cpp | 48 - .../rustc_llvm/llvm-wrapper/PassWrapper.cpp | 106 - .../llvm-wrapper/PassWrapper.cpp.bk | 1763 --------------- .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 1972 ----------------- 6 files changed, 1 insertion(+), 4116 deletions(-) delete mode 100644 compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp delete mode 100644 compiler/rustc_llvm/llvm-wrapper/Linker.cpp delete mode 100644 compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp delete mode 100644 compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp.bk delete mode 100644 compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index cde9f912418..345e2c7cc0d 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -209,11 +209,7 @@ fn main() { } rerun_if_changed_anything_in_dir(Path::new("llvm-wrapper")); - cfg.file("llvm-wrapper/PassWrapper.cpp") - .file("llvm-wrapper/RustWrapper.cpp") - .file("llvm-wrapper/ArchiveWrapper.cpp") - .file("llvm-wrapper/CoverageMappingWrapper.cpp") - .file("llvm-wrapper/Linker.cpp") + cfg.file("llvm-wrapper/CoverageMappingWrapper.cpp") .cpp(true) .cpp_link_stdlib(None) // we handle this below .compile("llvm-wrapper"); diff --git a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp deleted file mode 100644 index 5b6ba85bee9..00000000000 --- a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp +++ /dev/null @@ -1,222 +0,0 @@ -// #include "LLVMWrapper.h" - -// #include "llvm/Object/Archive.h" -// #include "llvm/Object/ArchiveWriter.h" -// #include "llvm/Support/Path.h" - -// using namespace llvm; -// using namespace llvm::object; - -// struct RustArchiveMember { -// const char *Filename; -// const char *Name; -// Archive::Child Child; - -// RustArchiveMember() -// : Filename(nullptr), Name(nullptr), -// Child(nullptr, nullptr, nullptr) -// { -// } -// ~RustArchiveMember() {} -// }; - -// struct RustArchiveIterator { -// bool First; -// Archive::child_iterator Cur; -// Archive::child_iterator End; -// std::unique_ptr Err; - -// RustArchiveIterator(Archive::child_iterator Cur, Archive::child_iterator End, -// std::unique_ptr Err) -// : First(true), -// Cur(Cur), -// End(End), -// Err(std::move(Err)) {} -// }; - -// enum class LLVMRustArchiveKind { -// GNU, -// BSD, -// DARWIN, -// COFF, -// }; - -// static Archive::Kind fromRust(LLVMRustArchiveKind Kind) { -// switch (Kind) { -// case LLVMRustArchiveKind::GNU: -// return Archive::K_GNU; -// case LLVMRustArchiveKind::BSD: -// return Archive::K_BSD; -// case LLVMRustArchiveKind::DARWIN: -// return Archive::K_DARWIN; -// case LLVMRustArchiveKind::COFF: -// return Archive::K_COFF; -// default: -// report_fatal_error("Bad ArchiveKind."); -// } -// } - -// typedef OwningBinary *LLVMRustArchiveRef; -// typedef RustArchiveMember *LLVMRustArchiveMemberRef; -// typedef Archive::Child *LLVMRustArchiveChildRef; -// typedef Archive::Child const *LLVMRustArchiveChildConstRef; -// typedef RustArchiveIterator *LLVMRustArchiveIteratorRef; - -// extern "C" LLVMRustArchiveRef LLVMRustOpenArchive(char *Path) { -// ErrorOr> BufOr = -// MemoryBuffer::getFile(Path, -1, false); -// if (!BufOr) { -// LLVMRustSetLastError(BufOr.getError().message().c_str()); -// return nullptr; -// } - -// Expected> ArchiveOr = -// Archive::create(BufOr.get()->getMemBufferRef()); - -// if (!ArchiveOr) { -// LLVMRustSetLastError(toString(ArchiveOr.takeError()).c_str()); -// return nullptr; -// } - -// OwningBinary *Ret = new OwningBinary( -// std::move(ArchiveOr.get()), std::move(BufOr.get())); - -// return Ret; -// } - -// extern "C" void LLVMRustDestroyArchive(LLVMRustArchiveRef RustArchive) { -// delete RustArchive; -// } - -// extern "C" LLVMRustArchiveIteratorRef -// LLVMRustArchiveIteratorNew(LLVMRustArchiveRef RustArchive) { -// Archive *Archive = RustArchive->getBinary(); -// std::unique_ptr Err = std::make_unique(Error::success()); -// auto Cur = Archive->child_begin(*Err); -// if (*Err) { -// LLVMRustSetLastError(toString(std::move(*Err)).c_str()); -// return nullptr; -// } -// auto End = Archive->child_end(); -// return new RustArchiveIterator(Cur, End, std::move(Err)); -// } - -// extern "C" LLVMRustArchiveChildConstRef -// LLVMRustArchiveIteratorNext(LLVMRustArchiveIteratorRef RAI) { -// if (RAI->Cur == RAI->End) -// return nullptr; - -// // Advancing the iterator validates the next child, and this can -// // uncover an error. LLVM requires that we check all Errors, -// // so we only advance the iterator if we actually need to fetch -// // the next child. -// // This means we must not advance the iterator in the *first* call, -// // but instead advance it *before* fetching the child in all later calls. -// if (!RAI->First) { -// ++RAI->Cur; -// if (*RAI->Err) { -// LLVMRustSetLastError(toString(std::move(*RAI->Err)).c_str()); -// return nullptr; -// } -// } else { -// RAI->First = false; -// } - -// if (RAI->Cur == RAI->End) -// return nullptr; - -// const Archive::Child &Child = *RAI->Cur.operator->(); -// Archive::Child *Ret = new Archive::Child(Child); - -// return Ret; -// } - -// extern "C" void LLVMRustArchiveChildFree(LLVMRustArchiveChildRef Child) { -// delete Child; -// } - -// extern "C" void LLVMRustArchiveIteratorFree(LLVMRustArchiveIteratorRef RAI) { -// delete RAI; -// } - -// extern "C" const char * -// LLVMRustArchiveChildName(LLVMRustArchiveChildConstRef Child, size_t *Size) { -// Expected NameOrErr = Child->getName(); -// if (!NameOrErr) { -// // rustc_codegen_llvm currently doesn't use this error string, but it might be -// // useful in the future, and in the mean time this tells LLVM that the -// // error was not ignored and that it shouldn't abort the process. -// LLVMRustSetLastError(toString(NameOrErr.takeError()).c_str()); -// return nullptr; -// } -// StringRef Name = NameOrErr.get(); -// *Size = Name.size(); -// return Name.data(); -// } - -// extern "C" const char *LLVMRustArchiveChildData(LLVMRustArchiveChildRef Child, -// size_t *Size) { -// StringRef Buf; -// Expected BufOrErr = Child->getBuffer(); -// if (!BufOrErr) { -// LLVMRustSetLastError(toString(BufOrErr.takeError()).c_str()); -// return nullptr; -// } -// Buf = BufOrErr.get(); -// *Size = Buf.size(); -// return Buf.data(); -// } - -// extern "C" LLVMRustArchiveMemberRef -// LLVMRustArchiveMemberNew(char *Filename, char *Name, -// LLVMRustArchiveChildRef Child) { -// RustArchiveMember *Member = new RustArchiveMember; -// Member->Filename = Filename; -// Member->Name = Name; -// if (Child) -// Member->Child = *Child; -// return Member; -// } - -// extern "C" void LLVMRustArchiveMemberFree(LLVMRustArchiveMemberRef Member) { -// delete Member; -// } - -// extern "C" LLVMRustResult -// LLVMRustWriteArchive(char *Dst, size_t NumMembers, -// const LLVMRustArchiveMemberRef *NewMembers, -// bool WriteSymbtab, LLVMRustArchiveKind RustKind) { - -// std::vector Members; -// auto Kind = fromRust(RustKind); - -// for (size_t I = 0; I < NumMembers; I++) { -// auto Member = NewMembers[I]; -// assert(Member->Name); -// if (Member->Filename) { -// Expected MOrErr = -// NewArchiveMember::getFile(Member->Filename, true); -// if (!MOrErr) { -// LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); -// return LLVMRustResult::Failure; -// } -// MOrErr->MemberName = sys::path::filename(MOrErr->MemberName); -// Members.push_back(std::move(*MOrErr)); -// } else { -// Expected MOrErr = -// NewArchiveMember::getOldMember(Member->Child, true); -// if (!MOrErr) { -// LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); -// return LLVMRustResult::Failure; -// } -// Members.push_back(std::move(*MOrErr)); -// } -// } - -// auto Result = writeArchive(Dst, Members, WriteSymbtab, Kind, true, false); -// if (!Result) -// return LLVMRustResult::Success; -// LLVMRustSetLastError(toString(std::move(Result)).c_str()); - -// return LLVMRustResult::Failure; -// } diff --git a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp deleted file mode 100644 index aacb97f0696..00000000000 --- a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// #include "llvm/Linker/Linker.h" - -// #include "LLVMWrapper.h" - -// using namespace llvm; - -// struct RustLinker { -// Linker L; -// LLVMContext &Ctx; - -// RustLinker(Module &M) : -// L(M), -// Ctx(M.getContext()) -// {} -// }; - -// extern "C" RustLinker* -// LLVMRustLinkerNew(LLVMModuleRef DstRef) { -// Module *Dst = unwrap(DstRef); - -// return new RustLinker(*Dst); -// } - -// extern "C" void -// LLVMRustLinkerFree(RustLinker *L) { -// delete L; -// } - -// extern "C" bool -// LLVMRustLinkerAdd(RustLinker *L, char *BC, size_t Len) { -// std::unique_ptr Buf = -// MemoryBuffer::getMemBufferCopy(StringRef(BC, Len)); - -// Expected> SrcOrError = -// llvm::getLazyBitcodeModule(Buf->getMemBufferRef(), L->Ctx); -// if (!SrcOrError) { -// LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); -// return false; -// } - -// auto Src = std::move(*SrcOrError); - -// if (L->L.linkInModule(std::move(Src))) { -// LLVMRustSetLastError(""); -// return false; -// } -// return true; -// } diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp deleted file mode 100644 index 18329a520bf..00000000000 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Instrumentation/InstrProfiling.h" -#include "llvm/Transforms/Instrumentation.h" - -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/InitializePasses.h" -#include "llvm/IR/AutoUpgrade.h" -#include "llvm/IR/AssemblyAnnotationWriter.h" -#include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Verifier.h" -#include "llvm/Object/ObjectFile.h" -#include "llvm/Object/IRObjectFile.h" -#include "llvm/Passes/PassBuilder.h" -#include "llvm/Passes/PassPlugin.h" -#include "llvm/Passes/StandardInstrumentations.h" -#include "llvm/Support/CBindingWrapping.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/Host.h" -#include "llvm/MC/TargetRegistry.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" -#include "llvm/Transforms/IPO/AlwaysInliner.h" -#include "llvm/Transforms/IPO/FunctionImport.h" -#include "llvm/Transforms/Utils/AddDiscriminators.h" -#include "llvm/Transforms/Utils/FunctionImportUtils.h" -#include "llvm/LTO/LTO.h" -#include "llvm/Bitcode/BitcodeWriterPass.h" -#include "llvm-c/Transforms/PassManagerBuilder.h" - -#include "llvm/Transforms/Instrumentation.h" -#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" -#include "llvm/Support/TimeProfiler.h" -#include "llvm/Transforms/Instrumentation/GCOVProfiler.h" -#include "llvm/Transforms/Instrumentation/InstrProfiling.h" -#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" -#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" -#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" -#include "llvm/Transforms/Utils/CanonicalizeAliases.h" -#include "llvm/Transforms/Utils/NameAnonGlobals.h" -#include "llvm/Transforms/Utils.h" - -using namespace llvm; - -ModulePass *createInstrumentationPass() { - // Options - https://github.com/llvm-mirror/llvm/blob/2c4ca6832fa6b306ee6a7010bfb80a3f2596f824/include/llvm/Transforms/Instrumentation.h#L129C1-L148C1 - InstrProfOptions Options; - - Options.InstrProfileOutput = "rust.profraw"; - // Options.Atomic = true; - - // No longer legacy once we updated to newer passes - // return new InstrProfilingLegacyPass(Options, false); - return createInstrProfilingLegacyPass(Options, false); -} - -extern "C" void LLVMRustAddInstrumentationPass(LLVMPassManagerRef PM) { - - unwrap(PM)->add(createInstrumentationPass()); - // unwrap(PM)->addPass(InstrProfiling(Options, false)); -} - -// https://github.com/rust-lang/rust/blob/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp#L923-L930 -extern "C" void LLVMRustRunInstrumentationPass(LLVMModuleRef M) { - // Options - InstrProfOptions Options; - Options.InstrProfileOutput = "rust.profraw"; - Options.Atomic = true; - - // Create the analysis managers. - // These must be declared in this order so that they are destroyed in the - // correct order due to inter-analysis-manager references. - LoopAnalysisManager LAM; - FunctionAnalysisManager FAM; - CGSCCAnalysisManager CGAM; - ModuleAnalysisManager MAM; - - // Create the new pass manager builder. - // Take a look at the PassBuilder constructor parameters for more - // customization, e.g. specifying a TargetMachine or various debugging - // options. - PassBuilder PB; - - // Register all the basic analyses with the managers. - PB.registerModuleAnalyses(MAM); - PB.registerCGSCCAnalyses(CGAM); - PB.registerFunctionAnalyses(FAM); - PB.registerLoopAnalyses(LAM); - PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); - - // Create the pass manager. - // This one corresponds to a typical -O2 optimization pipeline. - // ModulePassManager MPM = PB.buildPerModuleDefaultPipeline(OptimizationLevel::O0); - ModulePassManager MPM = PB.buildO0DefaultPipeline(OptimizationLevel::O0, /* PreLinkLTO */ false); - - // TODO - this needed to be updated in LLVM >=18 - MPM.addPass(InstrProfiling(Options, false)); - - // Optimize the IR! - // MPM.run(MyModule, MAM); - MPM.run(*unwrap(M), MAM); - - -} \ No newline at end of file diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp.bk b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp.bk deleted file mode 100644 index c7634175414..00000000000 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp.bk +++ /dev/null @@ -1,1763 +0,0 @@ -// #include - -// #include -// #include - -// #include "LLVMWrapper.h" - -// #include "llvm/Analysis/AliasAnalysis.h" -// #include "llvm/Analysis/TargetLibraryInfo.h" -// #include "llvm/Analysis/TargetTransformInfo.h" -// #include "llvm/CodeGen/TargetSubtargetInfo.h" -// #include "llvm/InitializePasses.h" -// #include "llvm/IR/AutoUpgrade.h" -// #include "llvm/IR/AssemblyAnnotationWriter.h" -// #include "llvm/IR/IntrinsicInst.h" -// #include "llvm/IR/Verifier.h" -// #include "llvm/Object/ObjectFile.h" -// #include "llvm/Object/IRObjectFile.h" -// #include "llvm/Passes/PassBuilder.h" -// #include "llvm/Passes/PassPlugin.h" -// #include "llvm/Passes/StandardInstrumentations.h" -// #include "llvm/Support/CBindingWrapping.h" -// #include "llvm/Support/FileSystem.h" -// #include "llvm/Support/Host.h" -// #if LLVM_VERSION_LT(14, 0) -// #include "llvm/Support/TargetRegistry.h" -// #else -// #include "llvm/MC/TargetRegistry.h" -// #endif -// #include "llvm/Target/TargetMachine.h" -// #include "llvm/Transforms/IPO/PassManagerBuilder.h" -// #include "llvm/Transforms/IPO/AlwaysInliner.h" -// #include "llvm/Transforms/IPO/FunctionImport.h" -// #include "llvm/Transforms/Utils/AddDiscriminators.h" -// #include "llvm/Transforms/Utils/FunctionImportUtils.h" -// #include "llvm/LTO/LTO.h" -// #include "llvm/Bitcode/BitcodeWriterPass.h" -// #include "llvm-c/Transforms/PassManagerBuilder.h" - -// #include "llvm/Transforms/Instrumentation.h" -// #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" -// #include "llvm/Support/TimeProfiler.h" -// #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" -// #include "llvm/Transforms/Instrumentation/InstrProfiling.h" -// #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" -// #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" -// #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" -// #include "llvm/Transforms/Utils/CanonicalizeAliases.h" -// #include "llvm/Transforms/Utils/NameAnonGlobals.h" -// #include "llvm/Transforms/Utils.h" - -// using namespace llvm; - -// typedef struct LLVMOpaquePass *LLVMPassRef; -// typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; - -// DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) -// DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) - -// extern "C" void LLVMInitializePasses() { -// PassRegistry &Registry = *PassRegistry::getPassRegistry(); -// initializeCore(Registry); -// initializeCodeGen(Registry); -// initializeScalarOpts(Registry); -// initializeVectorization(Registry); -// initializeIPO(Registry); -// initializeAnalysis(Registry); -// initializeTransformUtils(Registry); -// initializeInstCombine(Registry); -// initializeInstrumentation(Registry); -// initializeTarget(Registry); -// } - -// extern "C" void LLVMTimeTraceProfilerInitialize() { -// timeTraceProfilerInitialize( -// /* TimeTraceGranularity */ 0, -// /* ProcName */ "rustc"); -// } - -// extern "C" void LLVMTimeTraceProfilerFinishThread() { -// timeTraceProfilerFinishThread(); -// } - -// extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { -// StringRef FN(FileName); -// std::error_code EC; -// raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); - -// timeTraceProfilerWrite(OS); -// timeTraceProfilerCleanup(); -// } - -// extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { -// #if LLVM_VERSION_LT(15, 0) -// StringRef SR(PassName); -// PassRegistry *PR = PassRegistry::getPassRegistry(); - -// const PassInfo *PI = PR->getPassInfo(SR); -// if (PI) { -// return wrap(PI->createPass()); -// } -// return nullptr; -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) { -// #if LLVM_VERSION_LT(15, 0) -// const bool CompileKernel = false; -// const bool UseAfterScope = true; - -// return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope)); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) { -// #if LLVM_VERSION_LT(15, 0) -// const bool CompileKernel = false; - -// return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover)); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) { -// #if LLVM_VERSION_LT(15, 0) -// const bool CompileKernel = false; - -// return wrap(createMemorySanitizerLegacyPassPass( -// MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel})); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() { -// #if LLVM_VERSION_LT(15, 0) -// return wrap(createThreadSanitizerLegacyPassPass()); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) { -// #if LLVM_VERSION_LT(15, 0) -// const bool CompileKernel = false; - -// return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover)); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { -// #if LLVM_VERSION_LT(15, 0) -// assert(RustPass); -// Pass *Pass = unwrap(RustPass); -// PassManagerBase *PMB = unwrap(PMR); -// PMB->add(Pass); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() { -// #if LLVM_VERSION_LT(15, 0) -// return LLVMPassManagerBuilderCreate(); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) { -// #if LLVM_VERSION_LT(15, 0) -// LLVMPassManagerBuilderDispose(PMB); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager( -// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { -// #if LLVM_VERSION_LT(15, 0) -// LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager( -// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { -// #if LLVM_VERSION_LT(15, 0) -// LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager( -// LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) { -// #if LLVM_VERSION_LT(15, 0) -// LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" -// void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( -// LLVMPassManagerBuilderRef PMBR, -// LLVMPassManagerRef PMR -// ) { -// #if LLVM_VERSION_LT(15, 0) -// unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold( -// LLVMPassManagerBuilderRef PMB, unsigned Threshold) { -// #if LLVM_VERSION_LT(15, 0) -// LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// extern "C" -// void LLVMRustAddLastExtensionPasses( -// LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { -// #if LLVM_VERSION_LT(15, 0) -// auto AddExtensionPasses = [Passes, NumPasses]( -// const PassManagerBuilder &Builder, PassManagerBase &PM) { -// for (size_t I = 0; I < NumPasses; I++) { -// PM.add(unwrap(Passes[I])); -// } -// }; -// // Add the passes to both of the pre-finalization extension points, -// // so they are run for optimized and non-optimized builds. -// unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, -// AddExtensionPasses); -// unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, -// AddExtensionPasses); -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// #ifdef LLVM_COMPONENT_X86 -// #define SUBTARGET_X86 SUBTARGET(X86) -// #else -// #define SUBTARGET_X86 -// #endif - -// #ifdef LLVM_COMPONENT_ARM -// #define SUBTARGET_ARM SUBTARGET(ARM) -// #else -// #define SUBTARGET_ARM -// #endif - -// #ifdef LLVM_COMPONENT_AARCH64 -// #define SUBTARGET_AARCH64 SUBTARGET(AArch64) -// #else -// #define SUBTARGET_AARCH64 -// #endif - -// #ifdef LLVM_COMPONENT_AVR -// #define SUBTARGET_AVR SUBTARGET(AVR) -// #else -// #define SUBTARGET_AVR -// #endif - -// #ifdef LLVM_COMPONENT_M68k -// #define SUBTARGET_M68K SUBTARGET(M68k) -// #else -// #define SUBTARGET_M68K -// #endif - -// #ifdef LLVM_COMPONENT_MIPS -// #define SUBTARGET_MIPS SUBTARGET(Mips) -// #else -// #define SUBTARGET_MIPS -// #endif - -// #ifdef LLVM_COMPONENT_POWERPC -// #define SUBTARGET_PPC SUBTARGET(PPC) -// #else -// #define SUBTARGET_PPC -// #endif - -// #ifdef LLVM_COMPONENT_SYSTEMZ -// #define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) -// #else -// #define SUBTARGET_SYSTEMZ -// #endif - -// #ifdef LLVM_COMPONENT_MSP430 -// #define SUBTARGET_MSP430 SUBTARGET(MSP430) -// #else -// #define SUBTARGET_MSP430 -// #endif - -// #ifdef LLVM_COMPONENT_RISCV -// #define SUBTARGET_RISCV SUBTARGET(RISCV) -// #else -// #define SUBTARGET_RISCV -// #endif - -// #ifdef LLVM_COMPONENT_SPARC -// #define SUBTARGET_SPARC SUBTARGET(Sparc) -// #else -// #define SUBTARGET_SPARC -// #endif - -// #ifdef LLVM_COMPONENT_HEXAGON -// #define SUBTARGET_HEXAGON SUBTARGET(Hexagon) -// #else -// #define SUBTARGET_HEXAGON -// #endif - -// #define GEN_SUBTARGETS \ -// SUBTARGET_X86 \ -// SUBTARGET_ARM \ -// SUBTARGET_AARCH64 \ -// SUBTARGET_AVR \ -// SUBTARGET_M68K \ -// SUBTARGET_MIPS \ -// SUBTARGET_PPC \ -// SUBTARGET_SYSTEMZ \ -// SUBTARGET_MSP430 \ -// SUBTARGET_SPARC \ -// SUBTARGET_HEXAGON \ -// SUBTARGET_RISCV \ - -// #define SUBTARGET(x) \ -// namespace llvm { \ -// extern const SubtargetFeatureKV x##FeatureKV[]; \ -// extern const SubtargetFeatureKV x##SubTypeKV[]; \ -// } - -// GEN_SUBTARGETS -// #undef SUBTARGET - -// extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, -// const char *Feature) { -// TargetMachine *Target = unwrap(TM); -// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); -// return MCInfo->checkFeatures(std::string("+") + Feature); -// } - -// enum class LLVMRustCodeModel { -// Tiny, -// Small, -// Kernel, -// Medium, -// Large, -// None, -// }; - -// static Optional fromRust(LLVMRustCodeModel Model) { -// switch (Model) { -// case LLVMRustCodeModel::Tiny: -// return CodeModel::Tiny; -// case LLVMRustCodeModel::Small: -// return CodeModel::Small; -// case LLVMRustCodeModel::Kernel: -// return CodeModel::Kernel; -// case LLVMRustCodeModel::Medium: -// return CodeModel::Medium; -// case LLVMRustCodeModel::Large: -// return CodeModel::Large; -// case LLVMRustCodeModel::None: -// return None; -// default: -// report_fatal_error("Bad CodeModel."); -// } -// } - -// enum class LLVMRustCodeGenOptLevel { -// None, -// Less, -// Default, -// Aggressive, -// }; - -// static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { -// switch (Level) { -// case LLVMRustCodeGenOptLevel::None: -// return CodeGenOpt::None; -// case LLVMRustCodeGenOptLevel::Less: -// return CodeGenOpt::Less; -// case LLVMRustCodeGenOptLevel::Default: -// return CodeGenOpt::Default; -// case LLVMRustCodeGenOptLevel::Aggressive: -// return CodeGenOpt::Aggressive; -// default: -// report_fatal_error("Bad CodeGenOptLevel."); -// } -// } - -// enum class LLVMRustPassBuilderOptLevel { -// O0, -// O1, -// O2, -// O3, -// Os, -// Oz, -// }; - -// #if LLVM_VERSION_LT(14,0) -// using OptimizationLevel = PassBuilder::OptimizationLevel; -// #endif - -// static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { -// switch (Level) { -// case LLVMRustPassBuilderOptLevel::O0: -// return OptimizationLevel::O0; -// case LLVMRustPassBuilderOptLevel::O1: -// return OptimizationLevel::O1; -// case LLVMRustPassBuilderOptLevel::O2: -// return OptimizationLevel::O2; -// case LLVMRustPassBuilderOptLevel::O3: -// return OptimizationLevel::O3; -// case LLVMRustPassBuilderOptLevel::Os: -// return OptimizationLevel::Os; -// case LLVMRustPassBuilderOptLevel::Oz: -// return OptimizationLevel::Oz; -// default: -// report_fatal_error("Bad PassBuilderOptLevel."); -// } -// } - -// enum class LLVMRustRelocModel { -// Static, -// PIC, -// DynamicNoPic, -// ROPI, -// RWPI, -// ROPIRWPI, -// }; - -// static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { -// switch (RustReloc) { -// case LLVMRustRelocModel::Static: -// return Reloc::Static; -// case LLVMRustRelocModel::PIC: -// return Reloc::PIC_; -// case LLVMRustRelocModel::DynamicNoPic: -// return Reloc::DynamicNoPIC; -// case LLVMRustRelocModel::ROPI: -// return Reloc::ROPI; -// case LLVMRustRelocModel::RWPI: -// return Reloc::RWPI; -// case LLVMRustRelocModel::ROPIRWPI: -// return Reloc::ROPI_RWPI; -// } -// report_fatal_error("Bad RelocModel."); -// } - -// #ifdef LLVM_RUSTLLVM -// /// getLongestEntryLength - Return the length of the longest entry in the table. -// template -// static size_t getLongestEntryLength(ArrayRef Table) { -// size_t MaxLen = 0; -// for (auto &I : Table) -// MaxLen = std::max(MaxLen, std::strlen(I.Key)); -// return MaxLen; -// } - -// extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { -// const TargetMachine *Target = unwrap(TM); -// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); -// const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); -// const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); -// const ArrayRef CPUTable = MCInfo->getCPUTable(); -// unsigned MaxCPULen = getLongestEntryLength(CPUTable); - -// printf("Available CPUs for this target:\n"); -// if (HostArch == TargetArch) { -// const StringRef HostCPU = sys::getHostCPUName(); -// printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", -// MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); -// } -// for (auto &CPU : CPUTable) -// printf(" %-*s\n", MaxCPULen, CPU.Key); -// printf("\n"); -// } - -// extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { -// const TargetMachine *Target = unwrap(TM); -// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); -// const ArrayRef FeatTable = MCInfo->getFeatureTable(); -// return FeatTable.size(); -// } - -// extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, -// const char** Feature, const char** Desc) { -// const TargetMachine *Target = unwrap(TM); -// const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); -// const ArrayRef FeatTable = MCInfo->getFeatureTable(); -// const SubtargetFeatureKV Feat = FeatTable[Index]; -// *Feature = Feat.Key; -// *Desc = Feat.Desc; -// } - -// #else - -// extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { -// printf("Target CPU help is not supported by this LLVM version.\n\n"); -// } - -// extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) { -// return 0; -// } - -// extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {} -// #endif - -// extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { -// StringRef Name = sys::getHostCPUName(); -// *len = Name.size(); -// return Name.data(); -// } - -// extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( -// const char *TripleStr, const char *CPU, const char *Feature, -// const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, -// LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, -// bool FunctionSections, -// bool DataSections, -// bool UniqueSectionNames, -// bool TrapUnreachable, -// bool Singlethread, -// bool AsmComments, -// bool EmitStackSizeSection, -// bool RelaxELFRelocations, -// bool UseInitArray, -// const char *SplitDwarfFile) { - -// auto OptLevel = fromRust(RustOptLevel); -// auto RM = fromRust(RustReloc); -// auto CM = fromRust(RustCM); - -// std::string Error; -// Triple Trip(Triple::normalize(TripleStr)); -// const llvm::Target *TheTarget = -// TargetRegistry::lookupTarget(Trip.getTriple(), Error); -// if (TheTarget == nullptr) { -// LLVMRustSetLastError(Error.c_str()); -// return nullptr; -// } - -// TargetOptions Options; - -// Options.FloatABIType = FloatABI::Default; -// if (UseSoftFloat) { -// Options.FloatABIType = FloatABI::Soft; -// } -// Options.DataSections = DataSections; -// Options.FunctionSections = FunctionSections; -// Options.UniqueSectionNames = UniqueSectionNames; -// Options.MCOptions.AsmVerbose = AsmComments; -// Options.MCOptions.PreserveAsmComments = AsmComments; -// Options.MCOptions.ABIName = ABIStr; -// if (SplitDwarfFile) { -// Options.MCOptions.SplitDwarfFile = SplitDwarfFile; -// } -// Options.RelaxELFRelocations = RelaxELFRelocations; -// Options.UseInitArray = UseInitArray; - -// if (TrapUnreachable) { -// // Tell LLVM to codegen `unreachable` into an explicit trap instruction. -// // This limits the extent of possible undefined behavior in some cases, as -// // it prevents control flow from "falling through" into whatever code -// // happens to be laid out next in memory. -// Options.TrapUnreachable = true; -// } - -// if (Singlethread) { -// Options.ThreadModel = ThreadModel::Single; -// } - -// Options.EmitStackSizeSection = EmitStackSizeSection; - -// TargetMachine *TM = TheTarget->createTargetMachine( -// Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); -// return wrap(TM); -// } - -// extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { -// delete unwrap(TM); -// } - -// extern "C" void LLVMRustConfigurePassManagerBuilder( -// LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, -// bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, -// const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath, -// int SizeLevel) { -// #if LLVM_VERSION_LT(15, 0) -// unwrap(PMBR)->MergeFunctions = MergeFunctions; -// unwrap(PMBR)->SLPVectorize = SLPVectorize; -// unwrap(PMBR)->OptLevel = fromRust(OptLevel); -// unwrap(PMBR)->LoopVectorize = LoopVectorize; -// unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; -// unwrap(PMBR)->SizeLevel = SizeLevel; -// unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0; - -// if (PGOGenPath) { -// assert(!PGOUsePath && !PGOSampleUsePath); -// unwrap(PMBR)->EnablePGOInstrGen = true; -// unwrap(PMBR)->PGOInstrGen = PGOGenPath; -// } else if (PGOUsePath) { -// assert(!PGOSampleUsePath); -// unwrap(PMBR)->PGOInstrUse = PGOUsePath; -// } else if (PGOSampleUsePath) { -// unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath; -// } -// #else -// report_fatal_error("Legacy PM not supported with LLVM 15"); -// #endif -// } - -// // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` -// // field of a PassManagerBuilder, we expose our own method of doing so. -// extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, -// LLVMModuleRef M, -// bool DisableSimplifyLibCalls) { -// Triple TargetTriple(unwrap(M)->getTargetTriple()); -// TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); -// if (DisableSimplifyLibCalls) -// TLI->disableAllFunctions(); -// unwrap(PMBR)->LibraryInfo = TLI; -// } - -// // Unfortunately, the LLVM C API doesn't provide a way to create the -// // TargetLibraryInfo pass, so we use this method to do so. -// extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, -// bool DisableSimplifyLibCalls) { -// Triple TargetTriple(unwrap(M)->getTargetTriple()); -// TargetLibraryInfoImpl TLII(TargetTriple); -// if (DisableSimplifyLibCalls) -// TLII.disableAllFunctions(); -// unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); -// } - -// // Unfortunately, the LLVM C API doesn't provide an easy way of iterating over -// // all the functions in a module, so we do that manually here. You'll find -// // similar code in clang's BackendUtil.cpp file. -// extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, -// LLVMModuleRef M) { -// llvm::legacy::FunctionPassManager *P = -// unwrap(PMR); -// P->doInitialization(); - -// // Upgrade all calls to old intrinsics first. -// for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) -// UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove - -// for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; -// ++I) -// if (!I->isDeclaration()) -// P->run(*I); - -// P->doFinalization(); -// } - -// extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { -// // Initializing the command-line options more than once is not allowed. So, -// // check if they've already been initialized. (This could happen if we're -// // being called from rustpkg, for example). If the arguments change, then -// // that's just kinda unfortunate. -// static bool Initialized = false; -// if (Initialized) -// return; -// Initialized = true; -// cl::ParseCommandLineOptions(Argc, Argv); -// } - -// enum class LLVMRustFileType { -// AssemblyFile, -// ObjectFile, -// }; - -// static CodeGenFileType fromRust(LLVMRustFileType Type) { -// switch (Type) { -// case LLVMRustFileType::AssemblyFile: -// return CGFT_AssemblyFile; -// case LLVMRustFileType::ObjectFile: -// return CGFT_ObjectFile; -// default: -// report_fatal_error("Bad FileType."); -// } -// } - -// extern "C" LLVMRustResult -// LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, -// LLVMModuleRef M, const char *Path, const char *DwoPath, -// LLVMRustFileType RustFileType) { -// llvm::legacy::PassManager *PM = unwrap(PMR); -// auto FileType = fromRust(RustFileType); - -// std::string ErrorInfo; -// std::error_code EC; -// raw_fd_ostream OS(Path, EC, sys::fs::OF_None); -// if (EC) -// ErrorInfo = EC.message(); -// if (ErrorInfo != "") { -// LLVMRustSetLastError(ErrorInfo.c_str()); -// return LLVMRustResult::Failure; -// } - -// buffer_ostream BOS(OS); -// if (DwoPath) { -// raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None); -// EC.clear(); -// if (EC) -// ErrorInfo = EC.message(); -// if (ErrorInfo != "") { -// LLVMRustSetLastError(ErrorInfo.c_str()); -// return LLVMRustResult::Failure; -// } -// buffer_ostream DBOS(DOS); -// unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); -// PM->run(*unwrap(M)); -// } else { -// unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); -// PM->run(*unwrap(M)); -// } - -// // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output -// // stream (OS), so the only real safe place to delete this is here? Don't we -// // wish this was written in Rust? -// LLVMDisposePassManager(PMR); -// return LLVMRustResult::Success; -// } - -// extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler -// const char*, // pass name -// const char*); // IR name -// extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler - -// std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { -// if (any_isa(WrappedIr)) -// return any_cast(WrappedIr)->getName().str(); -// if (any_isa(WrappedIr)) -// return any_cast(WrappedIr)->getName().str(); -// if (any_isa(WrappedIr)) -// return any_cast(WrappedIr)->getName().str(); -// if (any_isa(WrappedIr)) -// return any_cast(WrappedIr)->getName(); -// return ""; -// } - - -// void LLVMSelfProfileInitializeCallbacks( -// PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, -// LLVMRustSelfProfileBeforePassCallback BeforePassCallback, -// LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { -// PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( -// StringRef Pass, llvm::Any Ir) { -// std::string PassName = Pass.str(); -// std::string IrName = LLVMRustwrappedIrGetName(Ir); -// BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); -// }); - -// PIC.registerAfterPassCallback( -// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, -// const PreservedAnalyses &Preserved) { -// AfterPassCallback(LlvmSelfProfiler); -// }); - -// PIC.registerAfterPassInvalidatedCallback( -// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { -// AfterPassCallback(LlvmSelfProfiler); -// }); - -// PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( -// StringRef Pass, llvm::Any Ir) { -// std::string PassName = Pass.str(); -// std::string IrName = LLVMRustwrappedIrGetName(Ir); -// BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); -// }); - -// PIC.registerAfterAnalysisCallback( -// [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { -// AfterPassCallback(LlvmSelfProfiler); -// }); -// } - -// enum class LLVMRustOptStage { -// PreLinkNoLTO, -// PreLinkThinLTO, -// PreLinkFatLTO, -// ThinLTO, -// FatLTO, -// }; - -// struct LLVMRustSanitizerOptions { -// bool SanitizeAddress; -// bool SanitizeAddressRecover; -// bool SanitizeMemory; -// bool SanitizeMemoryRecover; -// int SanitizeMemoryTrackOrigins; -// bool SanitizeThread; -// bool SanitizeHWAddress; -// bool SanitizeHWAddressRecover; -// }; - -// extern "C" LLVMRustResult -// LLVMRustOptimizeWithNewPassManager( -// LLVMModuleRef ModuleRef, -// LLVMTargetMachineRef TMRef, -// LLVMRustPassBuilderOptLevel OptLevelRust, -// LLVMRustOptStage OptStage, -// bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, -// bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, -// bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, -// LLVMRustSanitizerOptions *SanitizerOptions, -// const char *PGOGenPath, const char *PGOUsePath, -// bool InstrumentCoverage, bool InstrumentGCOV, -// const char *PGOSampleUsePath, bool DebugInfoForProfiling, -// void* LlvmSelfProfiler, -// LLVMRustSelfProfileBeforePassCallback BeforePassCallback, -// LLVMRustSelfProfileAfterPassCallback AfterPassCallback, -// const char *ExtraPasses, size_t ExtraPassesLen, -// const char *LLVMPlugins, size_t LLVMPluginsLen) { -// Module *TheModule = unwrap(ModuleRef); -// TargetMachine *TM = unwrap(TMRef); -// OptimizationLevel OptLevel = fromRust(OptLevelRust); - - -// PipelineTuningOptions PTO; -// PTO.LoopUnrolling = UnrollLoops; -// PTO.LoopInterleaving = UnrollLoops; -// PTO.LoopVectorization = LoopVectorize; -// PTO.SLPVectorization = SLPVectorize; -// PTO.MergeFunctions = MergeFunctions; - -// // FIXME: We may want to expose this as an option. -// bool DebugPassManager = false; - -// PassInstrumentationCallbacks PIC; -// StandardInstrumentations SI(DebugPassManager); -// SI.registerCallbacks(PIC); - -// if (LlvmSelfProfiler){ -// LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); -// } - -// Optional PGOOpt; -// if (PGOGenPath) { -// assert(!PGOUsePath && !PGOSampleUsePath); -// PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr, -// PGOOptions::NoCSAction, DebugInfoForProfiling); -// } else if (PGOUsePath) { -// assert(!PGOSampleUsePath); -// PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse, -// PGOOptions::NoCSAction, DebugInfoForProfiling); -// } else if (PGOSampleUsePath) { -// PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse, -// PGOOptions::NoCSAction, DebugInfoForProfiling); -// } else if (DebugInfoForProfiling) { -// PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction, -// PGOOptions::NoCSAction, DebugInfoForProfiling); -// } - -// #if LLVM_VERSION_GE(13, 0) -// PassBuilder PB(TM, PTO, PGOOpt, &PIC); -// LoopAnalysisManager LAM; -// FunctionAnalysisManager FAM; -// CGSCCAnalysisManager CGAM; -// ModuleAnalysisManager MAM; -// #else -// PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC); -// LoopAnalysisManager LAM(DebugPassManager); -// FunctionAnalysisManager FAM(DebugPassManager); -// CGSCCAnalysisManager CGAM(DebugPassManager); -// ModuleAnalysisManager MAM(DebugPassManager); -// #endif - -// FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); - -// Triple TargetTriple(TheModule->getTargetTriple()); -// std::unique_ptr TLII(new TargetLibraryInfoImpl(TargetTriple)); -// if (DisableSimplifyLibCalls) -// TLII->disableAllFunctions(); -// FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); - -// PB.registerModuleAnalyses(MAM); -// PB.registerCGSCCAnalyses(CGAM); -// PB.registerFunctionAnalyses(FAM); -// PB.registerLoopAnalyses(LAM); -// PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); - -// // We manually collect pipeline callbacks so we can apply them at O0, where the -// // PassBuilder does not create a pipeline. -// std::vector> -// PipelineStartEPCallbacks; -// std::vector> -// OptimizerLastEPCallbacks; - -// if (VerifyIR) { -// PipelineStartEPCallbacks.push_back( -// [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { -// MPM.addPass(VerifierPass()); -// } -// ); -// } - -// if (InstrumentGCOV) { -// PipelineStartEPCallbacks.push_back( -// [](ModulePassManager &MPM, OptimizationLevel Level) { -// MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); -// } -// ); -// } - -// if (InstrumentCoverage) { -// PipelineStartEPCallbacks.push_back( -// [](ModulePassManager &MPM, OptimizationLevel Level) { -// InstrProfOptions Options; -// MPM.addPass(InstrProfiling(Options, false)); -// } -// ); -// } - -// if (SanitizerOptions) { -// if (SanitizerOptions->SanitizeMemory) { -// MemorySanitizerOptions Options( -// SanitizerOptions->SanitizeMemoryTrackOrigins, -// SanitizerOptions->SanitizeMemoryRecover, -// /*CompileKernel=*/false); -// OptimizerLastEPCallbacks.push_back( -// [Options](ModulePassManager &MPM, OptimizationLevel Level) { -// #if LLVM_VERSION_GE(14, 0) -// MPM.addPass(ModuleMemorySanitizerPass(Options)); -// #else -// MPM.addPass(MemorySanitizerPass(Options)); -// #endif -// MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); -// } -// ); -// } - -// if (SanitizerOptions->SanitizeThread) { -// OptimizerLastEPCallbacks.push_back( -// [](ModulePassManager &MPM, OptimizationLevel Level) { -// #if LLVM_VERSION_GE(14, 0) -// MPM.addPass(ModuleThreadSanitizerPass()); -// #else -// MPM.addPass(ThreadSanitizerPass()); -// #endif -// MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); -// } -// ); -// } - -// if (SanitizerOptions->SanitizeAddress) { -// OptimizerLastEPCallbacks.push_back( -// [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { -// #if LLVM_VERSION_LT(15, 0) -// MPM.addPass(RequireAnalysisPass()); -// #endif -// #if LLVM_VERSION_GE(14, 0) -// AddressSanitizerOptions opts = AddressSanitizerOptions{ -// /*CompileKernel=*/false, -// SanitizerOptions->SanitizeAddressRecover, -// /*UseAfterScope=*/true, -// AsanDetectStackUseAfterReturnMode::Runtime, -// }; -// MPM.addPass(ModuleAddressSanitizerPass(opts)); -// #else -// MPM.addPass(ModuleAddressSanitizerPass( -// /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover)); -// MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( -// /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover, -// /*UseAfterScope=*/true))); -// #endif -// } -// ); -// } -// if (SanitizerOptions->SanitizeHWAddress) { -// OptimizerLastEPCallbacks.push_back( -// [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { -// #if LLVM_VERSION_GE(14, 0) -// HWAddressSanitizerOptions opts( -// /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, -// /*DisableOptimization=*/false); -// MPM.addPass(HWAddressSanitizerPass(opts)); -// #else -// MPM.addPass(HWAddressSanitizerPass( -// /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover)); -// #endif -// } -// ); -// } -// } - -// if (LLVMPluginsLen) { -// auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); -// SmallVector Plugins; -// PluginsStr.split(Plugins, ',', -1, false); -// for (auto PluginPath: Plugins) { -// auto Plugin = PassPlugin::Load(PluginPath.str()); -// if (!Plugin) { -// LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); -// continue; -// } -// Plugin->registerPassBuilderCallbacks(PB); -// } -// } - -// #if LLVM_VERSION_GE(13, 0) -// ModulePassManager MPM; -// #else -// ModulePassManager MPM(DebugPassManager); -// #endif -// bool NeedThinLTOBufferPasses = UseThinLTOBuffers; -// if (!NoPrepopulatePasses) { -// // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead. -// // At the same time, the LTO pipelines do support O0 and using them is required. -// bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; -// if (OptLevel == OptimizationLevel::O0 && !IsLTO) { -// for (const auto &C : PipelineStartEPCallbacks) -// PB.registerPipelineStartEPCallback(C); -// for (const auto &C : OptimizerLastEPCallbacks) -// PB.registerOptimizerLastEPCallback(C); - -// // Pass false as we manually schedule ThinLTOBufferPasses below. -// MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); -// } else { -// for (const auto &C : PipelineStartEPCallbacks) -// PB.registerPipelineStartEPCallback(C); -// if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { -// for (const auto &C : OptimizerLastEPCallbacks) -// PB.registerOptimizerLastEPCallback(C); -// } - -// switch (OptStage) { -// case LLVMRustOptStage::PreLinkNoLTO: -// MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); -// break; -// case LLVMRustOptStage::PreLinkThinLTO: -// MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); -// // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback -// // passes may still run afterwards. This means we need to run the buffer passes again. -// // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks -// // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks. -// if (OptimizerLastEPCallbacks.empty()) -// NeedThinLTOBufferPasses = false; -// for (const auto &C : OptimizerLastEPCallbacks) -// C(MPM, OptLevel); -// break; -// case LLVMRustOptStage::PreLinkFatLTO: -// MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); -// NeedThinLTOBufferPasses = false; -// break; -// case LLVMRustOptStage::ThinLTO: -// // FIXME: Does it make sense to pass the ModuleSummaryIndex? -// // It only seems to be needed for C++ specific optimizations. -// MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); -// break; -// case LLVMRustOptStage::FatLTO: -// MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); -// break; -// } -// } -// } else { -// // We're not building any of the default pipelines but we still want to -// // add the verifier, instrumentation, etc passes if they were requested -// for (const auto &C : PipelineStartEPCallbacks) -// C(MPM, OptLevel); -// for (const auto &C : OptimizerLastEPCallbacks) -// C(MPM, OptLevel); -// } - -// if (ExtraPassesLen) { -// if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { -// std::string ErrMsg = toString(std::move(Err)); -// LLVMRustSetLastError(ErrMsg.c_str()); -// return LLVMRustResult::Failure; -// } -// } - -// if (NeedThinLTOBufferPasses) { -// MPM.addPass(CanonicalizeAliasesPass()); -// MPM.addPass(NameAnonGlobalPass()); -// } - -// // Upgrade all calls to old intrinsics first. -// for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) -// UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove - -// MPM.run(*TheModule, MAM); -// return LLVMRustResult::Success; -// } - -// // Callback to demangle function name -// // Parameters: -// // * name to be demangled -// // * name len -// // * output buffer -// // * output buffer len -// // Returns len of demangled string, or 0 if demangle failed. -// typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); - - -// namespace { - -// class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { -// DemangleFn Demangle; -// std::vector Buf; - -// public: -// RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} - -// // Return empty string if demangle failed -// // or if name does not need to be demangled -// StringRef CallDemangle(StringRef name) { -// if (!Demangle) { -// return StringRef(); -// } - -// if (Buf.size() < name.size() * 2) { -// // Semangled name usually shorter than mangled, -// // but allocate twice as much memory just in case -// Buf.resize(name.size() * 2); -// } - -// auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); -// if (!R) { -// // Demangle failed. -// return StringRef(); -// } - -// auto Demangled = StringRef(Buf.data(), R); -// if (Demangled == name) { -// // Do not print anything if demangled name is equal to mangled. -// return StringRef(); -// } - -// return Demangled; -// } - -// void emitFunctionAnnot(const Function *F, -// formatted_raw_ostream &OS) override { -// StringRef Demangled = CallDemangle(F->getName()); -// if (Demangled.empty()) { -// return; -// } - -// OS << "; " << Demangled << "\n"; -// } - -// void emitInstructionAnnot(const Instruction *I, -// formatted_raw_ostream &OS) override { -// const char *Name; -// const Value *Value; -// if (const CallInst *CI = dyn_cast(I)) { -// Name = "call"; -// Value = CI->getCalledOperand(); -// } else if (const InvokeInst* II = dyn_cast(I)) { -// Name = "invoke"; -// Value = II->getCalledOperand(); -// } else { -// // Could demangle more operations, e. g. -// // `store %place, @function`. -// return; -// } - -// if (!Value->hasName()) { -// return; -// } - -// StringRef Demangled = CallDemangle(Value->getName()); -// if (Demangled.empty()) { -// return; -// } - -// OS << "; " << Name << " " << Demangled << "\n"; -// } -// }; - -// } // namespace - -// extern "C" LLVMRustResult -// LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { -// std::string ErrorInfo; -// std::error_code EC; -// raw_fd_ostream OS(Path, EC, sys::fs::OF_None); -// if (EC) -// ErrorInfo = EC.message(); -// if (ErrorInfo != "") { -// LLVMRustSetLastError(ErrorInfo.c_str()); -// return LLVMRustResult::Failure; -// } - -// RustAssemblyAnnotationWriter AAW(Demangle); -// formatted_raw_ostream FOS(OS); -// unwrap(M)->print(FOS, &AAW); - -// return LLVMRustResult::Success; -// } - -// extern "C" void LLVMRustPrintPasses() { -// LLVMInitializePasses(); -// struct MyListener : PassRegistrationListener { -// void passEnumerate(const PassInfo *Info) { -// StringRef PassArg = Info->getPassArgument(); -// StringRef PassName = Info->getPassName(); -// if (!PassArg.empty()) { -// // These unsigned->signed casts could theoretically overflow, but -// // realistically never will (and even if, the result is implementation -// // defined rather plain UB). -// printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), -// (int)PassName.size(), PassName.data()); -// } -// } -// } Listener; - -// PassRegistry *PR = PassRegistry::getPassRegistry(); -// PR->enumerateWith(&Listener); -// } - -// extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, -// bool AddLifetimes) { -// unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); -// } - -// extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, -// size_t Len) { -// llvm::legacy::PassManager passes; - -// auto PreserveFunctions = [=](const GlobalValue &GV) { -// for (size_t I = 0; I < Len; I++) { -// if (GV.getName() == Symbols[I]) { -// return true; -// } -// } -// return false; -// }; - -// passes.add(llvm::createInternalizePass(PreserveFunctions)); - -// passes.run(*unwrap(M)); -// } - -// extern "C" void -// LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, -// LLVMTargetMachineRef TMR) { -// TargetMachine *Target = unwrap(TMR); -// unwrap(Module)->setDataLayout(Target->createDataLayout()); -// } - -// extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { -// unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); -// } - -// extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { -// unwrap(M)->setPIELevel(PIELevel::Level::Large); -// } - -// extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, -// LLVMRustCodeModel Model) { -// auto CM = fromRust(Model); -// if (!CM.hasValue()) -// return; -// unwrap(M)->setCodeModel(*CM); -// } - -// // Here you'll find an implementation of ThinLTO as used by the Rust compiler -// // right now. This ThinLTO support is only enabled on "recent ish" versions of -// // LLVM, and otherwise it's just blanket rejected from other compilers. -// // -// // Most of this implementation is straight copied from LLVM. At the time of -// // this writing it wasn't *quite* suitable to reuse more code from upstream -// // for our purposes, but we should strive to upstream this support once it's -// // ready to go! I figure we may want a bit of testing locally first before -// // sending this upstream to LLVM. I hear though they're quite eager to receive -// // feedback like this! -// // -// // If you're reading this code and wondering "what in the world" or you're -// // working "good lord by LLVM upgrade is *still* failing due to these bindings" -// // then fear not! (ok maybe fear a little). All code here is mostly based -// // on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. -// // -// // You'll find that the general layout here roughly corresponds to the `run` -// // method in that file as well as `ProcessThinLTOModule`. Functions are -// // specifically commented below as well, but if you're updating this code -// // or otherwise trying to understand it, the LLVM source will be useful in -// // interpreting the mysteries within. -// // -// // Otherwise I'll apologize in advance, it probably requires a relatively -// // significant investment on your part to "truly understand" what's going on -// // here. Not saying I do myself, but it took me awhile staring at LLVM's source -// // and various online resources about ThinLTO to make heads or tails of all -// // this. - -// // This is a shared data structure which *must* be threadsafe to share -// // read-only amongst threads. This also corresponds basically to the arguments -// // of the `ProcessThinLTOModule` function in the LLVM source. -// struct LLVMRustThinLTOData { -// // The combined index that is the global analysis over all modules we're -// // performing ThinLTO for. This is mostly managed by LLVM. -// ModuleSummaryIndex Index; - -// // All modules we may look at, stored as in-memory serialized versions. This -// // is later used when inlining to ensure we can extract any module to inline -// // from. -// StringMap ModuleMap; - -// // A set that we manage of everything we *don't* want internalized. Note that -// // this includes all transitive references right now as well, but it may not -// // always! -// DenseSet GUIDPreservedSymbols; - -// // Not 100% sure what these are, but they impact what's internalized and -// // what's inlined across modules, I believe. -// StringMap ImportLists; -// StringMap ExportLists; -// StringMap ModuleToDefinedGVSummaries; -// StringMap> ResolvedODR; - -// LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} -// }; - -// // Just an argument to the `LLVMRustCreateThinLTOData` function below. -// struct LLVMRustThinLTOModule { -// const char *identifier; -// const char *data; -// size_t len; -// }; - -// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it -// // does. -// static const GlobalValueSummary * -// getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { -// auto StrongDefForLinker = llvm::find_if( -// GVSummaryList, [](const std::unique_ptr &Summary) { -// auto Linkage = Summary->linkage(); -// return !GlobalValue::isAvailableExternallyLinkage(Linkage) && -// !GlobalValue::isWeakForLinker(Linkage); -// }); -// if (StrongDefForLinker != GVSummaryList.end()) -// return StrongDefForLinker->get(); - -// auto FirstDefForLinker = llvm::find_if( -// GVSummaryList, [](const std::unique_ptr &Summary) { -// auto Linkage = Summary->linkage(); -// return !GlobalValue::isAvailableExternallyLinkage(Linkage); -// }); -// if (FirstDefForLinker == GVSummaryList.end()) -// return nullptr; -// return FirstDefForLinker->get(); -// } - -// // The main entry point for creating the global ThinLTO analysis. The structure -// // here is basically the same as before threads are spawned in the `run` -// // function of `lib/LTO/ThinLTOCodeGenerator.cpp`. -// extern "C" LLVMRustThinLTOData* -// LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, -// int num_modules, -// const char **preserved_symbols, -// int num_symbols) { -// auto Ret = std::make_unique(); - -// // Load each module's summary and merge it into one combined index -// for (int i = 0; i < num_modules; i++) { -// auto module = &modules[i]; -// StringRef buffer(module->data, module->len); -// MemoryBufferRef mem_buffer(buffer, module->identifier); - -// Ret->ModuleMap[module->identifier] = mem_buffer; - -// if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { -// LLVMRustSetLastError(toString(std::move(Err)).c_str()); -// return nullptr; -// } -// } - -// // Collect for each module the list of function it defines (GUID -> Summary) -// Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); - -// // Convert the preserved symbols set from string to GUID, this is then needed -// // for internalization. -// for (int i = 0; i < num_symbols; i++) { -// auto GUID = GlobalValue::getGUID(preserved_symbols[i]); -// Ret->GUIDPreservedSymbols.insert(GUID); -// } - -// // Collect the import/export lists for all modules from the call-graph in the -// // combined index -// // -// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` -// auto deadIsPrevailing = [&](GlobalValue::GUID G) { -// return PrevailingType::Unknown; -// }; -// // We don't have a complete picture in our use of ThinLTO, just our immediate -// // crate, so we need `ImportEnabled = false` to limit internalization. -// // Otherwise, we sometimes lose `static` values -- see #60184. -// computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, -// deadIsPrevailing, /* ImportEnabled = */ false); -// ComputeCrossModuleImport( -// Ret->Index, -// Ret->ModuleToDefinedGVSummaries, -// Ret->ImportLists, -// Ret->ExportLists -// ); - -// // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it -// // impacts the caching. -// // -// // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this -// // being lifted from `lib/LTO/LTO.cpp` as well -// DenseMap PrevailingCopy; -// for (auto &I : Ret->Index) { -// if (I.second.SummaryList.size() > 1) -// PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); -// } -// auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { -// const auto &Prevailing = PrevailingCopy.find(GUID); -// if (Prevailing == PrevailingCopy.end()) -// return true; -// return Prevailing->second == S; -// }; -// auto recordNewLinkage = [&](StringRef ModuleIdentifier, -// GlobalValue::GUID GUID, -// GlobalValue::LinkageTypes NewLinkage) { -// Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; -// }; - -// #if LLVM_VERSION_GE(13,0) -// // Uses FromPrevailing visibility scheme which works for many binary -// // formats. We probably could and should use ELF visibility scheme for many of -// // our targets, however. -// lto::Config conf; -// thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, -// Ret->GUIDPreservedSymbols); -// #else -// thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, -// Ret->GUIDPreservedSymbols); -// #endif -// // Here we calculate an `ExportedGUIDs` set for use in the `isExported` -// // callback below. This callback below will dictate the linkage for all -// // summaries in the index, and we basically just only want to ensure that dead -// // symbols are internalized. Otherwise everything that's already external -// // linkage will stay as external, and internal will stay as internal. -// std::set ExportedGUIDs; -// for (auto &List : Ret->Index) { -// for (auto &GVS: List.second.SummaryList) { -// if (GlobalValue::isLocalLinkage(GVS->linkage())) -// continue; -// auto GUID = GVS->getOriginalName(); -// if (GVS->flags().Live) -// ExportedGUIDs.insert(GUID); -// } -// } -// auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { -// const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); -// return (ExportList != Ret->ExportLists.end() && -// ExportList->second.count(VI)) || -// ExportedGUIDs.count(VI.getGUID()); -// }; -// thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); - -// return Ret.release(); -// } - -// extern "C" void -// LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { -// delete Data; -// } - -// // Below are the various passes that happen *per module* when doing ThinLTO. -// // -// // In other words, these are the functions that are all run concurrently -// // with one another, one per module. The passes here correspond to the analysis -// // passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the -// // `ProcessThinLTOModule` function. Here they're split up into separate steps -// // so rustc can save off the intermediate bytecode between each step. - -// static bool -// clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { -// // When linking an ELF shared object, dso_local should be dropped. We -// // conservatively do this for -fpic. -// bool ClearDSOLocalOnDeclarations = -// TM.getTargetTriple().isOSBinFormatELF() && -// TM.getRelocationModel() != Reloc::Static && -// Mod.getPIELevel() == PIELevel::Default; -// return ClearDSOLocalOnDeclarations; -// } - -// extern "C" bool -// LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, -// LLVMTargetMachineRef TM) { -// Module &Mod = *unwrap(M); -// TargetMachine &Target = *unwrap(TM); - -// bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); -// bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); - -// if (error) { -// LLVMRustSetLastError("renameModuleForThinLTO failed"); -// return false; -// } -// return true; -// } - -// extern "C" bool -// LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { -// Module &Mod = *unwrap(M); -// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); -// #if LLVM_VERSION_GE(14, 0) -// thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); -// #else -// thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); -// #endif -// return true; -// } - -// extern "C" bool -// LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { -// Module &Mod = *unwrap(M); -// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); -// thinLTOInternalizeModule(Mod, DefinedGlobals); -// return true; -// } - -// extern "C" bool -// LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, -// LLVMTargetMachineRef TM) { -// Module &Mod = *unwrap(M); -// TargetMachine &Target = *unwrap(TM); - -// const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); -// auto Loader = [&](StringRef Identifier) { -// const auto &Memory = Data->ModuleMap.lookup(Identifier); -// auto &Context = Mod.getContext(); -// auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); - -// if (!MOrErr) -// return MOrErr; - -// // The rest of this closure is a workaround for -// // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports -// // we accidentally import wasm custom sections into different modules, -// // duplicating them by in the final output artifact. -// // -// // The issue is worked around here by manually removing the -// // `wasm.custom_sections` named metadata node from any imported module. This -// // we know isn't used by any optimization pass so there's no need for it to -// // be imported. -// // -// // Note that the metadata is currently lazily loaded, so we materialize it -// // here before looking up if there's metadata inside. The `FunctionImporter` -// // will immediately materialize metadata anyway after an import, so this -// // shouldn't be a perf hit. -// if (Error Err = (*MOrErr)->materializeMetadata()) { -// Expected> Ret(std::move(Err)); -// return Ret; -// } - -// auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); -// if (WasmCustomSections) -// WasmCustomSections->eraseFromParent(); - -// return MOrErr; -// }; -// bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); -// FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal); -// Expected Result = Importer.importFunctions(Mod, ImportList); -// if (!Result) { -// LLVMRustSetLastError(toString(Result.takeError()).c_str()); -// return false; -// } -// return true; -// } - -// // This struct and various functions are sort of a hack right now, but the -// // problem is that we've got in-memory LLVM modules after we generate and -// // optimize all codegen-units for one compilation in rustc. To be compatible -// // with the LTO support above we need to serialize the modules plus their -// // ThinLTO summary into memory. -// // -// // This structure is basically an owned version of a serialize module, with -// // a ThinLTO summary attached. -// struct LLVMRustThinLTOBuffer { -// std::string data; -// }; - -// extern "C" LLVMRustThinLTOBuffer* -// LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { -// auto Ret = std::make_unique(); -// { -// raw_string_ostream OS(Ret->data); -// { -// legacy::PassManager PM; -// if (is_thin) { -// PM.add(createWriteThinLTOBitcodePass(OS)); -// } else { -// PM.add(createBitcodeWriterPass(OS)); -// } -// PM.run(*unwrap(M)); -// } -// } -// return Ret.release(); -// } - -// extern "C" void -// LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { -// delete Buffer; -// } - -// extern "C" const void* -// LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { -// return Buffer->data.data(); -// } - -// extern "C" size_t -// LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { -// return Buffer->data.length(); -// } - -// // This is what we used to parse upstream bitcode for actual ThinLTO -// // processing. We'll call this once per module optimized through ThinLTO, and -// // it'll be called concurrently on many threads. -// extern "C" LLVMModuleRef -// LLVMRustParseBitcodeForLTO(LLVMContextRef Context, -// const char *data, -// size_t len, -// const char *identifier) { -// StringRef Data(data, len); -// MemoryBufferRef Buffer(Data, identifier); -// unwrap(Context)->enableDebugTypeODRUniquing(); -// Expected> SrcOrError = -// parseBitcodeFile(Buffer, *unwrap(Context)); -// if (!SrcOrError) { -// LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); -// return nullptr; -// } -// return wrap(std::move(*SrcOrError).release()); -// } - -// // Find the bitcode section in the object file data and return it as a slice. -// // Fail if the bitcode section is present but empty. -// // -// // On success, the return value is the pointer to the start of the slice and -// // `out_len` is filled with the (non-zero) length. On failure, the return value -// // is `nullptr` and `out_len` is set to zero. -// extern "C" const char* -// LLVMRustGetBitcodeSliceFromObjectData(const char *data, -// size_t len, -// size_t *out_len) { -// *out_len = 0; - -// StringRef Data(data, len); -// MemoryBufferRef Buffer(Data, ""); // The id is unused. - -// Expected BitcodeOrError = -// object::IRObjectFile::findBitcodeInMemBuffer(Buffer); -// if (!BitcodeOrError) { -// LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str()); -// return nullptr; -// } - -// *out_len = BitcodeOrError->getBufferSize(); -// return BitcodeOrError->getBufferStart(); -// } - -// // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See -// // the comment in `back/lto.rs` for why this exists. -// extern "C" void -// LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, -// DICompileUnit **A, -// DICompileUnit **B) { -// Module *M = unwrap(Mod); -// DICompileUnit **Cur = A; -// DICompileUnit **Next = B; -// for (DICompileUnit *CU : M->debug_compile_units()) { -// *Cur = CU; -// Cur = Next; -// Next = nullptr; -// if (Cur == nullptr) -// break; -// } -// } - -// // Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See -// // the comment in `back/lto.rs` for why this exists. -// extern "C" void -// LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { -// Module *M = unwrap(Mod); - -// // If the original source module didn't have a `DICompileUnit` then try to -// // merge all the existing compile units. If there aren't actually any though -// // then there's not much for us to do so return. -// if (Unit == nullptr) { -// for (DICompileUnit *CU : M->debug_compile_units()) { -// Unit = CU; -// break; -// } -// if (Unit == nullptr) -// return; -// } - -// // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and -// // process it recursively. Note that we used to specifically iterate over -// // instructions to ensure we feed everything into it, but `processModule` -// // started doing this the same way in LLVM 7 (commit d769eb36ab2b8). -// DebugInfoFinder Finder; -// Finder.processModule(*M); - -// // After we've found all our debuginfo, rewrite all subprograms to point to -// // the same `DICompileUnit`. -// for (auto &F : Finder.subprograms()) { -// F->replaceUnit(Unit); -// } - -// // Erase any other references to other `DICompileUnit` instances, the verifier -// // will later ensure that we don't actually have any other stale references to -// // worry about. -// auto *MD = M->getNamedMetadata("llvm.dbg.cu"); -// MD->clearOperands(); -// MD->addOperand(Unit); -// } - -// // Computes the LTO cache key for the provided 'ModId' in the given 'Data', -// // storing the result in 'KeyOut'. -// // Currently, this cache key is a SHA-1 hash of anything that could affect -// // the result of optimizing this module (e.g. module imports, exports, liveness -// // of access globals, etc). -// // The precise details are determined by LLVM in `computeLTOCacheKey`, which is -// // used during the normal linker-plugin incremental thin-LTO process. -// extern "C" void -// LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) { -// SmallString<40> Key; -// llvm::lto::Config conf; -// const auto &ImportList = Data->ImportLists.lookup(ModId); -// const auto &ExportList = Data->ExportLists.lookup(ModId); -// const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); -// const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); -// std::set CfiFunctionDefs; -// std::set CfiFunctionDecls; - -// // Based on the 'InProcessThinBackend' constructor in LLVM -// for (auto &Name : Data->Index.cfiFunctionDefs()) -// CfiFunctionDefs.insert( -// GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); -// for (auto &Name : Data->Index.cfiFunctionDecls()) -// CfiFunctionDecls.insert( -// GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); - -// llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, -// ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls -// ); - -// LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); -// } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp deleted file mode 100644 index 56886275a7b..00000000000 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ /dev/null @@ -1,1972 +0,0 @@ -// #include "LLVMWrapper.h" -// #include "llvm/IR/DebugInfoMetadata.h" -// #include "llvm/IR/DiagnosticHandler.h" -// #include "llvm/IR/DiagnosticInfo.h" -// #include "llvm/IR/DiagnosticPrinter.h" -// #include "llvm/IR/GlobalVariable.h" -// #include "llvm/IR/Instructions.h" -// #include "llvm/IR/Intrinsics.h" -// #include "llvm/IR/IntrinsicsARM.h" -// #include "llvm/IR/Mangler.h" -// #include "llvm/Object/Archive.h" -// #include "llvm/Object/COFFImportFile.h" -// #include "llvm/Object/ObjectFile.h" -// #include "llvm/Pass.h" -// #include "llvm/Bitcode/BitcodeWriterPass.h" -// #include "llvm/Support/Signals.h" -// #include "llvm/ADT/Optional.h" - -// #include - -// //===----------------------------------------------------------------------=== -// // -// // This file defines alternate interfaces to core functions that are more -// // readily callable by Rust's FFI. -// // -// //===----------------------------------------------------------------------=== - -// using namespace llvm; -// using namespace llvm::sys; -// using namespace llvm::object; - -// // LLVMAtomicOrdering is already an enum - don't create another -// // one. -// static AtomicOrdering fromRust(LLVMAtomicOrdering Ordering) { -// switch (Ordering) { -// case LLVMAtomicOrderingNotAtomic: -// return AtomicOrdering::NotAtomic; -// case LLVMAtomicOrderingUnordered: -// return AtomicOrdering::Unordered; -// case LLVMAtomicOrderingMonotonic: -// return AtomicOrdering::Monotonic; -// case LLVMAtomicOrderingAcquire: -// return AtomicOrdering::Acquire; -// case LLVMAtomicOrderingRelease: -// return AtomicOrdering::Release; -// case LLVMAtomicOrderingAcquireRelease: -// return AtomicOrdering::AcquireRelease; -// case LLVMAtomicOrderingSequentiallyConsistent: -// return AtomicOrdering::SequentiallyConsistent; -// } - -// report_fatal_error("Invalid LLVMAtomicOrdering value!"); -// } - -// static LLVM_THREAD_LOCAL char *LastError; - -// // Custom error handler for fatal LLVM errors. -// // -// // Notably it exits the process with code 101, unlike LLVM's default of 1. -// static void FatalErrorHandler(void *UserData, -// #if LLVM_VERSION_LT(14, 0) -// const std::string& Reason, -// #else -// const char* Reason, -// #endif -// bool GenCrashDiag) { -// // Do the same thing that the default error handler does. -// std::cerr << "LLVM ERROR: " << Reason << std::endl; - -// // Since this error handler exits the process, we have to run any cleanup that -// // LLVM would run after handling the error. This might change with an LLVM -// // upgrade. -// sys::RunInterruptHandlers(); - -// exit(101); -// } - -// extern "C" void LLVMRustInstallFatalErrorHandler() { -// install_fatal_error_handler(FatalErrorHandler); -// } - -// extern "C" void LLVMRustDisableSystemDialogsOnCrash() { -// sys::DisableSystemDialogsOnCrash(); -// } - -// extern "C" char *LLVMRustGetLastError(void) { -// char *Ret = LastError; -// LastError = nullptr; -// return Ret; -// } - -// extern "C" void LLVMRustSetLastError(const char *Err) { -// free((void *)LastError); -// LastError = strdup(Err); -// } - -// extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames) { -// auto ctx = new LLVMContext(); -// ctx->setDiscardValueNames(shouldDiscardNames); -// return wrap(ctx); -// } - -// extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, -// const char *Triple) { -// unwrap(M)->setTargetTriple(Triple::normalize(Triple)); -// } - -// extern "C" void LLVMRustPrintPassTimings() { -// raw_fd_ostream OS(2, false); // stderr. -// TimerGroup::printAll(OS); -// } - -// extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, -// size_t NameLen) { -// return wrap(unwrap(M)->getNamedValue(StringRef(Name, NameLen))); -// } - -// extern "C" LLVMValueRef LLVMRustGetOrInsertFunction(LLVMModuleRef M, -// const char *Name, -// size_t NameLen, -// LLVMTypeRef FunctionTy) { -// return wrap(unwrap(M) -// ->getOrInsertFunction(StringRef(Name, NameLen), -// unwrap(FunctionTy)) -// .getCallee() -// ); -// } - -// extern "C" LLVMValueRef -// LLVMRustGetOrInsertGlobal(LLVMModuleRef M, const char *Name, size_t NameLen, LLVMTypeRef Ty) { -// Module *Mod = unwrap(M); -// StringRef NameRef(Name, NameLen); - -// // We don't use Module::getOrInsertGlobal because that returns a Constant*, -// // which may either be the real GlobalVariable*, or a constant bitcast of it -// // if our type doesn't match the original declaration. We always want the -// // GlobalVariable* so we can access linkage, visibility, etc. -// GlobalVariable *GV = Mod->getGlobalVariable(NameRef, true); -// if (!GV) -// GV = new GlobalVariable(*Mod, unwrap(Ty), false, -// GlobalValue::ExternalLinkage, nullptr, NameRef); -// return wrap(GV); -// } - -// extern "C" LLVMValueRef -// LLVMRustInsertPrivateGlobal(LLVMModuleRef M, LLVMTypeRef Ty) { -// return wrap(new GlobalVariable(*unwrap(M), -// unwrap(Ty), -// false, -// GlobalValue::PrivateLinkage, -// nullptr)); -// } - -// extern "C" LLVMTypeRef LLVMRustMetadataTypeInContext(LLVMContextRef C) { -// return wrap(Type::getMetadataTy(*unwrap(C))); -// } - -// static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { -// switch (Kind) { -// case AlwaysInline: -// return Attribute::AlwaysInline; -// case ByVal: -// return Attribute::ByVal; -// case Cold: -// return Attribute::Cold; -// case InlineHint: -// return Attribute::InlineHint; -// case MinSize: -// return Attribute::MinSize; -// case Naked: -// return Attribute::Naked; -// case NoAlias: -// return Attribute::NoAlias; -// case NoCapture: -// return Attribute::NoCapture; -// case NoCfCheck: -// return Attribute::NoCfCheck; -// case NoInline: -// return Attribute::NoInline; -// case NonNull: -// return Attribute::NonNull; -// case NoRedZone: -// return Attribute::NoRedZone; -// case NoReturn: -// return Attribute::NoReturn; -// case NoUnwind: -// return Attribute::NoUnwind; -// case OptimizeForSize: -// return Attribute::OptimizeForSize; -// case ReadOnly: -// return Attribute::ReadOnly; -// case SExt: -// return Attribute::SExt; -// case StructRet: -// return Attribute::StructRet; -// case UWTable: -// return Attribute::UWTable; -// case ZExt: -// return Attribute::ZExt; -// case InReg: -// return Attribute::InReg; -// case SanitizeThread: -// return Attribute::SanitizeThread; -// case SanitizeAddress: -// return Attribute::SanitizeAddress; -// case SanitizeMemory: -// return Attribute::SanitizeMemory; -// case NonLazyBind: -// return Attribute::NonLazyBind; -// case OptimizeNone: -// return Attribute::OptimizeNone; -// case ReturnsTwice: -// return Attribute::ReturnsTwice; -// case ReadNone: -// return Attribute::ReadNone; -// case InaccessibleMemOnly: -// return Attribute::InaccessibleMemOnly; -// case SanitizeHWAddress: -// return Attribute::SanitizeHWAddress; -// case WillReturn: -// return Attribute::WillReturn; -// case StackProtectReq: -// return Attribute::StackProtectReq; -// case StackProtectStrong: -// return Attribute::StackProtectStrong; -// case StackProtect: -// return Attribute::StackProtect; -// case NoUndef: -// return Attribute::NoUndef; -// case SanitizeMemTag: -// return Attribute::SanitizeMemTag; -// case ShadowCallStack: -// return Attribute::ShadowCallStack; -// case AllocSize: -// return Attribute::AllocSize; -// #if LLVM_VERSION_GE(15, 0) -// case AllocatedPointer: -// return Attribute::AllocatedPointer; -// case AllocAlign: -// return Attribute::AllocAlign; -// #endif -// } -// report_fatal_error("bad AttributeKind"); -// } - -// template static inline void AddAttributes(T *t, unsigned Index, -// LLVMAttributeRef *Attrs, size_t AttrsLen) { -// AttributeList PAL = t->getAttributes(); -// AttributeList PALNew; -// #if LLVM_VERSION_LT(14, 0) -// AttrBuilder B; -// for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen)) -// B.addAttribute(unwrap(Attr)); -// PALNew = PAL.addAttributes(t->getContext(), Index, B); -// #else -// AttrBuilder B(t->getContext()); -// for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen)) -// B.addAttribute(unwrap(Attr)); -// PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B); -// #endif -// t->setAttributes(PALNew); -// } - -// extern "C" void LLVMRustAddFunctionAttributes(LLVMValueRef Fn, unsigned Index, -// LLVMAttributeRef *Attrs, size_t AttrsLen) { -// Function *F = unwrap(Fn); -// AddAttributes(F, Index, Attrs, AttrsLen); -// } - -// extern "C" void LLVMRustAddCallSiteAttributes(LLVMValueRef Instr, unsigned Index, -// LLVMAttributeRef *Attrs, size_t AttrsLen) { -// CallBase *Call = unwrap(Instr); -// AddAttributes(Call, Index, Attrs, AttrsLen); -// } - -// extern "C" LLVMAttributeRef LLVMRustCreateAttrNoValue(LLVMContextRef C, -// LLVMRustAttribute RustAttr) { -// return wrap(Attribute::get(*unwrap(C), fromRust(RustAttr))); -// } - -// extern "C" LLVMAttributeRef LLVMRustCreateAlignmentAttr(LLVMContextRef C, -// uint64_t Bytes) { -// return wrap(Attribute::getWithAlignment(*unwrap(C), llvm::Align(Bytes))); -// } - -// extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableAttr(LLVMContextRef C, -// uint64_t Bytes) { -// return wrap(Attribute::getWithDereferenceableBytes(*unwrap(C), Bytes)); -// } - -// extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableOrNullAttr(LLVMContextRef C, -// uint64_t Bytes) { -// return wrap(Attribute::getWithDereferenceableOrNullBytes(*unwrap(C), Bytes)); -// } - -// extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C, LLVMTypeRef Ty) { -// return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty))); -// } - -// extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C, LLVMTypeRef Ty) { -// return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty))); -// } - -// extern "C" LLVMAttributeRef LLVMRustCreateElementTypeAttr(LLVMContextRef C, LLVMTypeRef Ty) { -// #if LLVM_VERSION_GE(15, 0) -// return wrap(Attribute::get(*unwrap(C), Attribute::ElementType, unwrap(Ty))); -// #else -// report_fatal_error("Should not be needed on LLVM < 15"); -// #endif -// } - -// extern "C" LLVMAttributeRef LLVMRustCreateUWTableAttr(LLVMContextRef C, bool Async) { -// #if LLVM_VERSION_LT(15, 0) -// return wrap(Attribute::get(*unwrap(C), Attribute::UWTable)); -// #else -// return wrap(Attribute::getWithUWTableKind( -// *unwrap(C), Async ? UWTableKind::Async : UWTableKind::Sync)); -// #endif -// } - -// extern "C" LLVMAttributeRef LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) { -// return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, None)); -// } - -// #if LLVM_VERSION_GE(15, 0) - -// // These values **must** match ffi::AllocKindFlags. -// // It _happens_ to match the LLVM values of llvm::AllocFnKind, -// // but that's happenstance and we do explicit conversions before -// // passing them to LLVM. -// enum class LLVMRustAllocKindFlags : uint64_t { -// Unknown = 0, -// Alloc = 1, -// Realloc = 1 << 1, -// Free = 1 << 2, -// Uninitialized = 1 << 3, -// Zeroed = 1 << 4, -// Aligned = 1 << 5, -// }; - -// static LLVMRustAllocKindFlags operator&(LLVMRustAllocKindFlags A, LLVMRustAllocKindFlags B) { -// return static_cast(static_cast(A) & -// static_cast(B)); -// } - -// static bool isSet(LLVMRustAllocKindFlags F) { return F != LLVMRustAllocKindFlags::Unknown; } - -// static llvm::AllocFnKind allocKindFromRust(LLVMRustAllocKindFlags F) { -// llvm::AllocFnKind AFK = llvm::AllocFnKind::Unknown; -// if (isSet(F & LLVMRustAllocKindFlags::Alloc)) { -// AFK |= llvm::AllocFnKind::Alloc; -// } -// if (isSet(F & LLVMRustAllocKindFlags::Realloc)) { -// AFK |= llvm::AllocFnKind::Realloc; -// } -// if (isSet(F & LLVMRustAllocKindFlags::Free)) { -// AFK |= llvm::AllocFnKind::Free; -// } -// if (isSet(F & LLVMRustAllocKindFlags::Uninitialized)) { -// AFK |= llvm::AllocFnKind::Uninitialized; -// } -// if (isSet(F & LLVMRustAllocKindFlags::Zeroed)) { -// AFK |= llvm::AllocFnKind::Zeroed; -// } -// if (isSet(F & LLVMRustAllocKindFlags::Aligned)) { -// AFK |= llvm::AllocFnKind::Aligned; -// } -// return AFK; -// } -// #endif - -// extern "C" LLVMAttributeRef LLVMRustCreateAllocKindAttr(LLVMContextRef C, uint64_t AllocKindArg) { -// #if LLVM_VERSION_GE(15, 0) -// return wrap(Attribute::get(*unwrap(C), Attribute::AllocKind, -// static_cast(allocKindFromRust(static_cast(AllocKindArg))))); -// #else -// report_fatal_error( -// "allockind attributes are new in LLVM 15 and should not be used on older LLVMs"); -// #endif -// } - -// // Enable a fast-math flag -// // -// // https://llvm.org/docs/LangRef.html#fast-math-flags -// extern "C" void LLVMRustSetFastMath(LLVMValueRef V) { -// if (auto I = dyn_cast(unwrap(V))) { -// I->setFast(true); -// } -// } - -// extern "C" LLVMValueRef -// LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source, -// const char *Name, LLVMAtomicOrdering Order) { -// Value *Ptr = unwrap(Source); -// LoadInst *LI = unwrap(B)->CreateLoad(unwrap(Ty), Ptr, Name); -// LI->setAtomic(fromRust(Order)); -// return wrap(LI); -// } - -// extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B, -// LLVMValueRef V, -// LLVMValueRef Target, -// LLVMAtomicOrdering Order) { -// StoreInst *SI = unwrap(B)->CreateStore(unwrap(V), unwrap(Target)); -// SI->setAtomic(fromRust(Order)); -// return wrap(SI); -// } - -// // FIXME: Use the C-API LLVMBuildAtomicCmpXchg and LLVMSetWeak -// // once we raise our minimum support to LLVM 10. -// extern "C" LLVMValueRef -// LLVMRustBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Target, -// LLVMValueRef Old, LLVMValueRef Source, -// LLVMAtomicOrdering Order, -// LLVMAtomicOrdering FailureOrder, LLVMBool Weak) { -// #if LLVM_VERSION_GE(13,0) -// // Rust probably knows the alignment of the target value and should be able to -// // specify something more precise than MaybeAlign here. See also -// // https://reviews.llvm.org/D97224 which may be a useful reference. -// AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg( -// unwrap(Target), unwrap(Old), unwrap(Source), llvm::MaybeAlign(), fromRust(Order), -// fromRust(FailureOrder)); -// #else -// AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg( -// unwrap(Target), unwrap(Old), unwrap(Source), fromRust(Order), -// fromRust(FailureOrder)); -// #endif -// ACXI->setWeak(Weak); -// return wrap(ACXI); -// } - -// enum class LLVMRustSynchronizationScope { -// SingleThread, -// CrossThread, -// }; - -// static SyncScope::ID fromRust(LLVMRustSynchronizationScope Scope) { -// switch (Scope) { -// case LLVMRustSynchronizationScope::SingleThread: -// return SyncScope::SingleThread; -// case LLVMRustSynchronizationScope::CrossThread: -// return SyncScope::System; -// default: -// report_fatal_error("bad SynchronizationScope."); -// } -// } - -// extern "C" LLVMValueRef -// LLVMRustBuildAtomicFence(LLVMBuilderRef B, LLVMAtomicOrdering Order, -// LLVMRustSynchronizationScope Scope) { -// return wrap(unwrap(B)->CreateFence(fromRust(Order), fromRust(Scope))); -// } - -// enum class LLVMRustAsmDialect { -// Att, -// Intel, -// }; - -// static InlineAsm::AsmDialect fromRust(LLVMRustAsmDialect Dialect) { -// switch (Dialect) { -// case LLVMRustAsmDialect::Att: -// return InlineAsm::AD_ATT; -// case LLVMRustAsmDialect::Intel: -// return InlineAsm::AD_Intel; -// default: -// report_fatal_error("bad AsmDialect."); -// } -// } - -// extern "C" LLVMValueRef -// LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen, -// char *Constraints, size_t ConstraintsLen, -// LLVMBool HasSideEffects, LLVMBool IsAlignStack, -// LLVMRustAsmDialect Dialect, LLVMBool CanThrow) { -// #if LLVM_VERSION_GE(13, 0) -// return wrap(InlineAsm::get(unwrap(Ty), -// StringRef(AsmString, AsmStringLen), -// StringRef(Constraints, ConstraintsLen), -// HasSideEffects, IsAlignStack, -// fromRust(Dialect), CanThrow)); -// #else -// return wrap(InlineAsm::get(unwrap(Ty), -// StringRef(AsmString, AsmStringLen), -// StringRef(Constraints, ConstraintsLen), -// HasSideEffects, IsAlignStack, -// fromRust(Dialect))); -// #endif -// } - -// extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints, -// size_t ConstraintsLen) { -// #if LLVM_VERSION_LT(15, 0) -// return InlineAsm::Verify(unwrap(Ty), -// StringRef(Constraints, ConstraintsLen)); -// #else -// // llvm::Error converts to true if it is an error. -// return !llvm::errorToBool(InlineAsm::verify( -// unwrap(Ty), StringRef(Constraints, ConstraintsLen))); -// #endif -// } - -// extern "C" void LLVMRustAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm, -// size_t AsmLen) { -// unwrap(M)->appendModuleInlineAsm(StringRef(Asm, AsmLen)); -// } - -// typedef DIBuilder *LLVMRustDIBuilderRef; - -// template DIT *unwrapDIPtr(LLVMMetadataRef Ref) { -// return (DIT *)(Ref ? unwrap(Ref) : nullptr); -// } - -// #define DIDescriptor DIScope -// #define DIArray DINodeArray -// #define unwrapDI unwrapDIPtr - -// // These values **must** match debuginfo::DIFlags! They also *happen* -// // to match LLVM, but that isn't required as we do giant sets of -// // matching below. The value shouldn't be directly passed to LLVM. -// enum class LLVMRustDIFlags : uint32_t { -// FlagZero = 0, -// FlagPrivate = 1, -// FlagProtected = 2, -// FlagPublic = 3, -// FlagFwdDecl = (1 << 2), -// FlagAppleBlock = (1 << 3), -// FlagBlockByrefStruct = (1 << 4), -// FlagVirtual = (1 << 5), -// FlagArtificial = (1 << 6), -// FlagExplicit = (1 << 7), -// FlagPrototyped = (1 << 8), -// FlagObjcClassComplete = (1 << 9), -// FlagObjectPointer = (1 << 10), -// FlagVector = (1 << 11), -// FlagStaticMember = (1 << 12), -// FlagLValueReference = (1 << 13), -// FlagRValueReference = (1 << 14), -// FlagExternalTypeRef = (1 << 15), -// FlagIntroducedVirtual = (1 << 18), -// FlagBitField = (1 << 19), -// FlagNoReturn = (1 << 20), -// // Do not add values that are not supported by the minimum LLVM -// // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def -// }; - -// inline LLVMRustDIFlags operator&(LLVMRustDIFlags A, LLVMRustDIFlags B) { -// return static_cast(static_cast(A) & -// static_cast(B)); -// } - -// inline LLVMRustDIFlags operator|(LLVMRustDIFlags A, LLVMRustDIFlags B) { -// return static_cast(static_cast(A) | -// static_cast(B)); -// } - -// inline LLVMRustDIFlags &operator|=(LLVMRustDIFlags &A, LLVMRustDIFlags B) { -// return A = A | B; -// } - -// inline bool isSet(LLVMRustDIFlags F) { return F != LLVMRustDIFlags::FlagZero; } - -// inline LLVMRustDIFlags visibility(LLVMRustDIFlags F) { -// return static_cast(static_cast(F) & 0x3); -// } - -// static DINode::DIFlags fromRust(LLVMRustDIFlags Flags) { -// DINode::DIFlags Result = DINode::DIFlags::FlagZero; - -// switch (visibility(Flags)) { -// case LLVMRustDIFlags::FlagPrivate: -// Result |= DINode::DIFlags::FlagPrivate; -// break; -// case LLVMRustDIFlags::FlagProtected: -// Result |= DINode::DIFlags::FlagProtected; -// break; -// case LLVMRustDIFlags::FlagPublic: -// Result |= DINode::DIFlags::FlagPublic; -// break; -// default: -// // The rest are handled below -// break; -// } - -// if (isSet(Flags & LLVMRustDIFlags::FlagFwdDecl)) { -// Result |= DINode::DIFlags::FlagFwdDecl; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagAppleBlock)) { -// Result |= DINode::DIFlags::FlagAppleBlock; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagVirtual)) { -// Result |= DINode::DIFlags::FlagVirtual; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagArtificial)) { -// Result |= DINode::DIFlags::FlagArtificial; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagExplicit)) { -// Result |= DINode::DIFlags::FlagExplicit; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagPrototyped)) { -// Result |= DINode::DIFlags::FlagPrototyped; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagObjcClassComplete)) { -// Result |= DINode::DIFlags::FlagObjcClassComplete; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagObjectPointer)) { -// Result |= DINode::DIFlags::FlagObjectPointer; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagVector)) { -// Result |= DINode::DIFlags::FlagVector; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagStaticMember)) { -// Result |= DINode::DIFlags::FlagStaticMember; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagLValueReference)) { -// Result |= DINode::DIFlags::FlagLValueReference; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagRValueReference)) { -// Result |= DINode::DIFlags::FlagRValueReference; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagIntroducedVirtual)) { -// Result |= DINode::DIFlags::FlagIntroducedVirtual; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagBitField)) { -// Result |= DINode::DIFlags::FlagBitField; -// } -// if (isSet(Flags & LLVMRustDIFlags::FlagNoReturn)) { -// Result |= DINode::DIFlags::FlagNoReturn; -// } - -// return Result; -// } - -// // These values **must** match debuginfo::DISPFlags! They also *happen* -// // to match LLVM, but that isn't required as we do giant sets of -// // matching below. The value shouldn't be directly passed to LLVM. -// enum class LLVMRustDISPFlags : uint32_t { -// SPFlagZero = 0, -// SPFlagVirtual = 1, -// SPFlagPureVirtual = 2, -// SPFlagLocalToUnit = (1 << 2), -// SPFlagDefinition = (1 << 3), -// SPFlagOptimized = (1 << 4), -// SPFlagMainSubprogram = (1 << 5), -// // Do not add values that are not supported by the minimum LLVM -// // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def -// // (In LLVM < 8, createFunction supported these as separate bool arguments.) -// }; - -// inline LLVMRustDISPFlags operator&(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { -// return static_cast(static_cast(A) & -// static_cast(B)); -// } - -// inline LLVMRustDISPFlags operator|(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { -// return static_cast(static_cast(A) | -// static_cast(B)); -// } - -// inline LLVMRustDISPFlags &operator|=(LLVMRustDISPFlags &A, LLVMRustDISPFlags B) { -// return A = A | B; -// } - -// inline bool isSet(LLVMRustDISPFlags F) { return F != LLVMRustDISPFlags::SPFlagZero; } - -// inline LLVMRustDISPFlags virtuality(LLVMRustDISPFlags F) { -// return static_cast(static_cast(F) & 0x3); -// } - -// static DISubprogram::DISPFlags fromRust(LLVMRustDISPFlags SPFlags) { -// DISubprogram::DISPFlags Result = DISubprogram::DISPFlags::SPFlagZero; - -// switch (virtuality(SPFlags)) { -// case LLVMRustDISPFlags::SPFlagVirtual: -// Result |= DISubprogram::DISPFlags::SPFlagVirtual; -// break; -// case LLVMRustDISPFlags::SPFlagPureVirtual: -// Result |= DISubprogram::DISPFlags::SPFlagPureVirtual; -// break; -// default: -// // The rest are handled below -// break; -// } - -// if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagLocalToUnit)) { -// Result |= DISubprogram::DISPFlags::SPFlagLocalToUnit; -// } -// if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagDefinition)) { -// Result |= DISubprogram::DISPFlags::SPFlagDefinition; -// } -// if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagOptimized)) { -// Result |= DISubprogram::DISPFlags::SPFlagOptimized; -// } -// if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagMainSubprogram)) { -// Result |= DISubprogram::DISPFlags::SPFlagMainSubprogram; -// } - -// return Result; -// } - -// enum class LLVMRustDebugEmissionKind { -// NoDebug, -// FullDebug, -// LineTablesOnly, -// }; - -// static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind) { -// switch (Kind) { -// case LLVMRustDebugEmissionKind::NoDebug: -// return DICompileUnit::DebugEmissionKind::NoDebug; -// case LLVMRustDebugEmissionKind::FullDebug: -// return DICompileUnit::DebugEmissionKind::FullDebug; -// case LLVMRustDebugEmissionKind::LineTablesOnly: -// return DICompileUnit::DebugEmissionKind::LineTablesOnly; -// default: -// report_fatal_error("bad DebugEmissionKind."); -// } -// } - -// enum class LLVMRustChecksumKind { -// None, -// MD5, -// SHA1, -// SHA256, -// }; - -// static Optional fromRust(LLVMRustChecksumKind Kind) { -// switch (Kind) { -// case LLVMRustChecksumKind::None: -// return None; -// case LLVMRustChecksumKind::MD5: -// return DIFile::ChecksumKind::CSK_MD5; -// case LLVMRustChecksumKind::SHA1: -// return DIFile::ChecksumKind::CSK_SHA1; -// case LLVMRustChecksumKind::SHA256: -// return DIFile::ChecksumKind::CSK_SHA256; -// default: -// report_fatal_error("bad ChecksumKind."); -// } -// } - -// extern "C" uint32_t LLVMRustDebugMetadataVersion() { -// return DEBUG_METADATA_VERSION; -// } - -// extern "C" uint32_t LLVMRustVersionPatch() { return LLVM_VERSION_PATCH; } - -// extern "C" uint32_t LLVMRustVersionMinor() { return LLVM_VERSION_MINOR; } - -// extern "C" uint32_t LLVMRustVersionMajor() { return LLVM_VERSION_MAJOR; } - -// extern "C" void LLVMRustAddModuleFlag( -// LLVMModuleRef M, -// Module::ModFlagBehavior MergeBehavior, -// const char *Name, -// uint32_t Value) { -// unwrap(M)->addModuleFlag(MergeBehavior, Name, Value); -// } - -// extern "C" bool LLVMRustHasModuleFlag(LLVMModuleRef M, const char *Name, -// size_t Len) { -// return unwrap(M)->getModuleFlag(StringRef(Name, Len)) != nullptr; -// } - -// extern "C" LLVMValueRef LLVMRustMetadataAsValue(LLVMContextRef C, LLVMMetadataRef MD) { -// return wrap(MetadataAsValue::get(*unwrap(C), unwrap(MD))); -// } - -// extern "C" void LLVMRustGlobalAddMetadata( -// LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) { -// unwrap(Global)->addMetadata(Kind, *unwrap(MD)); -// } - -// extern "C" LLVMRustDIBuilderRef LLVMRustDIBuilderCreate(LLVMModuleRef M) { -// return new DIBuilder(*unwrap(M)); -// } - -// extern "C" void LLVMRustDIBuilderDispose(LLVMRustDIBuilderRef Builder) { -// delete Builder; -// } - -// extern "C" void LLVMRustDIBuilderFinalize(LLVMRustDIBuilderRef Builder) { -// Builder->finalize(); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateCompileUnit( -// LLVMRustDIBuilderRef Builder, unsigned Lang, LLVMMetadataRef FileRef, -// const char *Producer, size_t ProducerLen, bool isOptimized, -// const char *Flags, unsigned RuntimeVer, -// const char *SplitName, size_t SplitNameLen, -// LLVMRustDebugEmissionKind Kind, -// uint64_t DWOId, bool SplitDebugInlining) { -// auto *File = unwrapDI(FileRef); - -// return wrap(Builder->createCompileUnit(Lang, File, StringRef(Producer, ProducerLen), -// isOptimized, Flags, RuntimeVer, -// StringRef(SplitName, SplitNameLen), -// fromRust(Kind), DWOId, SplitDebugInlining)); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFile( -// LLVMRustDIBuilderRef Builder, -// const char *Filename, size_t FilenameLen, -// const char *Directory, size_t DirectoryLen, LLVMRustChecksumKind CSKind, -// const char *Checksum, size_t ChecksumLen) { -// Optional llvmCSKind = fromRust(CSKind); -// Optional> CSInfo{}; -// if (llvmCSKind) -// CSInfo.emplace(*llvmCSKind, StringRef{Checksum, ChecksumLen}); -// return wrap(Builder->createFile(StringRef(Filename, FilenameLen), -// StringRef(Directory, DirectoryLen), -// CSInfo)); -// } - -// extern "C" LLVMMetadataRef -// LLVMRustDIBuilderCreateSubroutineType(LLVMRustDIBuilderRef Builder, -// LLVMMetadataRef ParameterTypes) { -// return wrap(Builder->createSubroutineType( -// DITypeRefArray(unwrap(ParameterTypes)))); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, -// const char *Name, size_t NameLen, -// const char *LinkageName, size_t LinkageNameLen, -// LLVMMetadataRef File, unsigned LineNo, -// LLVMMetadataRef Ty, unsigned ScopeLine, LLVMRustDIFlags Flags, -// LLVMRustDISPFlags SPFlags, LLVMValueRef MaybeFn, LLVMMetadataRef TParam, -// LLVMMetadataRef Decl) { -// DITemplateParameterArray TParams = -// DITemplateParameterArray(unwrap(TParam)); -// DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); -// DINode::DIFlags llvmFlags = fromRust(Flags); -// DISubprogram *Sub = Builder->createFunction( -// unwrapDI(Scope), -// StringRef(Name, NameLen), -// StringRef(LinkageName, LinkageNameLen), -// unwrapDI(File), LineNo, -// unwrapDI(Ty), ScopeLine, llvmFlags, -// llvmSPFlags, TParams, unwrapDIPtr(Decl)); -// if (MaybeFn) -// unwrap(MaybeFn)->setSubprogram(Sub); -// return wrap(Sub); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateBasicType( -// LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, -// uint64_t SizeInBits, unsigned Encoding) { -// return wrap(Builder->createBasicType(StringRef(Name, NameLen), SizeInBits, Encoding)); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTypedef( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Type, const char *Name, size_t NameLen, -// LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Scope) { -// return wrap(Builder->createTypedef( -// unwrap(Type), StringRef(Name, NameLen), unwrap(File), -// LineNo, unwrapDIPtr(Scope))); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef PointeeTy, -// uint64_t SizeInBits, uint32_t AlignInBits, unsigned AddressSpace, -// const char *Name, size_t NameLen) { -// return wrap(Builder->createPointerType(unwrapDI(PointeeTy), -// SizeInBits, AlignInBits, -// AddressSpace, -// StringRef(Name, NameLen))); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStructType( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, -// const char *Name, size_t NameLen, -// LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, -// uint32_t AlignInBits, LLVMRustDIFlags Flags, -// LLVMMetadataRef DerivedFrom, LLVMMetadataRef Elements, -// unsigned RunTimeLang, LLVMMetadataRef VTableHolder, -// const char *UniqueId, size_t UniqueIdLen) { -// return wrap(Builder->createStructType( -// unwrapDI(Scope), StringRef(Name, NameLen), -// unwrapDI(File), LineNumber, -// SizeInBits, AlignInBits, fromRust(Flags), unwrapDI(DerivedFrom), -// DINodeArray(unwrapDI(Elements)), RunTimeLang, -// unwrapDI(VTableHolder), StringRef(UniqueId, UniqueIdLen))); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantPart( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, -// const char *Name, size_t NameLen, -// LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, -// uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Discriminator, -// LLVMMetadataRef Elements, const char *UniqueId, size_t UniqueIdLen) { -// return wrap(Builder->createVariantPart( -// unwrapDI(Scope), StringRef(Name, NameLen), -// unwrapDI(File), LineNumber, -// SizeInBits, AlignInBits, fromRust(Flags), unwrapDI(Discriminator), -// DINodeArray(unwrapDI(Elements)), StringRef(UniqueId, UniqueIdLen))); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMemberType( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, -// const char *Name, size_t NameLen, -// LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits, -// uint32_t AlignInBits, uint64_t OffsetInBits, LLVMRustDIFlags Flags, -// LLVMMetadataRef Ty) { -// return wrap(Builder->createMemberType(unwrapDI(Scope), -// StringRef(Name, NameLen), -// unwrapDI(File), LineNo, -// SizeInBits, AlignInBits, OffsetInBits, -// fromRust(Flags), unwrapDI(Ty))); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, -// const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo, -// uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, LLVMValueRef Discriminant, -// LLVMRustDIFlags Flags, LLVMMetadataRef Ty) { -// llvm::ConstantInt* D = nullptr; -// if (Discriminant) { -// D = unwrap(Discriminant); -// } -// return wrap(Builder->createVariantMemberType(unwrapDI(Scope), -// StringRef(Name, NameLen), -// unwrapDI(File), LineNo, -// SizeInBits, AlignInBits, OffsetInBits, D, -// fromRust(Flags), unwrapDI(Ty))); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateLexicalBlock( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, -// LLVMMetadataRef File, unsigned Line, unsigned Col) { -// return wrap(Builder->createLexicalBlock(unwrapDI(Scope), -// unwrapDI(File), Line, Col)); -// } - -// extern "C" LLVMMetadataRef -// LLVMRustDIBuilderCreateLexicalBlockFile(LLVMRustDIBuilderRef Builder, -// LLVMMetadataRef Scope, -// LLVMMetadataRef File) { -// return wrap(Builder->createLexicalBlockFile(unwrapDI(Scope), -// unwrapDI(File))); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Context, -// const char *Name, size_t NameLen, -// const char *LinkageName, size_t LinkageNameLen, -// LLVMMetadataRef File, unsigned LineNo, -// LLVMMetadataRef Ty, bool IsLocalToUnit, LLVMValueRef V, -// LLVMMetadataRef Decl = nullptr, uint32_t AlignInBits = 0) { -// llvm::GlobalVariable *InitVal = cast(unwrap(V)); - -// llvm::DIExpression *InitExpr = nullptr; -// if (llvm::ConstantInt *IntVal = llvm::dyn_cast(InitVal)) { -// InitExpr = Builder->createConstantValueExpression( -// IntVal->getValue().getSExtValue()); -// } else if (llvm::ConstantFP *FPVal = -// llvm::dyn_cast(InitVal)) { -// InitExpr = Builder->createConstantValueExpression( -// FPVal->getValueAPF().bitcastToAPInt().getZExtValue()); -// } - -// llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression( -// unwrapDI(Context), StringRef(Name, NameLen), -// StringRef(LinkageName, LinkageNameLen), -// unwrapDI(File), LineNo, unwrapDI(Ty), IsLocalToUnit, -// /* isDefined */ true, -// InitExpr, unwrapDIPtr(Decl), -// /* templateParams */ nullptr, -// AlignInBits); - -// InitVal->setMetadata("dbg", VarExpr); - -// return wrap(VarExpr); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable( -// LLVMRustDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope, -// const char *Name, size_t NameLen, -// LLVMMetadataRef File, unsigned LineNo, -// LLVMMetadataRef Ty, bool AlwaysPreserve, LLVMRustDIFlags Flags, -// unsigned ArgNo, uint32_t AlignInBits) { -// if (Tag == 0x100) { // DW_TAG_auto_variable -// return wrap(Builder->createAutoVariable( -// unwrapDI(Scope), StringRef(Name, NameLen), -// unwrapDI(File), LineNo, -// unwrapDI(Ty), AlwaysPreserve, fromRust(Flags), AlignInBits)); -// } else { -// return wrap(Builder->createParameterVariable( -// unwrapDI(Scope), StringRef(Name, NameLen), ArgNo, -// unwrapDI(File), LineNo, -// unwrapDI(Ty), AlwaysPreserve, fromRust(Flags))); -// } -// } - -// extern "C" LLVMMetadataRef -// LLVMRustDIBuilderCreateArrayType(LLVMRustDIBuilderRef Builder, uint64_t Size, -// uint32_t AlignInBits, LLVMMetadataRef Ty, -// LLVMMetadataRef Subscripts) { -// return wrap( -// Builder->createArrayType(Size, AlignInBits, unwrapDI(Ty), -// DINodeArray(unwrapDI(Subscripts)))); -// } - -// extern "C" LLVMMetadataRef -// LLVMRustDIBuilderGetOrCreateSubrange(LLVMRustDIBuilderRef Builder, int64_t Lo, -// int64_t Count) { -// return wrap(Builder->getOrCreateSubrange(Lo, Count)); -// } - -// extern "C" LLVMMetadataRef -// LLVMRustDIBuilderGetOrCreateArray(LLVMRustDIBuilderRef Builder, -// LLVMMetadataRef *Ptr, unsigned Count) { -// Metadata **DataValue = unwrap(Ptr); -// return wrap( -// Builder->getOrCreateArray(ArrayRef(DataValue, Count)).get()); -// } - -// extern "C" LLVMValueRef LLVMRustDIBuilderInsertDeclareAtEnd( -// LLVMRustDIBuilderRef Builder, LLVMValueRef V, LLVMMetadataRef VarInfo, -// uint64_t *AddrOps, unsigned AddrOpsCount, LLVMMetadataRef DL, -// LLVMBasicBlockRef InsertAtEnd) { -// return wrap(Builder->insertDeclare( -// unwrap(V), unwrap(VarInfo), -// Builder->createExpression(llvm::ArrayRef(AddrOps, AddrOpsCount)), -// DebugLoc(cast(unwrap(DL))), -// unwrap(InsertAtEnd))); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerator( -// LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, -// int64_t Value, bool IsUnsigned) { -// return wrap(Builder->createEnumerator(StringRef(Name, NameLen), Value, IsUnsigned)); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, -// const char *Name, size_t NameLen, -// LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, -// uint32_t AlignInBits, LLVMMetadataRef Elements, -// LLVMMetadataRef ClassTy, bool IsScoped) { -// return wrap(Builder->createEnumerationType( -// unwrapDI(Scope), StringRef(Name, NameLen), -// unwrapDI(File), LineNumber, -// SizeInBits, AlignInBits, DINodeArray(unwrapDI(Elements)), -// unwrapDI(ClassTy), "", IsScoped)); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, -// const char *Name, size_t NameLen, -// LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, -// uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Elements, -// unsigned RunTimeLang, const char *UniqueId, size_t UniqueIdLen) { -// return wrap(Builder->createUnionType( -// unwrapDI(Scope), StringRef(Name, NameLen), unwrapDI(File), -// LineNumber, SizeInBits, AlignInBits, fromRust(Flags), -// DINodeArray(unwrapDI(Elements)), RunTimeLang, -// StringRef(UniqueId, UniqueIdLen))); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTemplateTypeParameter( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, -// const char *Name, size_t NameLen, LLVMMetadataRef Ty) { -// bool IsDefault = false; // FIXME: should we ever set this true? -// return wrap(Builder->createTemplateTypeParameter( -// unwrapDI(Scope), StringRef(Name, NameLen), unwrapDI(Ty), IsDefault)); -// } - -// extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateNameSpace( -// LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, -// const char *Name, size_t NameLen, bool ExportSymbols) { -// return wrap(Builder->createNameSpace( -// unwrapDI(Scope), StringRef(Name, NameLen), ExportSymbols -// )); -// } - -// extern "C" void -// LLVMRustDICompositeTypeReplaceArrays(LLVMRustDIBuilderRef Builder, -// LLVMMetadataRef CompositeTy, -// LLVMMetadataRef Elements, -// LLVMMetadataRef Params) { -// DICompositeType *Tmp = unwrapDI(CompositeTy); -// Builder->replaceArrays(Tmp, DINodeArray(unwrap(Elements)), -// DINodeArray(unwrap(Params))); -// } - -// extern "C" LLVMMetadataRef -// LLVMRustDIBuilderCreateDebugLocation(unsigned Line, unsigned Column, -// LLVMMetadataRef ScopeRef, -// LLVMMetadataRef InlinedAt) { -// MDNode *Scope = unwrapDIPtr(ScopeRef); -// DILocation *Loc = DILocation::get( -// Scope->getContext(), Line, Column, Scope, -// unwrapDIPtr(InlinedAt)); -// return wrap(Loc); -// } - -// extern "C" uint64_t LLVMRustDIBuilderCreateOpDeref() { -// return dwarf::DW_OP_deref; -// } - -// extern "C" uint64_t LLVMRustDIBuilderCreateOpPlusUconst() { -// return dwarf::DW_OP_plus_uconst; -// } - -// extern "C" void LLVMRustWriteTypeToString(LLVMTypeRef Ty, RustStringRef Str) { -// RawRustStringOstream OS(Str); -// unwrap(Ty)->print(OS); -// } - -// extern "C" void LLVMRustWriteValueToString(LLVMValueRef V, -// RustStringRef Str) { -// RawRustStringOstream OS(Str); -// if (!V) { -// OS << "(null)"; -// } else { -// OS << "("; -// unwrap(V)->getType()->print(OS); -// OS << ":"; -// unwrap(V)->print(OS); -// OS << ")"; -// } -// } - -// // LLVMArrayType function does not support 64-bit ElementCount -// extern "C" LLVMTypeRef LLVMRustArrayType(LLVMTypeRef ElementTy, -// uint64_t ElementCount) { -// return wrap(ArrayType::get(unwrap(ElementTy), ElementCount)); -// } - -// DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Twine, LLVMTwineRef) - -// extern "C" void LLVMRustWriteTwineToString(LLVMTwineRef T, RustStringRef Str) { -// RawRustStringOstream OS(Str); -// unwrap(T)->print(OS); -// } - -// extern "C" void LLVMRustUnpackOptimizationDiagnostic( -// LLVMDiagnosticInfoRef DI, RustStringRef PassNameOut, -// LLVMValueRef *FunctionOut, unsigned* Line, unsigned* Column, -// RustStringRef FilenameOut, RustStringRef MessageOut) { -// // Undefined to call this not on an optimization diagnostic! -// llvm::DiagnosticInfoOptimizationBase *Opt = -// static_cast(unwrap(DI)); - -// RawRustStringOstream PassNameOS(PassNameOut); -// PassNameOS << Opt->getPassName(); -// *FunctionOut = wrap(&Opt->getFunction()); - -// RawRustStringOstream FilenameOS(FilenameOut); -// DiagnosticLocation loc = Opt->getLocation(); -// if (loc.isValid()) { -// *Line = loc.getLine(); -// *Column = loc.getColumn(); -// FilenameOS << loc.getAbsolutePath(); -// } - -// RawRustStringOstream MessageOS(MessageOut); -// MessageOS << Opt->getMsg(); -// } - -// enum class LLVMRustDiagnosticLevel { -// Error, -// Warning, -// Note, -// Remark, -// }; - -// extern "C" void -// LLVMRustUnpackInlineAsmDiagnostic(LLVMDiagnosticInfoRef DI, -// LLVMRustDiagnosticLevel *LevelOut, -// unsigned *CookieOut, -// LLVMTwineRef *MessageOut) { -// // Undefined to call this not on an inline assembly diagnostic! -// llvm::DiagnosticInfoInlineAsm *IA = -// static_cast(unwrap(DI)); - -// *CookieOut = IA->getLocCookie(); -// *MessageOut = wrap(&IA->getMsgStr()); - -// switch (IA->getSeverity()) { -// case DS_Error: -// *LevelOut = LLVMRustDiagnosticLevel::Error; -// break; -// case DS_Warning: -// *LevelOut = LLVMRustDiagnosticLevel::Warning; -// break; -// case DS_Note: -// *LevelOut = LLVMRustDiagnosticLevel::Note; -// break; -// case DS_Remark: -// *LevelOut = LLVMRustDiagnosticLevel::Remark; -// break; -// default: -// report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); -// } -// } - -// extern "C" void LLVMRustWriteDiagnosticInfoToString(LLVMDiagnosticInfoRef DI, -// RustStringRef Str) { -// RawRustStringOstream OS(Str); -// DiagnosticPrinterRawOStream DP(OS); -// unwrap(DI)->print(DP); -// } - -// enum class LLVMRustDiagnosticKind { -// Other, -// InlineAsm, -// StackSize, -// DebugMetadataVersion, -// SampleProfile, -// OptimizationRemark, -// OptimizationRemarkMissed, -// OptimizationRemarkAnalysis, -// OptimizationRemarkAnalysisFPCommute, -// OptimizationRemarkAnalysisAliasing, -// OptimizationRemarkOther, -// OptimizationFailure, -// PGOProfile, -// Linker, -// Unsupported, -// SrcMgr, -// }; - -// static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) { -// switch (Kind) { -// case DK_InlineAsm: -// return LLVMRustDiagnosticKind::InlineAsm; -// case DK_StackSize: -// return LLVMRustDiagnosticKind::StackSize; -// case DK_DebugMetadataVersion: -// return LLVMRustDiagnosticKind::DebugMetadataVersion; -// case DK_SampleProfile: -// return LLVMRustDiagnosticKind::SampleProfile; -// case DK_OptimizationRemark: -// case DK_MachineOptimizationRemark: -// return LLVMRustDiagnosticKind::OptimizationRemark; -// case DK_OptimizationRemarkMissed: -// case DK_MachineOptimizationRemarkMissed: -// return LLVMRustDiagnosticKind::OptimizationRemarkMissed; -// case DK_OptimizationRemarkAnalysis: -// case DK_MachineOptimizationRemarkAnalysis: -// return LLVMRustDiagnosticKind::OptimizationRemarkAnalysis; -// case DK_OptimizationRemarkAnalysisFPCommute: -// return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisFPCommute; -// case DK_OptimizationRemarkAnalysisAliasing: -// return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisAliasing; -// case DK_PGOProfile: -// return LLVMRustDiagnosticKind::PGOProfile; -// case DK_Linker: -// return LLVMRustDiagnosticKind::Linker; -// case DK_Unsupported: -// return LLVMRustDiagnosticKind::Unsupported; -// #if LLVM_VERSION_GE(13, 0) -// case DK_SrcMgr: -// return LLVMRustDiagnosticKind::SrcMgr; -// #endif -// default: -// return (Kind >= DK_FirstRemark && Kind <= DK_LastRemark) -// ? LLVMRustDiagnosticKind::OptimizationRemarkOther -// : LLVMRustDiagnosticKind::Other; -// } -// } - -// extern "C" LLVMRustDiagnosticKind -// LLVMRustGetDiagInfoKind(LLVMDiagnosticInfoRef DI) { -// return toRust((DiagnosticKind)unwrap(DI)->getKind()); -// } - -// // This is kept distinct from LLVMGetTypeKind, because when -// // a new type kind is added, the Rust-side enum must be -// // updated or UB will result. -// extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) { -// switch (unwrap(Ty)->getTypeID()) { -// case Type::VoidTyID: -// return LLVMVoidTypeKind; -// case Type::HalfTyID: -// return LLVMHalfTypeKind; -// case Type::FloatTyID: -// return LLVMFloatTypeKind; -// case Type::DoubleTyID: -// return LLVMDoubleTypeKind; -// case Type::X86_FP80TyID: -// return LLVMX86_FP80TypeKind; -// case Type::FP128TyID: -// return LLVMFP128TypeKind; -// case Type::PPC_FP128TyID: -// return LLVMPPC_FP128TypeKind; -// case Type::LabelTyID: -// return LLVMLabelTypeKind; -// case Type::MetadataTyID: -// return LLVMMetadataTypeKind; -// case Type::IntegerTyID: -// return LLVMIntegerTypeKind; -// case Type::FunctionTyID: -// return LLVMFunctionTypeKind; -// case Type::StructTyID: -// return LLVMStructTypeKind; -// case Type::ArrayTyID: -// return LLVMArrayTypeKind; -// case Type::PointerTyID: -// return LLVMPointerTypeKind; -// case Type::FixedVectorTyID: -// return LLVMVectorTypeKind; -// case Type::X86_MMXTyID: -// return LLVMX86_MMXTypeKind; -// case Type::TokenTyID: -// return LLVMTokenTypeKind; -// case Type::ScalableVectorTyID: -// return LLVMScalableVectorTypeKind; -// case Type::BFloatTyID: -// return LLVMBFloatTypeKind; -// case Type::X86_AMXTyID: -// return LLVMX86_AMXTypeKind; -// #if LLVM_VERSION_GE(15, 0) && LLVM_VERSION_LT(16, 0) -// case Type::DXILPointerTyID: -// report_fatal_error("Rust does not support DirectX typed pointers."); -// break; -// #endif -// #if LLVM_VERSION_GE(16, 0) -// case Type::TypedPointerTyID: -// report_fatal_error("Rust does not support typed pointers."); -// break; -// #endif -// } -// report_fatal_error("Unhandled TypeID."); -// } - -// DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef) - -// #if LLVM_VERSION_LT(13, 0) -// using LLVMInlineAsmDiagHandlerTy = LLVMContext::InlineAsmDiagHandlerTy; -// #else -// using LLVMInlineAsmDiagHandlerTy = void*; -// #endif - -// extern "C" void LLVMRustSetInlineAsmDiagnosticHandler( -// LLVMContextRef C, LLVMInlineAsmDiagHandlerTy H, void *CX) { -// // Diagnostic handlers were unified in LLVM change 5de2d189e6ad, so starting -// // with LLVM 13 this function is gone. -// #if LLVM_VERSION_LT(13, 0) -// unwrap(C)->setInlineAsmDiagnosticHandler(H, CX); -// #endif -// } - -// extern "C" LLVMSMDiagnosticRef LLVMRustGetSMDiagnostic( -// LLVMDiagnosticInfoRef DI, unsigned *Cookie) { -// #if LLVM_VERSION_GE(13, 0) -// llvm::DiagnosticInfoSrcMgr *SM = static_cast(unwrap(DI)); -// *Cookie = SM->getLocCookie(); -// return wrap(&SM->getSMDiag()); -// #else -// report_fatal_error("Shouldn't get called on older versions"); -// #endif -// } - -// extern "C" bool LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef, -// RustStringRef MessageOut, -// RustStringRef BufferOut, -// LLVMRustDiagnosticLevel* LevelOut, -// unsigned* LocOut, -// unsigned* RangesOut, -// size_t* NumRanges) { -// SMDiagnostic& D = *unwrap(DRef); -// RawRustStringOstream MessageOS(MessageOut); -// MessageOS << D.getMessage(); - -// switch (D.getKind()) { -// case SourceMgr::DK_Error: -// *LevelOut = LLVMRustDiagnosticLevel::Error; -// break; -// case SourceMgr::DK_Warning: -// *LevelOut = LLVMRustDiagnosticLevel::Warning; -// break; -// case SourceMgr::DK_Note: -// *LevelOut = LLVMRustDiagnosticLevel::Note; -// break; -// case SourceMgr::DK_Remark: -// *LevelOut = LLVMRustDiagnosticLevel::Remark; -// break; -// default: -// report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); -// } - -// if (D.getLoc() == SMLoc()) -// return false; - -// const SourceMgr &LSM = *D.getSourceMgr(); -// const MemoryBuffer *LBuf = LSM.getMemoryBuffer(LSM.FindBufferContainingLoc(D.getLoc())); -// LLVMRustStringWriteImpl(BufferOut, LBuf->getBufferStart(), LBuf->getBufferSize()); - -// *LocOut = D.getLoc().getPointer() - LBuf->getBufferStart(); - -// *NumRanges = std::min(*NumRanges, D.getRanges().size()); -// size_t LineStart = *LocOut - (size_t)D.getColumnNo(); -// for (size_t i = 0; i < *NumRanges; i++) { -// RangesOut[i * 2] = LineStart + D.getRanges()[i].first; -// RangesOut[i * 2 + 1] = LineStart + D.getRanges()[i].second; -// } - -// return true; -// } - -// extern "C" LLVMValueRef LLVMRustBuildCleanupPad(LLVMBuilderRef B, -// LLVMValueRef ParentPad, -// unsigned ArgCount, -// LLVMValueRef *LLArgs, -// const char *Name) { -// Value **Args = unwrap(LLArgs); -// if (ParentPad == nullptr) { -// Type *Ty = Type::getTokenTy(unwrap(B)->getContext()); -// ParentPad = wrap(Constant::getNullValue(Ty)); -// } -// return wrap(unwrap(B)->CreateCleanupPad( -// unwrap(ParentPad), ArrayRef(Args, ArgCount), Name)); -// } - -// extern "C" LLVMValueRef LLVMRustBuildCleanupRet(LLVMBuilderRef B, -// LLVMValueRef CleanupPad, -// LLVMBasicBlockRef UnwindBB) { -// CleanupPadInst *Inst = cast(unwrap(CleanupPad)); -// return wrap(unwrap(B)->CreateCleanupRet(Inst, unwrap(UnwindBB))); -// } - -// extern "C" LLVMValueRef -// LLVMRustBuildCatchPad(LLVMBuilderRef B, LLVMValueRef ParentPad, -// unsigned ArgCount, LLVMValueRef *LLArgs, const char *Name) { -// Value **Args = unwrap(LLArgs); -// return wrap(unwrap(B)->CreateCatchPad( -// unwrap(ParentPad), ArrayRef(Args, ArgCount), Name)); -// } - -// extern "C" LLVMValueRef LLVMRustBuildCatchRet(LLVMBuilderRef B, -// LLVMValueRef Pad, -// LLVMBasicBlockRef BB) { -// return wrap(unwrap(B)->CreateCatchRet(cast(unwrap(Pad)), -// unwrap(BB))); -// } - -// extern "C" LLVMValueRef LLVMRustBuildCatchSwitch(LLVMBuilderRef B, -// LLVMValueRef ParentPad, -// LLVMBasicBlockRef BB, -// unsigned NumHandlers, -// const char *Name) { -// if (ParentPad == nullptr) { -// Type *Ty = Type::getTokenTy(unwrap(B)->getContext()); -// ParentPad = wrap(Constant::getNullValue(Ty)); -// } -// return wrap(unwrap(B)->CreateCatchSwitch(unwrap(ParentPad), unwrap(BB), -// NumHandlers, Name)); -// } - -// extern "C" void LLVMRustAddHandler(LLVMValueRef CatchSwitchRef, -// LLVMBasicBlockRef Handler) { -// Value *CatchSwitch = unwrap(CatchSwitchRef); -// cast(CatchSwitch)->addHandler(unwrap(Handler)); -// } - -// extern "C" OperandBundleDef *LLVMRustBuildOperandBundleDef(const char *Name, -// LLVMValueRef *Inputs, -// unsigned NumInputs) { -// return new OperandBundleDef(Name, makeArrayRef(unwrap(Inputs), NumInputs)); -// } - -// extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { -// delete Bundle; -// } - -// extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, -// LLVMValueRef *Args, unsigned NumArgs, -// OperandBundleDef *Bundle) { -// Value *Callee = unwrap(Fn); -// FunctionType *FTy = unwrap(Ty); -// unsigned Len = Bundle ? 1 : 0; -// ArrayRef Bundles = makeArrayRef(Bundle, Len); -// return wrap(unwrap(B)->CreateCall( -// FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), Bundles)); -// } - -// extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) { -// return wrap(llvm::Intrinsic::getDeclaration(unwrap(M), -// (llvm::Intrinsic::ID)llvm::Intrinsic::instrprof_increment)); -// } - -// extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, -// LLVMValueRef Dst, unsigned DstAlign, -// LLVMValueRef Src, unsigned SrcAlign, -// LLVMValueRef Size, bool IsVolatile) { -// return wrap(unwrap(B)->CreateMemCpy( -// unwrap(Dst), MaybeAlign(DstAlign), -// unwrap(Src), MaybeAlign(SrcAlign), -// unwrap(Size), IsVolatile)); -// } - -// extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B, -// LLVMValueRef Dst, unsigned DstAlign, -// LLVMValueRef Src, unsigned SrcAlign, -// LLVMValueRef Size, bool IsVolatile) { -// return wrap(unwrap(B)->CreateMemMove( -// unwrap(Dst), MaybeAlign(DstAlign), -// unwrap(Src), MaybeAlign(SrcAlign), -// unwrap(Size), IsVolatile)); -// } - -// extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, -// LLVMValueRef Dst, unsigned DstAlign, -// LLVMValueRef Val, -// LLVMValueRef Size, bool IsVolatile) { -// return wrap(unwrap(B)->CreateMemSet( -// unwrap(Dst), unwrap(Val), unwrap(Size), MaybeAlign(DstAlign), IsVolatile)); -// } - -// extern "C" LLVMValueRef -// LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, -// LLVMValueRef *Args, unsigned NumArgs, -// LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, -// OperandBundleDef *Bundle, const char *Name) { -// Value *Callee = unwrap(Fn); -// FunctionType *FTy = unwrap(Ty); -// unsigned Len = Bundle ? 1 : 0; -// ArrayRef Bundles = makeArrayRef(Bundle, Len); -// return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch), -// makeArrayRef(unwrap(Args), NumArgs), -// Bundles, Name)); -// } - -// extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, -// LLVMBasicBlockRef BB) { -// auto Point = unwrap(BB)->getFirstInsertionPt(); -// unwrap(B)->SetInsertPoint(unwrap(BB), Point); -// } - -// extern "C" void LLVMRustSetComdat(LLVMModuleRef M, LLVMValueRef V, -// const char *Name, size_t NameLen) { -// Triple TargetTriple(unwrap(M)->getTargetTriple()); -// GlobalObject *GV = unwrap(V); -// if (TargetTriple.supportsCOMDAT()) { -// StringRef NameRef(Name, NameLen); -// GV->setComdat(unwrap(M)->getOrInsertComdat(NameRef)); -// } -// } - -// enum class LLVMRustLinkage { -// ExternalLinkage = 0, -// AvailableExternallyLinkage = 1, -// LinkOnceAnyLinkage = 2, -// LinkOnceODRLinkage = 3, -// WeakAnyLinkage = 4, -// WeakODRLinkage = 5, -// AppendingLinkage = 6, -// InternalLinkage = 7, -// PrivateLinkage = 8, -// ExternalWeakLinkage = 9, -// CommonLinkage = 10, -// }; - -// static LLVMRustLinkage toRust(LLVMLinkage Linkage) { -// switch (Linkage) { -// case LLVMExternalLinkage: -// return LLVMRustLinkage::ExternalLinkage; -// case LLVMAvailableExternallyLinkage: -// return LLVMRustLinkage::AvailableExternallyLinkage; -// case LLVMLinkOnceAnyLinkage: -// return LLVMRustLinkage::LinkOnceAnyLinkage; -// case LLVMLinkOnceODRLinkage: -// return LLVMRustLinkage::LinkOnceODRLinkage; -// case LLVMWeakAnyLinkage: -// return LLVMRustLinkage::WeakAnyLinkage; -// case LLVMWeakODRLinkage: -// return LLVMRustLinkage::WeakODRLinkage; -// case LLVMAppendingLinkage: -// return LLVMRustLinkage::AppendingLinkage; -// case LLVMInternalLinkage: -// return LLVMRustLinkage::InternalLinkage; -// case LLVMPrivateLinkage: -// return LLVMRustLinkage::PrivateLinkage; -// case LLVMExternalWeakLinkage: -// return LLVMRustLinkage::ExternalWeakLinkage; -// case LLVMCommonLinkage: -// return LLVMRustLinkage::CommonLinkage; -// default: -// report_fatal_error("Invalid LLVMRustLinkage value!"); -// } -// } - -// static LLVMLinkage fromRust(LLVMRustLinkage Linkage) { -// switch (Linkage) { -// case LLVMRustLinkage::ExternalLinkage: -// return LLVMExternalLinkage; -// case LLVMRustLinkage::AvailableExternallyLinkage: -// return LLVMAvailableExternallyLinkage; -// case LLVMRustLinkage::LinkOnceAnyLinkage: -// return LLVMLinkOnceAnyLinkage; -// case LLVMRustLinkage::LinkOnceODRLinkage: -// return LLVMLinkOnceODRLinkage; -// case LLVMRustLinkage::WeakAnyLinkage: -// return LLVMWeakAnyLinkage; -// case LLVMRustLinkage::WeakODRLinkage: -// return LLVMWeakODRLinkage; -// case LLVMRustLinkage::AppendingLinkage: -// return LLVMAppendingLinkage; -// case LLVMRustLinkage::InternalLinkage: -// return LLVMInternalLinkage; -// case LLVMRustLinkage::PrivateLinkage: -// return LLVMPrivateLinkage; -// case LLVMRustLinkage::ExternalWeakLinkage: -// return LLVMExternalWeakLinkage; -// case LLVMRustLinkage::CommonLinkage: -// return LLVMCommonLinkage; -// } -// report_fatal_error("Invalid LLVMRustLinkage value!"); -// } - -// extern "C" LLVMRustLinkage LLVMRustGetLinkage(LLVMValueRef V) { -// return toRust(LLVMGetLinkage(V)); -// } - -// extern "C" void LLVMRustSetLinkage(LLVMValueRef V, -// LLVMRustLinkage RustLinkage) { -// LLVMSetLinkage(V, fromRust(RustLinkage)); -// } - -// extern "C" LLVMValueRef LLVMRustConstInBoundsGEP2(LLVMTypeRef Ty, -// LLVMValueRef ConstantVal, -// LLVMValueRef *ConstantIndices, -// unsigned NumIndices) { -// ArrayRef IdxList(unwrap(ConstantIndices, NumIndices), -// NumIndices); -// Constant *Val = unwrap(ConstantVal); -// return wrap(ConstantExpr::getInBoundsGetElementPtr(unwrap(Ty), Val, IdxList)); -// } - -// // Returns true if both high and low were successfully set. Fails in case constant wasn’t any of -// // the common sizes (1, 8, 16, 32, 64, 128 bits) -// extern "C" bool LLVMRustConstInt128Get(LLVMValueRef CV, bool sext, uint64_t *high, uint64_t *low) -// { -// auto C = unwrap(CV); -// if (C->getBitWidth() > 128) { return false; } -// APInt AP; -// #if LLVM_VERSION_GE(15, 0) -// if (sext) { -// AP = C->getValue().sext(128); -// } else { -// AP = C->getValue().zext(128); -// } -// #else -// if (sext) { -// AP = C->getValue().sextOrSelf(128); -// } else { -// AP = C->getValue().zextOrSelf(128); -// } -// #endif -// *low = AP.getLoBits(64).getZExtValue(); -// *high = AP.getHiBits(64).getZExtValue(); -// return true; -// } - -// enum class LLVMRustVisibility { -// Default = 0, -// Hidden = 1, -// Protected = 2, -// }; - -// static LLVMRustVisibility toRust(LLVMVisibility Vis) { -// switch (Vis) { -// case LLVMDefaultVisibility: -// return LLVMRustVisibility::Default; -// case LLVMHiddenVisibility: -// return LLVMRustVisibility::Hidden; -// case LLVMProtectedVisibility: -// return LLVMRustVisibility::Protected; -// } -// report_fatal_error("Invalid LLVMRustVisibility value!"); -// } - -// static LLVMVisibility fromRust(LLVMRustVisibility Vis) { -// switch (Vis) { -// case LLVMRustVisibility::Default: -// return LLVMDefaultVisibility; -// case LLVMRustVisibility::Hidden: -// return LLVMHiddenVisibility; -// case LLVMRustVisibility::Protected: -// return LLVMProtectedVisibility; -// } -// report_fatal_error("Invalid LLVMRustVisibility value!"); -// } - -// extern "C" LLVMRustVisibility LLVMRustGetVisibility(LLVMValueRef V) { -// return toRust(LLVMGetVisibility(V)); -// } - -// // Oh hey, a binding that makes sense for once? (because LLVM’s own do not) -// extern "C" LLVMValueRef LLVMRustBuildIntCast(LLVMBuilderRef B, LLVMValueRef Val, -// LLVMTypeRef DestTy, bool isSigned) { -// return wrap(unwrap(B)->CreateIntCast(unwrap(Val), unwrap(DestTy), isSigned, "")); -// } - -// extern "C" void LLVMRustSetVisibility(LLVMValueRef V, -// LLVMRustVisibility RustVisibility) { -// LLVMSetVisibility(V, fromRust(RustVisibility)); -// } - -// extern "C" void LLVMRustSetDSOLocal(LLVMValueRef Global, bool is_dso_local) { -// unwrap(Global)->setDSOLocal(is_dso_local); -// } - -// struct LLVMRustModuleBuffer { -// std::string data; -// }; - -// extern "C" LLVMRustModuleBuffer* -// LLVMRustModuleBufferCreate(LLVMModuleRef M) { -// auto Ret = std::make_unique(); -// { -// raw_string_ostream OS(Ret->data); -// { -// legacy::PassManager PM; -// PM.add(createBitcodeWriterPass(OS)); -// PM.run(*unwrap(M)); -// } -// } -// return Ret.release(); -// } - -// extern "C" void -// LLVMRustModuleBufferFree(LLVMRustModuleBuffer *Buffer) { -// delete Buffer; -// } - -// extern "C" const void* -// LLVMRustModuleBufferPtr(const LLVMRustModuleBuffer *Buffer) { -// return Buffer->data.data(); -// } - -// extern "C" size_t -// LLVMRustModuleBufferLen(const LLVMRustModuleBuffer *Buffer) { -// return Buffer->data.length(); -// } - -// extern "C" uint64_t -// LLVMRustModuleCost(LLVMModuleRef M) { -// auto f = unwrap(M)->functions(); -// return std::distance(std::begin(f), std::end(f)); -// } - -// // Vector reductions: -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceFAdd(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { -// return wrap(unwrap(B)->CreateFAddReduce(unwrap(Acc),unwrap(Src))); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceFMul(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { -// return wrap(unwrap(B)->CreateFMulReduce(unwrap(Acc),unwrap(Src))); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceAdd(LLVMBuilderRef B, LLVMValueRef Src) { -// return wrap(unwrap(B)->CreateAddReduce(unwrap(Src))); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceMul(LLVMBuilderRef B, LLVMValueRef Src) { -// return wrap(unwrap(B)->CreateMulReduce(unwrap(Src))); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceAnd(LLVMBuilderRef B, LLVMValueRef Src) { -// return wrap(unwrap(B)->CreateAndReduce(unwrap(Src))); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceOr(LLVMBuilderRef B, LLVMValueRef Src) { -// return wrap(unwrap(B)->CreateOrReduce(unwrap(Src))); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceXor(LLVMBuilderRef B, LLVMValueRef Src) { -// return wrap(unwrap(B)->CreateXorReduce(unwrap(Src))); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceMin(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) { -// return wrap(unwrap(B)->CreateIntMinReduce(unwrap(Src), IsSigned)); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceMax(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) { -// return wrap(unwrap(B)->CreateIntMaxReduce(unwrap(Src), IsSigned)); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceFMin(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { -// Instruction *I = unwrap(B)->CreateFPMinReduce(unwrap(Src)); -// I->setHasNoNaNs(NoNaN); -// return wrap(I); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildVectorReduceFMax(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { -// Instruction *I = unwrap(B)->CreateFPMaxReduce(unwrap(Src)); -// I->setHasNoNaNs(NoNaN); -// return wrap(I); -// } - -// extern "C" LLVMValueRef -// LLVMRustBuildMinNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { -// return wrap(unwrap(B)->CreateMinNum(unwrap(LHS),unwrap(RHS))); -// } -// extern "C" LLVMValueRef -// LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { -// return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS),unwrap(RHS))); -// } - -// // This struct contains all necessary info about a symbol exported from a DLL. -// struct LLVMRustCOFFShortExport { -// const char* name; -// bool ordinal_present; -// // The value of `ordinal` is only meaningful if `ordinal_present` is true. -// uint16_t ordinal; -// }; - -// // Machine must be a COFF machine type, as defined in PE specs. -// extern "C" LLVMRustResult LLVMRustWriteImportLibrary( -// const char* ImportName, -// const char* Path, -// const LLVMRustCOFFShortExport* Exports, -// size_t NumExports, -// uint16_t Machine, -// bool MinGW) -// { -// std::vector ConvertedExports; -// ConvertedExports.reserve(NumExports); - -// for (size_t i = 0; i < NumExports; ++i) { -// bool ordinal_present = Exports[i].ordinal_present; -// uint16_t ordinal = ordinal_present ? Exports[i].ordinal : 0; -// ConvertedExports.push_back(llvm::object::COFFShortExport{ -// Exports[i].name, // Name -// std::string{}, // ExtName -// std::string{}, // SymbolName -// std::string{}, // AliasTarget -// ordinal, // Ordinal -// ordinal_present, // Noname -// false, // Data -// false, // Private -// false // Constant -// }); -// } - -// auto Error = llvm::object::writeImportLibrary( -// ImportName, -// Path, -// ConvertedExports, -// static_cast(Machine), -// MinGW); -// if (Error) { -// std::string errorString; -// llvm::raw_string_ostream stream(errorString); -// stream << Error; -// stream.flush(); -// LLVMRustSetLastError(errorString.c_str()); -// return LLVMRustResult::Failure; -// } else { -// return LLVMRustResult::Success; -// } -// } - -// // Transfers ownership of DiagnosticHandler unique_ptr to the caller. -// extern "C" DiagnosticHandler * -// LLVMRustContextGetDiagnosticHandler(LLVMContextRef C) { -// std::unique_ptr DH = unwrap(C)->getDiagnosticHandler(); -// return DH.release(); -// } - -// // Sets unique_ptr to object of DiagnosticHandler to provide custom diagnostic -// // handling. Ownership of the handler is moved to the LLVMContext. -// extern "C" void LLVMRustContextSetDiagnosticHandler(LLVMContextRef C, -// DiagnosticHandler *DH) { -// unwrap(C)->setDiagnosticHandler(std::unique_ptr(DH)); -// } - -// using LLVMDiagnosticHandlerTy = DiagnosticHandler::DiagnosticHandlerTy; - -// // Configures a diagnostic handler that invokes provided callback when a -// // backend needs to emit a diagnostic. -// // -// // When RemarkAllPasses is true, remarks are enabled for all passes. Otherwise -// // the RemarkPasses array specifies individual passes for which remarks will be -// // enabled. -// extern "C" void LLVMRustContextConfigureDiagnosticHandler( -// LLVMContextRef C, LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, -// void *DiagnosticHandlerContext, bool RemarkAllPasses, -// const char * const * RemarkPasses, size_t RemarkPassesLen) { - -// class RustDiagnosticHandler final : public DiagnosticHandler { -// public: -// RustDiagnosticHandler(LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, -// void *DiagnosticHandlerContext, -// bool RemarkAllPasses, -// std::vector RemarkPasses) -// : DiagnosticHandlerCallback(DiagnosticHandlerCallback), -// DiagnosticHandlerContext(DiagnosticHandlerContext), -// RemarkAllPasses(RemarkAllPasses), -// RemarkPasses(RemarkPasses) {} - -// virtual bool handleDiagnostics(const DiagnosticInfo &DI) override { -// if (DiagnosticHandlerCallback) { -// DiagnosticHandlerCallback(DI, DiagnosticHandlerContext); -// return true; -// } -// return false; -// } - -// bool isAnalysisRemarkEnabled(StringRef PassName) const override { -// return isRemarkEnabled(PassName); -// } - -// bool isMissedOptRemarkEnabled(StringRef PassName) const override { -// return isRemarkEnabled(PassName); -// } - -// bool isPassedOptRemarkEnabled(StringRef PassName) const override { -// return isRemarkEnabled(PassName); -// } - -// bool isAnyRemarkEnabled() const override { -// return RemarkAllPasses || !RemarkPasses.empty(); -// } - -// private: -// bool isRemarkEnabled(StringRef PassName) const { -// if (RemarkAllPasses) -// return true; - -// for (auto &Pass : RemarkPasses) -// if (Pass == PassName) -// return true; - -// return false; -// } - -// LLVMDiagnosticHandlerTy DiagnosticHandlerCallback = nullptr; -// void *DiagnosticHandlerContext = nullptr; - -// bool RemarkAllPasses = false; -// std::vector RemarkPasses; -// }; - -// std::vector Passes; -// for (size_t I = 0; I != RemarkPassesLen; ++I) -// Passes.push_back(RemarkPasses[I]); - -// unwrap(C)->setDiagnosticHandler(std::make_unique( -// DiagnosticHandlerCallback, DiagnosticHandlerContext, RemarkAllPasses, Passes)); -// } - -// extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) { -// RawRustStringOstream OS(Str); -// GlobalValue *GV = unwrap(V); -// Mangler().getNameWithPrefix(OS, GV, true); -// } - -// // LLVMGetAggregateElement was added in LLVM 15. For earlier LLVM versions just -// // use its implementation. -// #if LLVM_VERSION_LT(15, 0) -// extern "C" LLVMValueRef LLVMGetAggregateElement(LLVMValueRef C, unsigned Idx) { -// return wrap(unwrap(C)->getAggregateElement(Idx)); -// } -// #endif - -// extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) { -// #if LLVM_VERSION_GE(15, 0) -// auto *CB = unwrap(CallSite); -// switch (CB->getIntrinsicID()) { -// case Intrinsic::arm_ldrex: -// return 0; -// case Intrinsic::arm_strex: -// return 1; -// } -// #endif -// return -1; -// } From 31523d7a44bd8911eb32ffb8cbf36ac8a8c1016c Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Tue, 16 Jan 2024 10:31:39 +0000 Subject: [PATCH 50/70] cleanup unused llvm-wrapper --- compiler/rustc_llvm/src/lib.rs | 185 --------------------------------- 1 file changed, 185 deletions(-) diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs index 989e4bc2ee7..8b137891791 100644 --- a/compiler/rustc_llvm/src/lib.rs +++ b/compiler/rustc_llvm/src/lib.rs @@ -1,186 +1 @@ -// #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -// // NOTE: This crate only exists to allow linking on mingw targets. - -// use libc::{c_char, size_t}; -// use std::cell::RefCell; -// use std::slice; - -// // NOTE: Trying to import this RustString will result in a BAD TIME. -// // Use the definition from `rustc_llvm_coverage` instead! -// #[repr(C)] -// pub struct RustString { -// pub bytes: RefCell>, -// } - -// impl RustString { -// pub fn len(&self) -> usize { -// self.bytes.borrow().len() -// } - -// pub fn is_empty(&self) -> bool { -// self.bytes.borrow().is_empty() -// } -// } - -// /// Appending to a Rust string -- used by RawRustStringOstream. -// #[no_mangle] -// pub unsafe extern "C" fn LLVMRustStringWriteImpl(sr: &RustString, ptr: *const c_char, size: size_t) { -// let slice = slice::from_raw_parts(ptr as *const u8, size as usize); - -// sr.bytes.borrow_mut().extend_from_slice(slice); -// } - -// /// Initialize targets enabled by the build script via `cfg(llvm_component = "...")`. -// /// N.B., this function can't be moved to `rustc_codegen_llvm` because of the `cfg`s. -// pub fn initialize_available_targets() { -// macro_rules! init_target( -// ($cfg:meta, $($method:ident),*) => { { -// #[cfg($cfg)] -// fn init() { -// extern "C" { -// $(fn $method();)* -// } -// unsafe { -// $($method();)* -// } -// } -// #[cfg(not($cfg))] -// fn init() { } -// init(); -// } } -// ); -// init_target!( -// llvm_component = "x86", -// LLVMInitializeX86TargetInfo, -// LLVMInitializeX86Target, -// LLVMInitializeX86TargetMC, -// LLVMInitializeX86AsmPrinter, -// LLVMInitializeX86AsmParser -// ); -// init_target!( -// llvm_component = "arm", -// LLVMInitializeARMTargetInfo, -// LLVMInitializeARMTarget, -// LLVMInitializeARMTargetMC, -// LLVMInitializeARMAsmPrinter, -// LLVMInitializeARMAsmParser -// ); -// init_target!( -// llvm_component = "aarch64", -// LLVMInitializeAArch64TargetInfo, -// LLVMInitializeAArch64Target, -// LLVMInitializeAArch64TargetMC, -// LLVMInitializeAArch64AsmPrinter, -// LLVMInitializeAArch64AsmParser -// ); -// init_target!( -// llvm_component = "amdgpu", -// LLVMInitializeAMDGPUTargetInfo, -// LLVMInitializeAMDGPUTarget, -// LLVMInitializeAMDGPUTargetMC, -// LLVMInitializeAMDGPUAsmPrinter, -// LLVMInitializeAMDGPUAsmParser -// ); -// init_target!( -// llvm_component = "avr", -// LLVMInitializeAVRTargetInfo, -// LLVMInitializeAVRTarget, -// LLVMInitializeAVRTargetMC, -// LLVMInitializeAVRAsmPrinter, -// LLVMInitializeAVRAsmParser -// ); -// init_target!( -// llvm_component = "m68k", -// LLVMInitializeM68kTargetInfo, -// LLVMInitializeM68kTarget, -// LLVMInitializeM68kTargetMC, -// LLVMInitializeM68kAsmPrinter, -// LLVMInitializeM68kAsmParser -// ); -// init_target!( -// llvm_component = "mips", -// LLVMInitializeMipsTargetInfo, -// LLVMInitializeMipsTarget, -// LLVMInitializeMipsTargetMC, -// LLVMInitializeMipsAsmPrinter, -// LLVMInitializeMipsAsmParser -// ); -// init_target!( -// llvm_component = "powerpc", -// LLVMInitializePowerPCTargetInfo, -// LLVMInitializePowerPCTarget, -// LLVMInitializePowerPCTargetMC, -// LLVMInitializePowerPCAsmPrinter, -// LLVMInitializePowerPCAsmParser -// ); -// init_target!( -// llvm_component = "systemz", -// LLVMInitializeSystemZTargetInfo, -// LLVMInitializeSystemZTarget, -// LLVMInitializeSystemZTargetMC, -// LLVMInitializeSystemZAsmPrinter, -// LLVMInitializeSystemZAsmParser -// ); -// init_target!( -// llvm_component = "jsbackend", -// LLVMInitializeJSBackendTargetInfo, -// LLVMInitializeJSBackendTarget, -// LLVMInitializeJSBackendTargetMC -// ); -// init_target!( -// llvm_component = "msp430", -// LLVMInitializeMSP430TargetInfo, -// LLVMInitializeMSP430Target, -// LLVMInitializeMSP430TargetMC, -// LLVMInitializeMSP430AsmPrinter, -// LLVMInitializeMSP430AsmParser -// ); -// init_target!( -// llvm_component = "riscv", -// LLVMInitializeRISCVTargetInfo, -// LLVMInitializeRISCVTarget, -// LLVMInitializeRISCVTargetMC, -// LLVMInitializeRISCVAsmPrinter, -// LLVMInitializeRISCVAsmParser -// ); -// init_target!( -// llvm_component = "sparc", -// LLVMInitializeSparcTargetInfo, -// LLVMInitializeSparcTarget, -// LLVMInitializeSparcTargetMC, -// LLVMInitializeSparcAsmPrinter, -// LLVMInitializeSparcAsmParser -// ); -// init_target!( -// llvm_component = "nvptx", -// LLVMInitializeNVPTXTargetInfo, -// LLVMInitializeNVPTXTarget, -// LLVMInitializeNVPTXTargetMC, -// LLVMInitializeNVPTXAsmPrinter -// ); -// init_target!( -// llvm_component = "hexagon", -// LLVMInitializeHexagonTargetInfo, -// LLVMInitializeHexagonTarget, -// LLVMInitializeHexagonTargetMC, -// LLVMInitializeHexagonAsmPrinter, -// LLVMInitializeHexagonAsmParser -// ); -// init_target!( -// llvm_component = "webassembly", -// LLVMInitializeWebAssemblyTargetInfo, -// LLVMInitializeWebAssemblyTarget, -// LLVMInitializeWebAssemblyTargetMC, -// LLVMInitializeWebAssemblyAsmPrinter, -// LLVMInitializeWebAssemblyAsmParser -// ); -// init_target!( -// llvm_component = "bpf", -// LLVMInitializeBPFTargetInfo, -// LLVMInitializeBPFTarget, -// LLVMInitializeBPFTargetMC, -// LLVMInitializeBPFAsmPrinter, -// LLVMInitializeBPFAsmParser -// ); -// } From faac5ce6d3d58d2072658e350ffbcdbcc2a5e09a Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Tue, 16 Jan 2024 10:34:59 +0000 Subject: [PATCH 51/70] remove old subrepo --- inkwell | 1 - 1 file changed, 1 deletion(-) delete mode 160000 inkwell diff --git a/inkwell b/inkwell deleted file mode 160000 index e02803fcf18..00000000000 --- a/inkwell +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e02803fcf18ccc1f389cc36aa035b46045a9d9c0 From 3b986ff027db542d227d1a65672ab10d3dda7a5c Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Tue, 16 Jan 2024 10:39:18 +0000 Subject: [PATCH 52/70] restore old lock file --- Cargo.lock | 1045 +++++++++++++++++++++------------------------------- 1 file changed, 427 insertions(+), 618 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8f3c20d724c..0f4862e8fdb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,23 +19,22 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if", "getrandom", "once_cell", "serde", "version_check", - "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] @@ -63,9 +62,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.5" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", @@ -77,36 +76,36 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] @@ -122,49 +121,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - -[[package]] -name = "async-channel" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" -dependencies = [ - "concurrent-queue", - "event-listener 4.0.1", - "event-listener-strategy", + "event-listener", "futures-core", - "pin-project-lite", ] [[package]] name = "async-executor" -version = "1.8.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" +checksum = "6fa3dc5f2a8564f07759c008b9109dc0d39de92a88d5588b8a5036d286383afb" dependencies = [ - "async-lock 3.2.0", + "async-lock", "async-task", "concurrent-queue", - "fastrand 2.0.1", - "futures-lite 2.1.0", + "fastrand 1.9.0", + "futures-lite", "slab", ] [[package]] name = "async-global-executor" -version = "2.4.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" dependencies = [ - "async-channel 2.1.1", + "async-channel", "async-executor", - "async-io 2.2.2", - "async-lock 3.2.0", + "async-io", + "async-lock", "blocking", - "futures-lite 2.1.0", + "futures-lite", "once_cell", ] @@ -174,57 +160,27 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ - "async-lock 2.8.0", + "async-lock", "autocfg", "cfg-if", "concurrent-queue", - "futures-lite 1.13.0", + "futures-lite", "log", "parking", - "polling 2.8.0", - "rustix 0.37.27", + "polling", + "rustix 0.37.25", "slab", - "socket2 0.4.10", + "socket2 0.4.9", "waker-fn", ] -[[package]] -name = "async-io" -version = "2.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afaa937395a620e33dc6a742c593c01aced20aa376ffb0f628121198578ccc7" -dependencies = [ - "async-lock 3.2.0", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite 2.1.0", - "parking", - "polling 3.3.1", - "rustix 0.38.28", - "slab", - "tracing", - "windows-sys 0.52.0", -] - [[package]] name = "async-lock" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" dependencies = [ - "event-listener 2.5.3", -] - -[[package]] -name = "async-lock" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c" -dependencies = [ - "event-listener 4.0.1", - "event-listener-strategy", - "pin-project-lite", + "event-listener", ] [[package]] @@ -233,15 +189,15 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" dependencies = [ - "async-channel 1.9.0", + "async-channel", "async-global-executor", - "async-io 1.13.0", - "async-lock 2.8.0", + "async-io", + "async-lock", "crossbeam-utils", "futures-channel", "futures-core", "futures-io", - "futures-lite 1.13.0", + "futures-lite", "gloo-timers", "kv-log-macro", "log", @@ -255,9 +211,9 @@ dependencies = [ [[package]] name = "async-task" -version = "4.6.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d90cd0b264dfdd8eb5bad0a2c217c1f88fa96a8573f40e7b12de23fb468f46" +checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae" [[package]] name = "atoi" @@ -270,19 +226,9 @@ dependencies = [ [[package]] name = "atomic-waker" -version = "1.1.2" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "atomic-write-file" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edcdbedc2236483ab103a53415653d6b4442ea6141baf1ffa85df29635e88436" -dependencies = [ - "nix", - "rand", -] +checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" [[package]] name = "atty" @@ -318,9 +264,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.5" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "base64ct" @@ -357,9 +303,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" dependencies = [ "serde", ] @@ -375,37 +321,36 @@ dependencies = [ [[package]] name = "blocking" -version = "1.5.1" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +checksum = "77231a1c8f801696fc0123ec6150ce92cffb8e164a02afb9c8ddee0e9b65ad65" dependencies = [ - "async-channel 2.1.1", - "async-lock 3.2.0", + "async-channel", + "async-lock", "async-task", - "fastrand 2.0.1", - "futures-io", - "futures-lite 2.1.0", - "piper", - "tracing", + "atomic-waker", + "fastrand 1.9.0", + "futures-lite", + "log", ] [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "bytecount" -version = "0.6.7" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" +checksum = "ad152d03a2c813c80bb94fedbf3a3f02b28f793e39e7c214c8a0bcc196343de7" [[package]] name = "byteorder" -version = "1.5.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" @@ -430,9 +375,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" dependencies = [ "android-tzdata", "iana-time-zone", @@ -461,23 +406,23 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.11" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" dependencies = [ "clap_builder", - "clap_derive 4.4.7", + "clap_derive 4.4.2", ] [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", - "clap_lex 0.6.0", + "clap_lex 0.5.1", "strsim", ] @@ -489,21 +434,21 @@ checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", "syn 1.0.109", ] [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck", - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", - "syn 2.0.41", + "syn 2.0.33", ] [[package]] @@ -517,9 +462,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "codespan-reporting" @@ -539,9 +484,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "concurrent-queue" -version = "2.4.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" dependencies = [ "crossbeam-utils", ] @@ -560,15 +505,15 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.6" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ "core-foundation-sys", "libc", @@ -576,15 +521,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -600,15 +545,25 @@ dependencies = [ [[package]] name = "crc-catalog" -version = "2.4.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -617,21 +572,22 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.16" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", "memoffset", + "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ "cfg-if", "crossbeam-utils", @@ -639,9 +595,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] @@ -663,7 +619,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown 0.14.0", "lock_api", "once_cell", "parking_lot_core", @@ -682,12 +638,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" -dependencies = [ - "powerfmt", -] +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" [[package]] name = "diff" @@ -748,9 +701,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" dependencies = [ "humantime", "is-terminal", @@ -767,12 +720,23 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", "libc", - "windows-sys 0.52.0", ] [[package]] @@ -792,27 +756,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "event-listener" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f2cdcf274580f2d63697192d744727b3198894b1bf02923643bf59e2c26712" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener 4.0.1", - "pin-project-lite", -] - [[package]] name = "fancy-regex" version = "0.11.0" @@ -834,9 +777,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "finl_unicode" @@ -846,12 +789,13 @@ checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" [[package]] name = "flume" -version = "0.11.0" +version = "0.10.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" dependencies = [ "futures-core", "futures-sink", + "pin-project", "spin 0.9.8", ] @@ -878,9 +822,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -897,9 +841,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.29" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -912,9 +856,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -922,15 +866,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -950,9 +894,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-lite" @@ -969,36 +913,23 @@ dependencies = [ "waker-fn", ] -[[package]] -name = "futures-lite" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143" -dependencies = [ - "fastrand 2.0.1", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-channel", "futures-core", @@ -1032,9 +963,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "js-sys", @@ -1045,9 +976,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "glob" @@ -1069,9 +1000,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.22" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -1079,7 +1010,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.1.0", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -1094,9 +1025,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ "ahash", "allocator-api2", @@ -1108,7 +1039,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.0", ] [[package]] @@ -1131,9 +1062,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hex" @@ -1143,9 +1074,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hkdf" -version = "0.12.4" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" dependencies = [ "hmac", ] @@ -1161,11 +1092,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] @@ -1179,9 +1110,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -1190,9 +1121,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", @@ -1219,9 +1150,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -1234,7 +1165,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.5", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -1243,16 +1174,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows", ] [[package]] @@ -1266,9 +1197,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.5.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1301,12 +1232,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.0", ] [[package]] @@ -1327,16 +1258,16 @@ name = "inkwell_internals" version = "0.8.0" source = "git+https://github.com/corbanvilla/inkwell.git?branch=v2.0#583a6c20522cf8a743b9daf214f254424bf14eb2" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", - "syn 2.0.41", + "syn 2.0.33", ] [[package]] name = "insta" -version = "1.34.0" +version = "1.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d64600be34b2fcfc267740a243fa7744441bb4947a619ac4e5bb6507f35fbfc" +checksum = "a0770b0a3d4c70567f0d58331f3088b0e4c4f56c9b8d764efe654b4a5d46de3a" dependencies = [ "console", "lazy_static", @@ -1360,16 +1291,16 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.2", "libc", "windows-sys 0.48.0", ] [[package]] name = "ipnet" -version = "2.9.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" @@ -1377,8 +1308,8 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.3", - "rustix 0.38.28", + "hermit-abi 0.3.2", + "rustix 0.38.13", "windows-sys 0.48.0", ] @@ -1400,26 +1331,17 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" -dependencies = [ - "either", -] - [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -1434,7 +1356,7 @@ dependencies = [ "anyhow", "base64", "bytecount", - "clap 4.4.11", + "clap 4.4.3", "fancy-regex", "fraction", "getrandom", @@ -1474,21 +1396,21 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libm" -version = "0.2.8" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "libsqlite3-sys" -version = "0.27.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" dependencies = [ "cc", "pkg-config", @@ -1509,9 +1431,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "lld_rs" @@ -1528,9 +1450,9 @@ dependencies = [ [[package]] name = "llvm-sys" -version = "140.1.3" +version = "140.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3dc78e9857c0231ec11e3bdccf63870493fdc7d0570b0ea7d50bf5df0cb1a0c" +checksum = "69b285f8682531b9b394dd9891977a2a28c47006e491bda944e1ca62ebab2664" dependencies = [ "cc", "lazy_static", @@ -1541,9 +1463,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -1575,7 +1497,7 @@ checksum = "a1d849148dbaf9661a6151d1ca82b13bb4c4c128146a88d05253b38d4e2f496c" dependencies = [ "beef", "fnv", - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", "regex-syntax 0.6.29", "syn 1.0.109", @@ -1583,19 +1505,18 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.6" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ - "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.6.4" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "memoffset" @@ -1629,9 +1550,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.10" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "wasi", @@ -1656,17 +1577,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "nix" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" -dependencies = [ - "bitflags 2.4.1", - "cfg-if", - "libc", -] - [[package]] name = "nom" version = "7.1.3" @@ -1778,9 +1688,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", "libm", @@ -1792,7 +1702,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.3", + "hermit-abi 0.3.2", "libc", ] @@ -1807,17 +1717,17 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl" -version = "0.10.61" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.0", "cfg-if", "foreign-types", "libc", @@ -1832,9 +1742,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", - "syn 2.0.41", + "syn 2.0.33", ] [[package]] @@ -1845,9 +1755,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.97" +version = "0.9.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" dependencies = [ "cc", "libc", @@ -1857,15 +1767,15 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.6.1" +version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" [[package]] name = "parking" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" [[package]] name = "parking_lot" @@ -1879,9 +1789,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", @@ -1907,9 +1817,29 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.33", +] [[package]] name = "pin-project-lite" @@ -1923,17 +1853,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "piper" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" -dependencies = [ - "atomic-waker", - "fastrand 2.0.1", - "futures-io", -] - [[package]] name = "pkcs1" version = "0.7.5" @@ -1997,7 +1916,7 @@ dependencies = [ "encoding_rs", "encoding_rs_io", "env_logger", - "indexmap 2.1.0", + "indexmap 2.0.0", "insta", "log", "plc_ast", @@ -2048,9 +1967,9 @@ name = "plc_xml" version = "0.1.0" dependencies = [ "html-escape", - "indexmap 2.1.0", + "indexmap 2.0.0", "insta", - "itertools 0.11.0", + "itertools", "logos", "plc_ast", "plc_diagnostics", @@ -2075,26 +1994,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "polling" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf63fa624ab313c11656b4cda960bfc46c410187ad493c41f6ba2d8c1e991c9e" -dependencies = [ - "cfg-if", - "concurrent-queue", - "pin-project-lite", - "rustix 0.38.28", - "tracing", - "windows-sys 0.52.0", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2118,7 +2017,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", "syn 1.0.109", "version_check", @@ -2130,7 +2029,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", "version_check", ] @@ -2146,9 +2045,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -2178,7 +2077,7 @@ version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", ] [[package]] @@ -2213,9 +2112,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -2223,44 +2122,46 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ + "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", + "num_cpus", ] [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.10.2" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.8.2", + "regex-syntax 0.7.5", ] [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.7.5", ] [[package]] @@ -2271,15 +2172,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "base64", "bytes", @@ -2312,14 +2213,16 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.6" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" dependencies = [ + "byteorder", "const-oid", "digest", "num-bigint-dig", "num-integer", + "num-iter", "num-traits", "pkcs1", "pkcs8", @@ -2356,9 +2259,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.27" +version = "0.37.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +checksum = "d4eb579851244c2c03e7c24f501c3432bed80b8f720af1d6e5b0e0f01555a035" dependencies = [ "bitflags 1.3.2", "errno", @@ -2370,15 +2273,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.0", "errno", "libc", - "linux-raw-sys 0.4.12", - "windows-sys 0.52.0", + "linux-raw-sys 0.4.7", + "windows-sys 0.48.0", ] [[package]] @@ -2390,10 +2293,10 @@ dependencies = [ "encoding_rs", "encoding_rs_io", "generational-arena", - "indexmap 2.1.0", + "indexmap 2.0.0", "inkwell", "insta", - "itertools 0.11.0", + "itertools", "lazy_static", "lld_rs", "log", @@ -2422,9 +2325,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "schannel" @@ -2466,35 +2369,35 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", - "syn 2.0.41", + "syn 2.0.33", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -2533,16 +2436,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", - "syn 2.0.41", + "syn 2.0.33", ] [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", @@ -2551,9 +2454,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", @@ -2568,9 +2471,9 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "signature" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" dependencies = [ "digest", "rand_core", @@ -2578,9 +2481,9 @@ dependencies = [ [[package]] name = "similar" -version = "2.3.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597" +checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" [[package]] name = "slab" @@ -2593,15 +2496,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "socket2" -version = "0.4.10" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -2609,9 +2512,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2634,9 +2537,9 @@ dependencies = [ [[package]] name = "spki" -version = "0.7.3" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", "der", @@ -2644,20 +2547,20 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.3" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +checksum = "6b7b278788e7be4d0d29c0f39497a0eef3fba6bbc8e70d8bf7fde46edeaa9e85" dependencies = [ - "itertools 0.12.0", + "itertools", "nom", "unicode_categories", ] [[package]] name = "sqlx" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dba03c279da73694ef99763320dea58b51095dfe87d001b1d4b5fe78ba8763cf" +checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" dependencies = [ "sqlx-core", "sqlx-macros", @@ -2668,12 +2571,12 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" +checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" dependencies = [ "ahash", - "async-io 1.13.0", + "async-io", "async-std", "atoi", "byteorder", @@ -2682,7 +2585,7 @@ dependencies = [ "crossbeam-queue", "dotenvy", "either", - "event-listener 2.5.3", + "event-listener", "futures-channel", "futures-core", "futures-intrusive", @@ -2690,7 +2593,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.1.0", + "indexmap 2.0.0", "log", "memchr", "native-tls", @@ -2709,11 +2612,11 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89961c00dc4d7dffb7aee214964b065072bff69e36ddb9e2c107541f75e4f2a5" +checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", "sqlx-core", "sqlx-macros-core", @@ -2722,18 +2625,17 @@ dependencies = [ [[package]] name = "sqlx-macros-core" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0bd4519486723648186a08785143599760f7cc81c52334a55d6a83ea1e20841" +checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" dependencies = [ "async-std", - "atomic-write-file", "dotenvy", "either", "heck", "hex", "once_cell", - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", "serde", "serde_json", @@ -2748,13 +2650,13 @@ dependencies = [ [[package]] name = "sqlx-mysql" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" +checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" dependencies = [ "atoi", "base64", - "bitflags 2.4.1", + "bitflags 2.4.0", "byteorder", "bytes", "crc", @@ -2790,13 +2692,13 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" +checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" dependencies = [ "atoi", "base64", - "bitflags 2.4.1", + "bitflags 2.4.0", "byteorder", "crc", "dotenvy", @@ -2829,9 +2731,9 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "210976b7d948c7ba9fced8ca835b11cbb2d677c59c79de41ac0d397e14547490" +checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" dependencies = [ "atoi", "flume", @@ -2847,7 +2749,6 @@ dependencies = [ "sqlx-core", "tracing", "url", - "urlencoding", ] [[package]] @@ -2890,27 +2791,27 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.41" +version = "2.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +checksum = "9caece70c63bfba29ec2fed841a09851b14a235c60010fa4de58089b6c025668" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", "unicode-ident", ] [[package]] name = "sysinfo" -version = "0.29.11" +version = "0.29.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" +checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2944,22 +2845,22 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", - "fastrand 2.0.1", + "fastrand 2.0.0", "redox_syscall", - "rustix 0.38.28", + "rustix 0.38.13", "windows-sys 0.48.0", ] [[package]] name = "termcolor" -version = "1.4.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] @@ -2972,32 +2873,31 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", - "syn 2.0.41", + "syn 2.0.33", ] [[package]] name = "time" -version = "0.3.31" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ "deranged", - "powerfmt", "serde", "time-core", "time-macros", @@ -3011,9 +2911,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -3035,9 +2935,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", "bytes", @@ -3045,15 +2945,15 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", - "socket2 0.5.5", + "socket2 0.5.4", "windows-sys 0.48.0", ] [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -3080,10 +2980,11 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ + "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -3092,41 +2993,41 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", - "syn 2.0.41", + "syn 2.0.33", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", ] [[package]] name = "try-lock" -version = "0.2.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" -version = "1.17.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" @@ -3151,9 +3052,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode-xid" @@ -3169,26 +3070,20 @@ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" [[package]] name = "url" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf8-width" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" +checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" [[package]] name = "utf8parse" @@ -3198,15 +3093,15 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.6.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" [[package]] name = "value-bag" -version = "1.4.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" +checksum = "d92ccd67fb88503048c01b59152a04effd0782d035a83a6d256ce6085f08f4a3" [[package]] name = "vcpkg" @@ -3222,9 +3117,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "waker-fn" -version = "1.1.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "want" @@ -3243,9 +3138,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3253,24 +3148,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", - "syn 2.0.41", + "syn 2.0.33", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if", "js-sys", @@ -3280,9 +3175,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote 1.0.33", "wasm-bindgen-macro-support", @@ -3290,28 +3185,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ - "proc-macro2 1.0.70", + "proc-macro2 1.0.67", "quote 1.0.33", - "syn 2.0.41", + "syn 2.0.33", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -3326,7 +3221,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.28", + "rustix 0.38.13", ] [[package]] @@ -3353,9 +3248,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ "winapi", ] @@ -3367,10 +3262,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-core" -version = "0.51.1" +name = "windows" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ "windows-targets 0.48.5", ] @@ -3393,15 +3288,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.0", -] - [[package]] name = "windows-targets" version = "0.42.2" @@ -3432,21 +3318,6 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] -[[package]] -name = "windows-targets" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" -dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -3459,12 +3330,6 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -3477,12 +3342,6 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -3495,12 +3354,6 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" -[[package]] -name = "windows_i686_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -3513,12 +3366,6 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" -[[package]] -name = "windows_i686_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -3531,12 +3378,6 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -3549,12 +3390,6 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -3567,12 +3402,6 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" - [[package]] name = "winreg" version = "0.50.0" @@ -3604,7 +3433,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-std", - "clap 4.4.11", + "clap 4.4.3", "plc_ast", "plc_source", "rusty", @@ -3631,28 +3460,8 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" -[[package]] -name = "zerocopy" -version = "0.7.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" -dependencies = [ - "proc-macro2 1.0.70", - "quote 1.0.33", - "syn 2.0.41", -] - [[package]] name = "zeroize" -version = "1.7.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" From 0303cd4a64786b308841da0050ed59d12f8254dd Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Tue, 16 Jan 2024 10:41:29 +0000 Subject: [PATCH 53/70] remove old package ref --- Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3025e4a0fb1..e5e455fc305 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,8 +77,6 @@ default-members = [".", "compiler/plc_driver", "compiler/plc_xml"] [workspace.dependencies] inkwell = { git = "https://github.com/corbanvilla/inkwell.git", branch = "v2.0", version = "0.2", features = ["llvm14-0"] } -# inkwell = { git = "https://github.com/HaithemLamri/inkwell.git", branch = "master", version = "0.2", default-features = false, features = ["llvm14-0"] } - encoding_rs = "0.8" encoding_rs_io = "0.1" log = "0.4" From e186cfe2cf620cfa00e2e987de9c12e140a0c8c7 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 17 Jan 2024 11:26:53 +0000 Subject: [PATCH 54/70] cleanup interfaces --- compiler/rustc_llvm_coverage/src/ffi.rs | 8 +- compiler/rustc_llvm_coverage/src/interface.rs | 218 ----------- .../rustc_llvm_coverage/src/interfaces.rs | 162 ++++++++ compiler/rustc_llvm_coverage/src/lib.rs | 357 ++++++++++-------- src/codegen.rs | 54 +-- src/codegen/generators/pou_generator.rs | 3 +- 6 files changed, 384 insertions(+), 418 deletions(-) delete mode 100644 compiler/rustc_llvm_coverage/src/interface.rs create mode 100644 compiler/rustc_llvm_coverage/src/interfaces.rs diff --git a/compiler/rustc_llvm_coverage/src/ffi.rs b/compiler/rustc_llvm_coverage/src/ffi.rs index a1ab38da4b3..ccac1390f40 100644 --- a/compiler/rustc_llvm_coverage/src/ffi.rs +++ b/compiler/rustc_llvm_coverage/src/ffi.rs @@ -4,12 +4,12 @@ // // Function interface definitions are taken from [here](https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_llvm/src/llvm/ffi.rs#L1864). // -use libc::{c_char, c_uint, c_void, size_t}; +use libc::{c_char, c_uint, size_t}; use std::slice; use super::types::*; use inkwell::values::PointerValue; -use llvm_sys::prelude::{LLVMModuleRef, LLVMPassManagerRef, LLVMValueRef}; +use llvm_sys::prelude::{LLVMModuleRef, LLVMValueRef}; /// Appending to a Rust string -- used by RawRustStringOstream. #[no_mangle] @@ -54,9 +54,7 @@ extern "C" { pub fn LLVMRustCoverageMappingVersion() -> u32; - // pub fn LLVMRustAddInstrumentationPass(PM: LLVMPassManagerRef); - // pub fn LLVMRustRunInstrumentationPass(M: LLVMModuleRef); - + #[allow(improper_ctypes)] pub fn LLVMRustAppendToUsed(M: LLVMModuleRef, V: PointerValue); } diff --git a/compiler/rustc_llvm_coverage/src/interface.rs b/compiler/rustc_llvm_coverage/src/interface.rs deleted file mode 100644 index 5fc5376899d..00000000000 --- a/compiler/rustc_llvm_coverage/src/interface.rs +++ /dev/null @@ -1,218 +0,0 @@ -use crate::write_filenames_section_to_buffer; - -use super::*; -use inkwell::context::Context; -use inkwell::intrinsics::Intrinsic; -use inkwell::module::Module; -use inkwell::passes::PassBuilderOptions; -use inkwell::targets::{CodeModel, InitializationConfig, RelocMode, Target, TargetMachine, TargetTriple}; -use inkwell::OptimizationLevel; -use std::ffi::CString; - -/// This represents a coverage mapping header that has been written to a module. -/// It is returned for debugging purposes and use with write_function_record. -pub struct WrittenCoverageMappingHeader { - pub mapping_version: u32, - pub filenames: Vec, - pub filenames_hash: u64, -} - -/// filenames: In Coverage Mapping Version > 6, first filename must be the compilation directory -pub fn write_coverage_mapping_header<'ctx>( - module: &Module<'ctx>, - filenames: Vec, -) -> WrittenCoverageMappingHeader { - // Get context - let context = module.get_context(); - - // Convert filenames to CStrings - let filenames_cstr = filenames.clone().into_iter().map(|f| CString::new(f).unwrap()).collect::>(); - let mut encoded_filename_buffer = RustString::new(); - write_filenames_section_to_buffer(&filenames_cstr, &mut encoded_filename_buffer); - let filenames_hash = hash_bytes(encoded_filename_buffer.bytes.borrow().to_vec()); - - // Get values - let mapping_version = mapping_version(); // versions are zero-indexed - assert_eq!(mapping_version, 5, "Only mapping version 6 is supported"); - let encoded_filenames_len = encoded_filename_buffer.len(); - - // Create mapping header types - let i32_type = context.i32_type(); - let i32_zero = i32_type.const_int(0, false); - let i32_cov_mapping_version = i32_type.const_int(mapping_version.into(), false); - let i32_filenames_len = i32_type.const_int(encoded_filenames_len as u64, false); - - // See LLVM Code Coverage Specification for details on this data structure - let cov_mapping_header = context.const_struct( - &[ - // Value 1 : Always zero - i32_zero.into(), - // Value 2 : Len(encoded_filenames) - i32_filenames_len.into(), - // Value 3 : Always zero - i32_zero.into(), - // Value 4 : Mapping version - i32_cov_mapping_version.into(), - ], - // https://github.com/rust-lang/rust/blob/e6707df0de337976dce7577e68fc57adcd5e4842/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs#L301 - false, - ); - - // Create filename value types - let i8_type = context.i8_type(); - let i8_filename_array = i8_type.const_array( - &encoded_filename_buffer - .bytes - .borrow() - .iter() - .map(|byte| i8_type.const_int(*byte as u64, false)) - .collect::>(), - ); - - // Create structure - let coverage_struct = context.const_struct(&[cov_mapping_header.into(), i8_filename_array.into()], false); - - // Write to module - save_cov_data_to_mod(module, coverage_struct); - - // Return header - WrittenCoverageMappingHeader { mapping_version, filenames, filenames_hash } -} - -pub struct FunctionRecord { - name: String, - name_md5_hash: u64, - structural_hash: u64, - virtual_file_mapping: Vec, - expressions: Vec, - mapping_regions: Vec, - mapping_buffer: RustString, - - // A.k.a. hash of all filenames in module - translation_unit_hash: u64, - is_used: bool, -} - -impl FunctionRecord { - pub fn new( - name: String, - structural_hash: u64, - // TODO - better names for these - function_filenames: Vec, - expressions: Vec, - mapping_regions: Vec, - is_used: bool, - - written_mapping_header: &WrittenCoverageMappingHeader, - ) -> Self { - let name_md5_hash = hash_str(&name); - - // Get indexes of function filenames in module file list - // TODO - hoist this into rusty - let mut virtual_file_mapping = Vec::new(); - for filename in function_filenames { - let filename_idx = written_mapping_header - .filenames - .iter() - .position(|f| f == &filename) - .expect("Unable to find function filename in module files"); - virtual_file_mapping.push(filename_idx.try_into().unwrap()); - } - - // Write mapping to buffer - let mut mapping_buffer = RustString::new(); - write_mapping_to_buffer( - virtual_file_mapping.clone(), - expressions.clone(), - mapping_regions.clone(), - &mut mapping_buffer, - ); - - FunctionRecord { - name, - name_md5_hash, - structural_hash, - virtual_file_mapping, - expressions, - is_used, - mapping_regions, - mapping_buffer, - translation_unit_hash: written_mapping_header.filenames_hash, - } - } - - pub fn write_to_module<'ctx>(&self, module: &Module<'ctx>) { - // Get context - let context = module.get_context(); - - // Create types - let i64_type = context.i64_type(); - let i32_type = context.i32_type(); - let i8_type = context.i8_type(); - - // Create values - let i64_name_md5_hash = i64_type.const_int(self.name_md5_hash, false); - let i32_mapping_len = i32_type.const_int(self.mapping_buffer.len() as u64, false); - let i64_structural_hash = i64_type.const_int(self.structural_hash, false); - let i64_translation_unit_hash = i64_type.const_int(self.translation_unit_hash, false); - - // Build mapping array - let i8_mapping_array = i8_type.const_array( - &self - .mapping_buffer - .bytes - .borrow() - .iter() - .map(|byte| i8_type.const_int(*byte as u64, false)) - .collect::>(), - ); - - // Create structure - let function_record_struct = context.const_struct( - &[ - i64_name_md5_hash.into(), - i32_mapping_len.into(), - i64_structural_hash.into(), - i64_translation_unit_hash.into(), - i8_mapping_array.into(), - ], - // https://github.com/rust-lang/rust/blob/e6707df0de337976dce7577e68fc57adcd5e4842/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs#L311 - true, - ); - - save_func_record_to_mod(&module, self.name_md5_hash, function_record_struct, self.is_used); - } -} - -/// This pass will not operate unless the module already has intrinsic calls. -/// See [here](https://github.com/llvm/llvm-project/blob/f28c006a5895fc0e329fe15fead81e37457cb1d1/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp#L539-L549) for why. -pub fn run_instrumentation_lowering_pass<'ctx>(module: &Module<'ctx>) { - // Setup - let context = module.get_context(); - let initialization_config = &InitializationConfig::default(); - inkwell::targets::Target::initialize_all(initialization_config); - - // Architecture Specifics - // Module.set_triple() is required because the pass needs to know it's compiling - // to ELF [here](https://github.com/llvm/llvm-project/blob/cfa30fa4852275eed0c59b81b5d8088d3e55f778/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp#L1191-L1199). - // TODO - pass this as a param - let triple = TargetTriple::create("x86_64-pc-linux-gnu"); - module.set_triple(&triple); - let target = Target::from_triple(&triple).unwrap(); - let machine = target - .create_target_machine( - &triple, - "generic", - "", - OptimizationLevel::None, - RelocMode::Default, - CodeModel::Default, - ) - .unwrap(); - - // Run pass (uses new pass manager) - let _ = module.run_passes("instrprof", &machine, PassBuilderOptions::create()); -} - -// TODO -// - investigate codegen diffs for function/function blocks/programs diff --git a/compiler/rustc_llvm_coverage/src/interfaces.rs b/compiler/rustc_llvm_coverage/src/interfaces.rs new file mode 100644 index 00000000000..dd886938214 --- /dev/null +++ b/compiler/rustc_llvm_coverage/src/interfaces.rs @@ -0,0 +1,162 @@ +/* + * Many of the functions in this file have been adapted from the + * `rustc` implementation of LLVM code coverage. + * + * https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs#L220-L221 + * + * TODO - Consider updating functions to reflect configurations in latest Rust (not 1.64) + * https://github.com/rust-lang/rust/blob/master/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +*/ + +const VAR_ALIGN_BYTES: u32 = 8; + +use std::string::FromUtf8Error; + +use super::*; +use crate::types::*; + +use inkwell::{ + module::Linkage, + values::{AsValueRef, FunctionValue, GlobalValue, StructValue}, + GlobalVisibility, +}; + +use libc::c_uint; +use std::ffi::CString; + +use inkwell::module::Module; + +/* == TODO - Refactor these helpers out */ +pub fn build_string(sr: &RustString) -> Result { + String::from_utf8(sr.bytes.borrow().clone()) +} +/* == END TODO */ + +/// Calls llvm::createPGOFuncNameVar() with the given function instance's +/// mangled function name. The LLVM API returns an llvm::GlobalVariable +/// containing the function name, with the specific variable name and linkage +/// required by LLVM InstrProf source-based coverage instrumentation. Use +/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per +/// `Instance`. +pub fn create_pgo_func_name_var<'ctx>(func: &FunctionValue<'ctx>) -> GlobalValue<'ctx> { + let pgo_function_ref = + unsafe { ffi::LLVMRustCoverageCreatePGOFuncNameVar(func.as_value_ref(), func.get_name().as_ptr()) }; + assert!(!pgo_function_ref.is_null()); + unsafe { GlobalValue::new(pgo_function_ref) } +} + +pub fn write_filenames_section_to_buffer<'a>( + filenames: impl IntoIterator, + buffer: &RustString, +) { + let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::>(); + unsafe { + ffi::LLVMRustCoverageWriteFilenamesSectionToBuffer(c_str_vec.as_ptr(), c_str_vec.len(), buffer); + } +} +//create params , call fucntion in codegen, print the buffer +pub fn write_mapping_to_buffer( + virtual_file_mapping: Vec, + expressions: Vec, + mapping_regions: Vec, + buffer: &mut RustString, +) { + unsafe { + ffi::LLVMRustCoverageWriteMappingToBuffer( + virtual_file_mapping.as_ptr(), + virtual_file_mapping.len() as c_uint, + expressions.as_ptr(), + expressions.len() as c_uint, + mapping_regions.as_ptr(), + mapping_regions.len() as c_uint, + buffer, + ); + } +} + +pub fn hash_str(strval: &str) -> u64 { + let strval = CString::new(strval).expect("null error converting hashable str to C string"); + unsafe { ffi::LLVMRustCoverageHashCString(strval.as_ptr()) } +} + +pub fn hash_bytes(bytes: Vec) -> u64 { + unsafe { ffi::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } +} + +pub fn get_mapping_version() -> u32 { + unsafe { ffi::LLVMRustCoverageMappingVersion() } +} + +pub fn save_cov_data_to_mod<'ctx>(module: &Module<'ctx>, cov_data_val: StructValue<'ctx>) { + let covmap_var_name = { + let mut s = RustString::new(); + unsafe { + ffi::LLVMRustCoverageWriteMappingVarNameToString(&mut s); + } + build_string(&mut s).expect("Rust Coverage Mapping var name failed UTF-8 conversion") + }; + + let covmap_section_name = { + let mut s = RustString::new(); + unsafe { + ffi::LLVMRustCoverageWriteMapSectionNameToString(module.as_mut_ptr(), &mut s); + } + build_string(&mut s).expect("Rust Coverage Mapping section name failed UTF-8 conversion") + }; + + let llglobal = module.add_global(cov_data_val.get_type(), None, covmap_var_name.as_str()); + llglobal.set_initializer(&cov_data_val); + llglobal.set_constant(true); + llglobal.set_linkage(Linkage::Private); + llglobal.set_section(Some(&covmap_section_name)); + llglobal.set_alignment(VAR_ALIGN_BYTES); + + // Mark as used to prevent removal by LLVM optimizations + unsafe { + ffi::LLVMRustAppendToUsed(module.as_mut_ptr(), llglobal.as_pointer_value()); + } +} + +pub fn save_func_record_to_mod<'ctx>( + module: &Module<'ctx>, + func_name_hash: u64, + func_record_val: StructValue<'ctx>, + is_used: bool, +) { + // Assign a name to the function record. This is used to merge duplicates. + // + // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that + // are included-but-not-used. If (or when) Rust generates functions that are + // included-but-not-used, note that a dummy description for a function included-but-not-used + // in a Crate can be replaced by full description provided by a different Crate. The two kinds + // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by + // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. + // TODO - investigate removing this (-Corban) + let func_record_var_name = format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); + + let func_record_section_name = { + let mut s = RustString::new(); + unsafe { + ffi::LLVMRustCoverageWriteFuncSectionNameToString(module.as_mut_ptr(), &mut s); + } + build_string(&mut s).expect("Rust Coverage function record section name failed UTF-8 conversion") + }; + + // Create types + let llglobal = module.add_global(func_record_val.get_type(), None, func_record_var_name.as_str()); + let comdat = module.get_or_insert_comdat(&func_record_var_name); + + // Assign + llglobal.set_initializer(&func_record_val); + llglobal.set_constant(true); + llglobal.set_linkage(Linkage::LinkOnceODR); + llglobal.set_visibility(GlobalVisibility::Hidden); + llglobal.set_section(Some(&func_record_section_name)); + llglobal.set_alignment(VAR_ALIGN_BYTES); + llglobal.set_comdat(comdat); + + // Mark as used to prevent removal by LLVM optimizations + unsafe { + ffi::LLVMRustAppendToUsed(module.as_mut_ptr(), llglobal.as_pointer_value()); + } +} diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index ef7df01d4b4..ba1f302733d 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -1,170 +1,231 @@ -/* - * Many of the functions in this file have been adapted from the - * `rustc` implementation of LLVM code coverage. - * - * https://github.com/rust-lang/rust/blob/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs#L220-L221 - * - * TODO - Consider updating functions to reflect configurations in latest Rust (not 1.64) - * https://github.com/rust-lang/rust/blob/master/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs - */ - -const VAR_ALIGN_BYTES: u32 = 8; - -use std::string::FromUtf8Error; - -mod ffi; -pub mod interface; +//! This library provides a Rust interface to LLVM's coverage mapping format. +//! +//! This module exists to provide intuitive and useful abstractions for +//! interacting with LLVM's coverage mapping functions. If you want to +//! interact directly with LLVM, use the [`interfaces`] or [`ffi`] modules. +//! +//! + +pub mod ffi; +pub mod interfaces; pub mod types; - +use interfaces::*; use types::*; -use inkwell::{ - comdat::*, - intrinsics::Intrinsic, - module::Linkage, - types::{AnyType, AsTypeRef, StructType}, - values::{AsValueRef, FunctionValue, GlobalValue, StructValue}, - GlobalVisibility, -}; - -use libc::c_uint; -use std::ffi::CString; - use inkwell::module::Module; -use inkwell::types::BasicType; -use llvm_sys::comdat::LLVMGetComdat; +use inkwell::passes::PassBuilderOptions; +use inkwell::targets::{CodeModel, InitializationConfig, RelocMode, Target, TargetTriple}; +use inkwell::OptimizationLevel; +use std::ffi::CString; -/* == TODO - Refactor these helpers out */ -pub fn build_string(sr: &RustString) -> Result { - String::from_utf8(sr.bytes.borrow().clone()) -} -/* == END TODO */ - -/// Calls llvm::createPGOFuncNameVar() with the given function instance's -/// mangled function name. The LLVM API returns an llvm::GlobalVariable -/// containing the function name, with the specific variable name and linkage -/// required by LLVM InstrProf source-based coverage instrumentation. Use -/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per -/// `Instance`. -pub fn create_pgo_func_name_var<'ctx>(func: &FunctionValue<'ctx>) -> GlobalValue<'ctx> { - let pgo_function_ref = - unsafe { ffi::LLVMRustCoverageCreatePGOFuncNameVar(func.as_value_ref(), func.get_name().as_ptr()) }; - assert!(!pgo_function_ref.is_null()); - unsafe { GlobalValue::new(pgo_function_ref) } +/// This represents a coverage mapping header that has been written to a module. +/// It is returned for debugging purposes and use with write_function_record. +pub struct CoverageMappingHeader { + pub mapping_version: u32, + pub filenames: Vec, + pub filenames_hash: u64, + pub encoded_filename_buffer: RustString, } -pub fn write_filenames_section_to_buffer<'a>( - filenames: impl IntoIterator, - buffer: &RustString, -) { - let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::>(); - unsafe { - ffi::LLVMRustCoverageWriteFilenamesSectionToBuffer(c_str_vec.as_ptr(), c_str_vec.len(), buffer); +impl CoverageMappingHeader { + pub fn new(filenames: Vec) -> Self { + // Get mapping version from LLVM + let mapping_version = get_mapping_version(); // versions are zero-indexed + assert_eq!(mapping_version, 5, "Only mapping version 6 is supported"); + + // Convert filenames to CStrings + let filenames_cstr = + filenames.clone().into_iter().map(|f| CString::new(f).unwrap()).collect::>(); + let mut encoded_filename_buffer = RustString::new(); + write_filenames_section_to_buffer(&filenames_cstr, &mut encoded_filename_buffer); + + // Calc file hash + let filenames_hash = hash_bytes(encoded_filename_buffer.bytes.borrow().to_vec()); + + CoverageMappingHeader { mapping_version, filenames, filenames_hash, encoded_filename_buffer } } -} -//create params , call fucntion in codegen, print the buffer -pub fn write_mapping_to_buffer( - virtual_file_mapping: Vec, - expressions: Vec, - mapping_regions: Vec, - buffer: &mut RustString, -) { - unsafe { - ffi::LLVMRustCoverageWriteMappingToBuffer( - virtual_file_mapping.as_ptr(), - virtual_file_mapping.len() as c_uint, - expressions.as_ptr(), - expressions.len() as c_uint, - mapping_regions.as_ptr(), - mapping_regions.len() as c_uint, - buffer, + + /// filenames: In Coverage Mapping Version > 6, first filename must be the compilation directory + pub fn write_coverage_mapping_header<'ctx>(&self, module: &Module<'ctx>) { + // Get context + let context = module.get_context(); + + // Create mapping header types + let i32_type = context.i32_type(); + let i32_zero = i32_type.const_int(0, false); + let i32_cov_mapping_version = i32_type.const_int(self.mapping_version.into(), false); + let i32_filenames_len = i32_type.const_int(self.encoded_filename_buffer.len() as u64, false); + + // See LLVM Code Coverage Specification for details on this data structure + let cov_mapping_header = context.const_struct( + &[ + // Value 1 : Always zero + i32_zero.into(), + // Value 2 : Len(encoded_filenames) + i32_filenames_len.into(), + // Value 3 : Always zero + i32_zero.into(), + // Value 4 : Mapping version + i32_cov_mapping_version.into(), + ], + // https://github.com/rust-lang/rust/blob/e6707df0de337976dce7577e68fc57adcd5e4842/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs#L301 + false, ); - } -} -pub fn hash_str(strval: &str) -> u64 { - let strval = CString::new(strval).expect("null error converting hashable str to C string"); - unsafe { ffi::LLVMRustCoverageHashCString(strval.as_ptr()) } -} + // Create filename value types + let i8_type = context.i8_type(); + let i8_filename_array = i8_type.const_array( + &self + .encoded_filename_buffer + .bytes + .borrow() + .iter() + .map(|byte| i8_type.const_int(*byte as u64, false)) + .collect::>(), + ); + + // Create structure + let coverage_struct = + context.const_struct(&[cov_mapping_header.into(), i8_filename_array.into()], false); -pub fn hash_bytes(bytes: Vec) -> u64 { - unsafe { ffi::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } + // Write to module + save_cov_data_to_mod(module, coverage_struct); + } } -pub fn mapping_version() -> u32 { - unsafe { ffi::LLVMRustCoverageMappingVersion() } +pub struct FunctionRecord { + pub name: String, + pub name_md5_hash: u64, + pub structural_hash: u64, + pub virtual_file_mapping: Vec, + pub expressions: Vec, + pub mapping_regions: Vec, + pub mapping_buffer: RustString, + + // A.k.a. hash of all filenames in module + pub translation_unit_hash: u64, + pub is_used: bool, } -pub fn save_cov_data_to_mod<'ctx>(module: &Module<'ctx>, cov_data_val: StructValue<'ctx>) { - let covmap_var_name = { - let mut s = RustString::new(); - unsafe { - ffi::LLVMRustCoverageWriteMappingVarNameToString(&mut s); +impl FunctionRecord { + pub fn new( + name: String, + structural_hash: u64, + // TODO - better names for these + function_filenames: Vec, + expressions: Vec, + mapping_regions: Vec, + is_used: bool, + + written_mapping_header: &CoverageMappingHeader, + ) -> Self { + let name_md5_hash = hash_str(&name); + + // Get indexes of function filenames in module file list + // TODO - hoist this into rusty + let mut virtual_file_mapping = Vec::new(); + for filename in function_filenames { + let filename_idx = written_mapping_header + .filenames + .iter() + .position(|f| f == &filename) + .expect("Unable to find function filename in module files"); + virtual_file_mapping.push(filename_idx.try_into().unwrap()); } - build_string(&mut s).expect("Rust Coverage Mapping var name failed UTF-8 conversion") - }; - let covmap_section_name = { - let mut s = RustString::new(); - unsafe { - ffi::LLVMRustCoverageWriteMapSectionNameToString(module.as_mut_ptr(), &mut s); + // Write mapping to buffer + let mut mapping_buffer = RustString::new(); + write_mapping_to_buffer( + virtual_file_mapping.clone(), + expressions.clone(), + mapping_regions.clone(), + &mut mapping_buffer, + ); + + FunctionRecord { + name, + name_md5_hash, + structural_hash, + virtual_file_mapping, + expressions, + is_used, + mapping_regions, + mapping_buffer, + translation_unit_hash: written_mapping_header.filenames_hash, } - build_string(&mut s).expect("Rust Coverage Mapping section name failed UTF-8 conversion") - }; - - let llglobal = module.add_global(cov_data_val.get_type(), None, covmap_var_name.as_str()); - llglobal.set_initializer(&cov_data_val); - llglobal.set_constant(true); - llglobal.set_linkage(Linkage::Private); - llglobal.set_section(Some(&covmap_section_name)); - llglobal.set_alignment(VAR_ALIGN_BYTES); - - // Mark as used to prevent removal by LLVM optimizations - unsafe { - ffi::LLVMRustAppendToUsed(module.as_mut_ptr(), llglobal.as_pointer_value()); } -} -pub fn save_func_record_to_mod<'ctx>( - module: &Module<'ctx>, - func_name_hash: u64, - func_record_val: StructValue<'ctx>, - is_used: bool, -) { - // Assign a name to the function record. This is used to merge duplicates. - // - // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that - // are included-but-not-used. If (or when) Rust generates functions that are - // included-but-not-used, note that a dummy description for a function included-but-not-used - // in a Crate can be replaced by full description provided by a different Crate. The two kinds - // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by - // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. - // TODO - investigate removing this (-Corban) - let func_record_var_name = format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); - - let func_record_section_name = { - let mut s = RustString::new(); - unsafe { - ffi::LLVMRustCoverageWriteFuncSectionNameToString(module.as_mut_ptr(), &mut s); - } - build_string(&mut s).expect("Rust Coverage function record section name failed UTF-8 conversion") - }; - - // Create types - let llglobal = module.add_global(func_record_val.get_type(), None, func_record_var_name.as_str()); - let comdat = module.get_or_insert_comdat(&func_record_var_name); - - // Assign - llglobal.set_initializer(&func_record_val); - llglobal.set_constant(true); - llglobal.set_linkage(Linkage::LinkOnceODR); - llglobal.set_visibility(GlobalVisibility::Hidden); - llglobal.set_section(Some(&func_record_section_name)); - llglobal.set_alignment(VAR_ALIGN_BYTES); - llglobal.set_comdat(comdat); - - // Mark as used to prevent removal by LLVM optimizations - unsafe { - ffi::LLVMRustAppendToUsed(module.as_mut_ptr(), llglobal.as_pointer_value()); + pub fn write_to_module<'ctx>(&self, module: &Module<'ctx>) { + // Get context + let context = module.get_context(); + + // Create types + let i64_type = context.i64_type(); + let i32_type = context.i32_type(); + let i8_type = context.i8_type(); + + // Create values + let i64_name_md5_hash = i64_type.const_int(self.name_md5_hash, false); + let i32_mapping_len = i32_type.const_int(self.mapping_buffer.len() as u64, false); + let i64_structural_hash = i64_type.const_int(self.structural_hash, false); + let i64_translation_unit_hash = i64_type.const_int(self.translation_unit_hash, false); + + // Build mapping array + let i8_mapping_array = i8_type.const_array( + &self + .mapping_buffer + .bytes + .borrow() + .iter() + .map(|byte| i8_type.const_int(*byte as u64, false)) + .collect::>(), + ); + + // Create structure + let function_record_struct = context.const_struct( + &[ + i64_name_md5_hash.into(), + i32_mapping_len.into(), + i64_structural_hash.into(), + i64_translation_unit_hash.into(), + i8_mapping_array.into(), + ], + // https://github.com/rust-lang/rust/blob/e6707df0de337976dce7577e68fc57adcd5e4842/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs#L311 + true, + ); + + save_func_record_to_mod(&module, self.name_md5_hash, function_record_struct, self.is_used); } } + +/// This pass will not operate unless the module already has intrinsic calls. +/// See [here](https://github.com/llvm/llvm-project/blob/f28c006a5895fc0e329fe15fead81e37457cb1d1/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp#L539-L549) for why. +pub fn run_instrumentation_lowering_pass<'ctx>(module: &Module<'ctx>) { + // Setup + let initialization_config = &InitializationConfig::default(); + inkwell::targets::Target::initialize_all(initialization_config); + + // Architecture Specifics + // Module.set_triple() is required because the pass needs to know it's compiling + // to ELF [here](https://github.com/llvm/llvm-project/blob/cfa30fa4852275eed0c59b81b5d8088d3e55f778/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp#L1191-L1199). + // TODO - pass this as a param + let triple = TargetTriple::create("x86_64-pc-linux-gnu"); + module.set_triple(&triple); + let target = Target::from_triple(&triple).unwrap(); + let machine = target + .create_target_machine( + &triple, + "generic", + "", + OptimizationLevel::None, + RelocMode::Default, + CodeModel::Default, + ) + .unwrap(); + + // Run pass (uses new pass manager) + let _ = module.run_passes("instrprof", &machine, PassBuilderOptions::create()); +} + +// TODO +// - investigate codegen diffs for function/function blocks/programs diff --git a/src/codegen.rs b/src/codegen.rs index 98da201a883..f334a45475e 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -1,7 +1,6 @@ // Copyright (c) 2020 Ghaith Hachem and Mathias Rieder use std::{ cell::RefCell, - ffi::CString, ops::Deref, path::{Path, PathBuf}, }; @@ -15,7 +14,6 @@ use self::{ pou_generator::{self, PouGenerator}, variable_generator::VariableGenerator, }, - instrument::InstrumentBuilder, llvm_index::LlvmTypedIndex, }; use crate::{ @@ -34,16 +32,13 @@ use inkwell::{ }; use inkwell::{ module::Module, - passes::{PassBuilderOptions, PassManager, PassManagerBuilder}, + passes::PassBuilderOptions, targets::{CodeModel, FileType, InitializationConfig, RelocMode}, }; use plc_ast::ast::{CompilationUnit, LinkageType}; use plc_diagnostics::diagnostics::Diagnostic; use plc_source::source_location::SourceLocation; -use rustc_llvm_coverage::{ - self, - types::{CounterExpression, CounterMappingRegion}, -}; +use rustc_llvm_coverage::types::CounterMappingRegion; mod debug; pub(crate) mod generators; @@ -104,20 +99,6 @@ impl<'ink> CodeGen<'ink> { let module = context.create_module(module_location); module.set_source_file_name(module_location); let debug = debug::DebugBuilderEnum::new(context, &module, root, optimization_level, debug_level); - - // let mut pm = PassManager::create(()); - - // let pass_manager_builder = PassManagerBuilder::create(); - // pass_manager_builder.populate_module_pass_manager(&pm); - - // unsafe { - // rustc_llvm_coverage::LLVMRustAddInstrumentationPass(pm.as_mut_ptr()); - // } - // let did_init = pm.initialize(); - // println!("Did init: {}", did_init); - // let did_finalize = pm.finalize(); - // println!("Did finalize: {:?}", did_finalize); - CodeGen { module, debug, module_location: module_location.to_string() } } @@ -236,9 +217,11 @@ impl<'ink> CodeGen<'ink> { self.debug.finalize(); log::debug!("{}", self.module.to_string()); + // TODO - configure cov builder here let filenames = vec!["/workspaces/corbanvilla_rusty".to_string(), self.module_location.clone()]; - let cov_header = - rustc_llvm_coverage::interface::write_coverage_mapping_header(&self.module, filenames); + let cov_header = rustc_llvm_coverage::CoverageMappingHeader::new(filenames); + cov_header.write_coverage_mapping_header(&self.module); + // let cov_header = rustc_llvm_coverage::write_coverage_mapping_header(&self.module, filenames); let prg_func = self.module.get_function("main").expect("Unable to get prg"); @@ -246,7 +229,7 @@ impl<'ink> CodeGen<'ink> { let mapping_regions: Vec = vec![CounterMappingRegion::code_region(counter1, 1, 1, 1, 2, 3)]; - let func_record = rustc_llvm_coverage::interface::FunctionRecord::new( + let func_record = rustc_llvm_coverage::FunctionRecord::new( "main".to_string(), 1, vec![self.module_location.clone()], @@ -258,7 +241,7 @@ impl<'ink> CodeGen<'ink> { func_record.write_to_module(&self.module); - rustc_llvm_coverage::interface::run_instrumentation_lowering_pass(&self.module); + rustc_llvm_coverage::run_instrumentation_lowering_pass(&self.module); #[cfg(feature = "verify")] { @@ -375,8 +358,6 @@ impl<'ink> GeneratedModule<'ink> { if let Some(parent) = output.parent() { std::fs::create_dir_all(parent)?; } - // Log passes - println!("Optimization level: {:?}", optimization_level.opt_params()); ////Run the passes machine .and_then(|it| { @@ -470,25 +451,6 @@ impl<'ink> GeneratedModule<'ink> { log::debug!("Output location: {}", output.to_string_lossy()); log::debug!("{}", self.persist_to_string()); - println!("Writing to IR"); - - // rustc_llvm_coverage::interface::run_legacy_coverage_pass(&self.module); - - // let pm = PassManager::create(()); - // unsafe { - // rustc_llvm_coverage::LLVMRustAddInstrumentationPass(pm.as_mut_ptr()); - // } - // let did_run = pm.run_on(&self.module); - // println!("Did run: {}", did_run); - // let did_init = pm.initialize(); - // println!("Did init: {}", did_init); - // let did_finalize = pm.finalize(); - // println!("Did finalize: {:?}", did_finalize); - - // unsafe { - // rustc_llvm_coverage::llvmrustruninstrumentationpass(self.module.as_mut_ptr()); - // } - self.module .print_to_file(&output) .map_err(|err| { diff --git a/src/codegen/generators/pou_generator.rs b/src/codegen/generators/pou_generator.rs index c53736e55b8..e803c78dc5f 100644 --- a/src/codegen/generators/pou_generator.rs +++ b/src/codegen/generators/pou_generator.rs @@ -395,7 +395,8 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { { let increment_intrinsic = Intrinsic::find("llvm.instrprof.increment").unwrap(); let increment_intrinsic_func = increment_intrinsic.get_declaration(module, &[]).unwrap(); - let pgo_func_var = rustc_llvm_coverage::create_pgo_func_name_var(¤t_function); + let pgo_func_var = + rustc_llvm_coverage::interfaces::create_pgo_func_name_var(¤t_function); // Create types let i64_type = self.llvm.context.i64_type(); From a54ce1ffb0faf9a33801006c08a0762472ee7385 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 17 Jan 2024 13:55:39 +0000 Subject: [PATCH 55/70] added emit to interfaces --- compiler/rustc_llvm_coverage/src/lib.rs | 39 +++++++++++++++++++++++++ src/codegen/generators/pou_generator.rs | 37 ++++++++++++++--------- 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index ba1f302733d..06f45c09073 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -9,12 +9,16 @@ pub mod ffi; pub mod interfaces; pub mod types; +use interfaces::create_pgo_func_name_var; use interfaces::*; use types::*; +use inkwell::builder::Builder; +use inkwell::intrinsics::Intrinsic; use inkwell::module::Module; use inkwell::passes::PassBuilderOptions; use inkwell::targets::{CodeModel, InitializationConfig, RelocMode, Target, TargetTriple}; +use inkwell::values::PointerValue; use inkwell::OptimizationLevel; use std::ffi::CString; @@ -108,6 +112,8 @@ pub struct FunctionRecord { } impl FunctionRecord { + /// TODO - Update to use a filename table, like + /// https://github.com/rust-lang/rust/blob/e6707df0de337976dce7577e68fc57adcd5e4842/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs#L155-L194 pub fn new( name: String, structural_hash: u64, @@ -227,5 +233,38 @@ pub fn run_instrumentation_lowering_pass<'ctx>(module: &Module<'ctx>) { let _ = module.run_passes("instrprof", &machine, PassBuilderOptions::create()); } +/// Emits a increment counter call at the current builder position. +/// +/// `pgo_function_var` is a pointer to the function's global name variable, +/// generated from [`create_pgo_func_name_var`]. +/// +/// TODO - verify the correctness of these lifetimes. +pub fn emit_counter_increment<'ink, 'ctx>( + builder: &Builder<'ink>, + module: &Module<'ctx>, + pgo_function_var: &PointerValue<'ink>, + structural_hash: u64, + num_counters: u32, + counter_idx: u64, +) { + let context = module.get_context(); + let increment_intrinsic = Intrinsic::find("llvm.instrprof.increment").unwrap(); + let increment_intrinsic_func = increment_intrinsic.get_declaration(module, &[]).unwrap(); + + // Create types + let i64_type = context.i64_type(); + let i32_type = context.i32_type(); + + let i64_hash = i64_type.const_int(structural_hash, false); + let i32_num_counters = i32_type.const_int(num_counters.into(), false); + let i64_counter_idx = i64_type.const_int(counter_idx, false); + + builder.build_call( + increment_intrinsic_func, + &[(*pgo_function_var).into(), i64_hash.into(), i32_num_counters.into(), i64_counter_idx.into()], + "increment_call", + ); +} + // TODO // - investigate codegen diffs for function/function blocks/programs diff --git a/src/codegen/generators/pou_generator.rs b/src/codegen/generators/pou_generator.rs index e803c78dc5f..e9b0a547c1b 100644 --- a/src/codegen/generators/pou_generator.rs +++ b/src/codegen/generators/pou_generator.rs @@ -393,25 +393,34 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { // This is where we generate the actual statements, but AFTER we initialize locals // TODO - add increments { - let increment_intrinsic = Intrinsic::find("llvm.instrprof.increment").unwrap(); - let increment_intrinsic_func = increment_intrinsic.get_declaration(module, &[]).unwrap(); + // let increment_intrinsic = Intrinsic::find("llvm.instrprof.increment").unwrap(); + // let increment_intrinsic_func = increment_intrinsic.get_declaration(module, &[]).unwrap(); let pgo_func_var = rustc_llvm_coverage::interfaces::create_pgo_func_name_var(¤t_function); - // Create types - let i64_type = self.llvm.context.i64_type(); - let i32_type = self.llvm.context.i32_type(); + rustc_llvm_coverage::emit_counter_increment( + &self.llvm.builder, + &module, + &pgo_func_var.as_pointer_value(), + 1, + 1, + 0, + ); - let i8_name_ptr = pgo_func_var.as_pointer_value(); - let i64_hash = i64_type.const_int(1, false); - let i32_num_counters = i32_type.const_int(1, false); - let i64_counter_idx = i64_type.const_int(0, false); + // // Create types + // let i64_type = self.llvm.context.i64_type(); + // let i32_type = self.llvm.context.i32_type(); - self.llvm.builder.build_call( - increment_intrinsic_func, - &[i8_name_ptr.into(), i64_hash.into(), i32_num_counters.into(), i64_counter_idx.into()], - "increment_call", - ); + // let i8_name_ptr = pgo_func_var.as_pointer_value(); + // let i64_hash = i64_type.const_int(1, false); + // let i32_num_counters = i32_type.const_int(1, false); + // let i64_counter_idx = i64_type.const_int(0, false); + + // self.llvm.builder.build_call( + // increment_intrinsic_func, + // &[i8_name_ptr.into(), i64_hash.into(), i32_num_counters.into(), i64_counter_idx.into()], + // "increment_call", + // ); } statement_gen.generate_body(&implementation.statements)?; statement_gen.generate_return_statement()?; From db0c56d877ffe9a3cbe0e3584244f24a3a1ce9c2 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Thu, 18 Jan 2024 11:36:19 +0000 Subject: [PATCH 56/70] function level code coverage --- compiler/plc_source/src/source_location.rs | 12 ++ src/codegen.rs | 65 +++++------ src/codegen/generators/pou_generator.rs | 35 ++---- src/codegen/instrument.rs | 128 +++++++++++++++++++-- 4 files changed, 172 insertions(+), 68 deletions(-) diff --git a/compiler/plc_source/src/source_location.rs b/compiler/plc_source/src/source_location.rs index beafedd4d77..0ac463b9127 100644 --- a/compiler/plc_source/src/source_location.rs +++ b/compiler/plc_source/src/source_location.rs @@ -296,6 +296,18 @@ impl SourceLocation { true } } + + // TOOD - there's probably a better way to do this + pub fn get_start_end(&self) -> (usize, usize, usize, usize) { + let span = self.get_span(); + if let CodeSpan::Range(range) = span { + let (start_line, start_col, end_line, end_col) = + (range.start.line, range.start.column, range.end.line, range.end.column); + (start_line, start_col, end_line, end_col) + } else { + panic!("Error: expected CodeSpan::Range, found {:?}", span); + } + } } /** diff --git a/src/codegen.rs b/src/codegen.rs index f334a45475e..4a03891a08e 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -35,6 +35,7 @@ use inkwell::{ passes::PassBuilderOptions, targets::{CodeModel, FileType, InitializationConfig, RelocMode}, }; +use instrument::CoverageInstrumentationBuilder; use plc_ast::ast::{CompilationUnit, LinkageType}; use plc_diagnostics::diagnostics::Diagnostic; use plc_source::source_location::SourceLocation; @@ -47,8 +48,6 @@ mod llvm_index; mod llvm_typesystem; #[cfg(test)] mod tests; -use rustc_llvm_coverage::*; -use types::*; /// A wrapper around the LLVM context to allow passing it without exposing the inkwell dependencies pub struct CodegenContext(Context); @@ -74,8 +73,8 @@ pub struct CodeGen<'ink> { pub module: Module<'ink>, /// the debugging module creates debug information at appropriate locations pub debug: DebugBuilderEnum<'ink>, - /// the instrumentation builder (to be later hoisted out of the codegen struct) - // pub instrument: Option< + /// the instrumentation builder (possibly later hoisted out of the codegen struct) + pub instrument: Option>, pub module_location: String, } @@ -99,7 +98,18 @@ impl<'ink> CodeGen<'ink> { let module = context.create_module(module_location); module.set_source_file_name(module_location); let debug = debug::DebugBuilderEnum::new(context, &module, root, optimization_level, debug_level); - CodeGen { module, debug, module_location: module_location.to_string() } + + // TODO - disable instr here + // TODO - local path shouldn't be hardcoded + let filenames = vec!["/workspaces/corbanvilla_rusty".to_string(), module_location.to_string()]; + let instr_builder = instrument::CoverageInstrumentationBuilder::new(context, filenames); + + CodeGen { + module, + debug, + instrument: Some(instr_builder), + module_location: module_location.to_string(), + } } pub fn generate_llvm_index( @@ -193,7 +203,7 @@ impl<'ink> CodeGen<'ink> { /// generates all TYPEs, GLOBAL-sections and POUs of the given CompilationUnit pub fn generate( - self, + mut self, context: &'ink CodegenContext, unit: &CompilationUnit, annotations: &AstAnnotations, @@ -204,12 +214,24 @@ impl<'ink> CodeGen<'ink> { let llvm = Llvm::new(context, context.create_builder()); let pou_generator = PouGenerator::new(llvm, global_index, annotations, llvm_index); + if let Some(instr_builder) = &mut self.instrument { + // Generate mapping header + instr_builder.initialize(&self.module); + + instr_builder.create_function_records(unit, llvm_index, &self.module); + } + //Generate the POU stubs in the first go to make sure they can be referenced. for implementation in &unit.implementations { //Don't generate external or generic functions if let Some(entry) = global_index.find_pou(implementation.name.as_str()) { if !entry.is_generic() && entry.get_linkage() != &LinkageType::External { - pou_generator.generate_implementation(implementation, &self.debug, &self.module)?; + pou_generator.generate_implementation( + implementation, + &self.debug, + &self.module, + &self.instrument, + )?; } } } @@ -217,31 +239,10 @@ impl<'ink> CodeGen<'ink> { self.debug.finalize(); log::debug!("{}", self.module.to_string()); - // TODO - configure cov builder here - let filenames = vec!["/workspaces/corbanvilla_rusty".to_string(), self.module_location.clone()]; - let cov_header = rustc_llvm_coverage::CoverageMappingHeader::new(filenames); - cov_header.write_coverage_mapping_header(&self.module); - // let cov_header = rustc_llvm_coverage::write_coverage_mapping_header(&self.module, filenames); - - let prg_func = self.module.get_function("main").expect("Unable to get prg"); - - let counter1 = Counter::counter_value_reference(CounterId::new(0)); - let mapping_regions: Vec = - vec![CounterMappingRegion::code_region(counter1, 1, 1, 1, 2, 3)]; - - let func_record = rustc_llvm_coverage::FunctionRecord::new( - "main".to_string(), - 1, - vec![self.module_location.clone()], - Vec::new(), - mapping_regions, - true, - &cov_header, - ); - - func_record.write_to_module(&self.module); - - rustc_llvm_coverage::run_instrumentation_lowering_pass(&self.module); + // Run the pass + if let Some(instr_builder) = &mut self.instrument { + instr_builder.finalize(&self.module); + } #[cfg(feature = "verify")] { diff --git a/src/codegen/generators/pou_generator.rs b/src/codegen/generators/pou_generator.rs index e9b0a547c1b..e55faa0fff2 100644 --- a/src/codegen/generators/pou_generator.rs +++ b/src/codegen/generators/pou_generator.rs @@ -10,6 +10,7 @@ use super::{ use crate::{ codegen::{ debug::{Debug, DebugBuilderEnum}, + instrument::CoverageInstrumentationBuilder, llvm_index::LlvmTypedIndex, }, index::{self, ImplementationType}, @@ -283,6 +284,7 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { implementation: &Implementation, debug: &DebugBuilderEnum<'ink>, module: &Module<'ctx>, + instrumentation: &Option>, ) -> Result<(), Diagnostic> { let context = self.llvm.context; let mut local_index = LlvmTypedIndex::create_child(self.llvm_index); @@ -390,38 +392,17 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { &function_context, debug, ); - // This is where we generate the actual statements, but AFTER we initialize locals - // TODO - add increments - { - // let increment_intrinsic = Intrinsic::find("llvm.instrprof.increment").unwrap(); - // let increment_intrinsic_func = increment_intrinsic.get_declaration(module, &[]).unwrap(); - let pgo_func_var = - rustc_llvm_coverage::interfaces::create_pgo_func_name_var(¤t_function); - rustc_llvm_coverage::emit_counter_increment( + // Emit instrumentation before funciton body + if let Some(instr_builder) = instrumentation { + instr_builder.emit_function_increment( &self.llvm.builder, - &module, - &pgo_func_var.as_pointer_value(), - 1, - 1, + module, + current_function.get_name().to_str().unwrap(), 0, ); - - // // Create types - // let i64_type = self.llvm.context.i64_type(); - // let i32_type = self.llvm.context.i32_type(); - - // let i8_name_ptr = pgo_func_var.as_pointer_value(); - // let i64_hash = i64_type.const_int(1, false); - // let i32_num_counters = i32_type.const_int(1, false); - // let i64_counter_idx = i64_type.const_int(0, false); - - // self.llvm.builder.build_call( - // increment_intrinsic_func, - // &[i8_name_ptr.into(), i64_hash.into(), i32_num_counters.into(), i64_counter_idx.into()], - // "increment_call", - // ); } + statement_gen.generate_body(&implementation.statements)?; statement_gen.generate_return_statement()?; } diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index bd3b3b95a8d..76aca51a2b3 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -1,19 +1,129 @@ +use super::LlvmTypedIndex; +use inkwell::builder::Builder; use inkwell::context::Context; -use rustc_llvm_coverage; +use inkwell::module::Module; +use inkwell::values::{FunctionValue, GlobalValue}; +use plc_ast::ast::CompilationUnit; +use plc_source::source_location::CodeSpan; +use rustc_llvm_coverage::types::{Counter, CounterId, CounterMappingRegion}; +use rustc_llvm_coverage::*; +use std::collections::HashMap; use std::ffi::CString; -pub struct InstrumentBuilder<'ink> { +pub struct CoverageInstrumentationBuilder<'ink> { context: &'ink Context, - files: Vec<&'ink str>, + files: Vec, + cov_mapping_header: Option, + function_pgos: HashMap)>, } -impl<'ink> InstrumentBuilder<'ink> { - pub fn new(context: &'ink Context, file_name: &'ink str) -> Self { - Self { context, files: vec![file_name] } +impl<'ink> CoverageInstrumentationBuilder<'ink> { + pub fn new(context: &'ink Context, files: Vec) -> Self { + Self { context, files, cov_mapping_header: None, function_pgos: HashMap::new() } } - pub fn write_header(&self) { - // Filenames - let cstring_filenames = self.files.iter().map(|f| CString::new(*f).unwrap()).collect::>(); + pub fn initialize(&mut self, module: &Module<'ink>) { + let cov_mapping_header = CoverageMappingHeader::new(self.files.clone()); + cov_mapping_header.write_coverage_mapping_header(module); + self.cov_mapping_header = Some(cov_mapping_header); + } + + pub fn create_function_records( + &mut self, + unit: &CompilationUnit, + llvm_index: &LlvmTypedIndex, + module: &Module<'ink>, + ) { + // Keep records + let mut function_records = Vec::new(); + + // Loop through functions in AST, create function records + for implementation in &unit.implementations { + let func_name = implementation.name.clone(); + // TODO - hash strucutrally + let struct_hash = rustc_llvm_coverage::interfaces::hash_str(&func_name); + // TODO - modify this? + let func_filenames = vec![unit.file_name.clone()]; + // TODO - use expression counters + let expressions = Vec::new(); + // TODO - file mapping table + let file_id = 1; + + // TODO - loop through dfs + let mut mapping_regions = Vec::new(); + { + let (start_line, start_col, end_line, end_col) = implementation.location.get_start_end(); + let counter = Counter::counter_value_reference(CounterId::new(0)); + let mapping_region = CounterMappingRegion::code_region( + counter, + file_id, + start_line.try_into().unwrap(), + start_col.try_into().unwrap(), + end_line.try_into().unwrap(), + end_col.try_into().unwrap(), + ); + mapping_regions.push(mapping_region); + } + + let is_used = true; + + let written_coverage_header = &self.cov_mapping_header.as_mut().unwrap(); + + let func = FunctionRecord::new( + func_name, + struct_hash, + func_filenames, + expressions, + mapping_regions, + is_used, + &written_coverage_header, + ); + + func.write_to_module(module); + + function_records.push(func); + } + + // Loop through LLVM definitions, create PGO vars + for function_record in function_records { + let func_name = function_record.name.clone(); + + // TODO - decide whether or not this needs to come from the module + // let func = llvm_index + // .find_associated_implementation(&func_name) + // .expect("Function not found in LLVM index"); + let func = module.get_function(&func_name).expect("Function not found in module!"); + + let func_pgo = rustc_llvm_coverage::interfaces::create_pgo_func_name_var(&func); + + &self.function_pgos.insert(func_name, (function_record, func_pgo)); + } + } + + pub fn emit_function_increment<'ctx>( + &self, + builder: &Builder<'ink>, + module: &Module<'ctx>, + func_name: &str, + counter_index: u64, + ) { + let (func_record, func_pgo_var) = self.function_pgos.get(func_name).unwrap(); + + // TODO - see if expressions need different num_counters + let pgo_pointer = func_pgo_var.as_pointer_value(); + let num_counters = func_record.mapping_regions.len(); + + rustc_llvm_coverage::emit_counter_increment( + builder, + module, + &pgo_pointer, + func_record.structural_hash, + num_counters.try_into().unwrap(), + counter_index, + ); + } + + pub fn finalize(&mut self, module: &Module<'ink>) { + run_instrumentation_lowering_pass(module); } } From 6b38d51ed0ef6b794ce2c6a37a1ce63c4672631b Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Sat, 20 Jan 2024 10:51:15 +0000 Subject: [PATCH 57/70] only instrument defined functions --- src/codegen/instrument.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index 76aca51a2b3..3e5096efb2d 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -3,7 +3,7 @@ use inkwell::builder::Builder; use inkwell::context::Context; use inkwell::module::Module; use inkwell::values::{FunctionValue, GlobalValue}; -use plc_ast::ast::CompilationUnit; +use plc_ast::ast::{CompilationUnit, LinkageType}; use plc_source::source_location::CodeSpan; use rustc_llvm_coverage::types::{Counter, CounterId, CounterMappingRegion}; use rustc_llvm_coverage::*; @@ -39,6 +39,11 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { // Loop through functions in AST, create function records for implementation in &unit.implementations { + // Skip non-internal functions (external links + built-ins) + if implementation.linkage != LinkageType::Internal { + continue; + } + let func_name = implementation.name.clone(); // TODO - hash strucutrally let struct_hash = rustc_llvm_coverage::interfaces::hash_str(&func_name); From a6a31f0487b45f4744065e1f99951bc9c6349ad1 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 22 Jan 2024 09:45:58 +0000 Subject: [PATCH 58/70] debug information --- src/codegen/instrument.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index 3e5096efb2d..28aebba4130 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -97,7 +97,9 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { // let func = llvm_index // .find_associated_implementation(&func_name) // .expect("Function not found in LLVM index"); - let func = module.get_function(&func_name).expect("Function not found in module!"); + let func = module + .get_function(&func_name) + .expect(&format!("Function not found in module: {}", func_name)); let func_pgo = rustc_llvm_coverage::interfaces::create_pgo_func_name_var(&func); From 07f1a9c55495c86084d7fe50a692e4c38b7c9e5c Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 22 Jan 2024 09:47:35 +0000 Subject: [PATCH 59/70] remove print --- src/codegen/generators/pou_generator.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/codegen/generators/pou_generator.rs b/src/codegen/generators/pou_generator.rs index e55faa0fff2..7f9918b2180 100644 --- a/src/codegen/generators/pou_generator.rs +++ b/src/codegen/generators/pou_generator.rs @@ -290,7 +290,6 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { let mut local_index = LlvmTypedIndex::create_child(self.llvm_index); let pou_name = &implementation.name; - println!("Generating implementation for {}", pou_name); let current_function = self.llvm_index.find_associated_implementation(pou_name).ok_or_else(|| { Diagnostic::codegen_error( From dc2161612b0a3a421fd7140d28c23375578897eb Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 22 Jan 2024 10:06:47 +0000 Subject: [PATCH 60/70] log before fail --- src/codegen/instrument.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index 28aebba4130..8206d9d5a88 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -93,10 +93,22 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { for function_record in function_records { let func_name = function_record.name.clone(); + // Check if not there: + if module.get_function(&func_name).is_none() { + if let Some(llvm_index_func) = llvm_index.find_associated_implementation(&func_name) { + println!( + "Func: {} not in module, but found in LLVM index func: {:?}", + &func_name, llvm_index_func + ); + } + } + // TODO - decide whether or not this needs to come from the module + // Theoretically, they should both be identical // let func = llvm_index // .find_associated_implementation(&func_name) // .expect("Function not found in LLVM index"); + let func = module .get_function(&func_name) .expect(&format!("Function not found in module: {}", func_name)); From fdc32b9f9c3edb802d75626d7863763023626610 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Mon, 22 Jan 2024 10:11:16 +0000 Subject: [PATCH 61/70] soft fail --- src/codegen/instrument.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index 8206d9d5a88..69ab5257ccf 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -43,6 +43,11 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { if implementation.linkage != LinkageType::Internal { continue; } + // Skip no-definition functions + if module.get_function(&implementation.name).is_none() { + println!("Skipping undefined function: {}", &implementation.name); + continue; + } let func_name = implementation.name.clone(); // TODO - hash strucutrally @@ -94,14 +99,16 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { let func_name = function_record.name.clone(); // Check if not there: - if module.get_function(&func_name).is_none() { - if let Some(llvm_index_func) = llvm_index.find_associated_implementation(&func_name) { - println!( - "Func: {} not in module, but found in LLVM index func: {:?}", - &func_name, llvm_index_func - ); - } - } + // if module.get_function(&func_name).is_none() { + // println!("Skipping undefined function: {}", &func_name); + // continue; + // // if let Some(llvm_index_func) = llvm_index.find_associated_implementation(&func_name) { + // // println!( + // // "Func: {} not in module, but found in LLVM index func: {:?}", + // // &func_name, llvm_index_func + // // ); + // // } + // } // TODO - decide whether or not this needs to come from the module // Theoretically, they should both be identical From 2b9bbeaf1c7617c47edd2673a4a209a52fc1515a Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Tue, 30 Jan 2024 08:47:18 +0000 Subject: [PATCH 62/70] working coverage gen for if statements --- compiler/plc_source/src/source_location.rs | 3 +- compiler/rustc_llvm_coverage/src/lib.rs | 1 + examples/hello_world.st | 12 + scripts/build.sh | 45 +++- src/codegen.rs | 1 + src/codegen/generators/pou_generator.rs | 3 +- src/codegen/generators/statement_generator.rs | 20 +- src/codegen/instrument.rs | 231 +++++++++++++----- 8 files changed, 249 insertions(+), 67 deletions(-) diff --git a/compiler/plc_source/src/source_location.rs b/compiler/plc_source/src/source_location.rs index 0ac463b9127..adbc1c5c8b2 100644 --- a/compiler/plc_source/src/source_location.rs +++ b/compiler/plc_source/src/source_location.rs @@ -302,7 +302,8 @@ impl SourceLocation { let span = self.get_span(); if let CodeSpan::Range(range) = span { let (start_line, start_col, end_line, end_col) = - (range.start.line, range.start.column, range.end.line, range.end.column); + // TODO - rusty devs will understand what needs to be done here w/ line + 1 + (range.start.line + 1, range.start.column + 1, range.end.line + 1, range.end.column + 1); (start_line, start_col, end_line, end_col) } else { panic!("Error: expected CodeSpan::Range, found {:?}", span); diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 06f45c09073..d01d0d200dc 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -14,6 +14,7 @@ use interfaces::*; use types::*; use inkwell::builder::Builder; +use inkwell::context::Context; use inkwell::intrinsics::Intrinsic; use inkwell::module::Module; use inkwell::passes::PassBuilderOptions; diff --git a/examples/hello_world.st b/examples/hello_world.st index 3ca0c54829e..3868c2c3c27 100644 --- a/examples/hello_world.st +++ b/examples/hello_world.st @@ -7,4 +7,16 @@ END_FUNCTION FUNCTION main : DINT puts('hello, world!$N'); + + if (1) then + puts('true$N'); + if (2) then + puts('truetrue$N'); + else + puts('truefalse$N'); + end_if; + puts('true$N'); + else + puts('false$N'); + end_if; END_FUNCTION diff --git a/scripts/build.sh b/scripts/build.sh index b0309bbe4a4..866fccea2bd 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -482,10 +482,51 @@ if [[ $build -ne 0 ]]; then # Test a program echo "-----------------------------------" echo "Running on example program:" - ./target/debug/plc --ir ./examples/simple_program.st + # ./target/debug/plc --ir ./examples/simple_program.st + # ./target/debug/plc --ir hello_world.st + ./target/debug/plc --ir ./examples/hello_world.st + # echo "-----------------------------------" + # cat ./hello_world.st.ll echo "-----------------------------------" - cat ./simple_program.st.ll + + # Cleanup prof + PROFRAW=./default.profraw + PROFEXPORT=./default.profexport + rm -f $PROFRAW $PROFEXPORT + rm -f hello_world.st.out + + # # Compile + echo "Compiling hello_world.st.ll to hello_world.st.out" + clang++-14 -fprofile-instr-generate -fcoverage-mapping -Wl,-u,__llvm_profile_runtime -O0 hello_world.st.ll -o hello_world.st.out + echo "Done!" + echo "-----------------------------------" + + # # Run + echo "Running program" + ./hello_world.st.out + echo "Done!" echo "-----------------------------------" + + # Show profdata + echo "Showing profdata" + llvm-profdata-14 show -all-functions $PROFRAW + echo "-----------------------------------" + + # Generate profdata + echo "Generating profdata" + llvm-profdata-14 merge $PROFRAW -o $PROFEXPORT + echo "-----------------------------------" + + # Show coverage + echo "Coverage" + llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT + llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT -show-regions + echo "-----------------------------------" + + echo "Export" + # llvm-cov-14 export -instr-profile=$PROFEXPORT hello_world.st.out + echo "-----------------------------------" + #Build the standard functions # run_std_build fi diff --git a/src/codegen.rs b/src/codegen.rs index 4a03891a08e..5e4001d61dd 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -103,6 +103,7 @@ impl<'ink> CodeGen<'ink> { // TODO - local path shouldn't be hardcoded let filenames = vec!["/workspaces/corbanvilla_rusty".to_string(), module_location.to_string()]; let instr_builder = instrument::CoverageInstrumentationBuilder::new(context, filenames); + // instrument::CoverageInstrumentationBuilder::new(/*context, */ &module, filenames); CodeGen { module, diff --git a/src/codegen/generators/pou_generator.rs b/src/codegen/generators/pou_generator.rs index 7f9918b2180..a81b2cd9421 100644 --- a/src/codegen/generators/pou_generator.rs +++ b/src/codegen/generators/pou_generator.rs @@ -390,9 +390,10 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { &local_index, &function_context, debug, + instrumentation, ); - // Emit instrumentation before funciton body + // Emit instrumentation before function body if let Some(instr_builder) = instrumentation { instr_builder.emit_function_increment( &self.llvm.builder, diff --git a/src/codegen/generators/statement_generator.rs b/src/codegen/generators/statement_generator.rs index 0a65c636f72..b7f98c9bf2d 100644 --- a/src/codegen/generators/statement_generator.rs +++ b/src/codegen/generators/statement_generator.rs @@ -6,6 +6,7 @@ use super::{ llvm::Llvm, }; use crate::{ + codegen::instrument::CoverageInstrumentationBuilder, codegen::{debug::Debug, llvm_typesystem::cast_if_needed}, codegen::{debug::DebugBuilderEnum, LlvmTypedIndex}, index::{ImplementationIndexEntry, Index}, @@ -16,6 +17,7 @@ use inkwell::{ basic_block::BasicBlock, builder::Builder, context::Context, + module::Module, values::{BasicValueEnum, FunctionValue, PointerValue}, }; use plc_ast::{ @@ -55,6 +57,7 @@ pub struct StatementCodeGenerator<'a, 'b> { pub current_loop_continue: Option>, pub debug: &'b DebugBuilderEnum<'a>, + pub instrumentation: &'b Option>, } impl<'a, 'b> StatementCodeGenerator<'a, 'b> { @@ -66,6 +69,7 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { llvm_index: &'b LlvmTypedIndex<'a>, linking_context: &'b FunctionContext<'a, 'b>, debug: &'b DebugBuilderEnum<'a>, + instrumentation: &'b Option>, ) -> StatementCodeGenerator<'a, 'b> { StatementCodeGenerator { llvm, @@ -78,10 +82,11 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { current_loop_exit: None, current_loop_continue: None, debug, + instrumentation, } } - /// convinience method to create an expression-generator + /// convenience method to create an expression-generator fn create_expr_generator(&'a self) -> ExpressionCodeGenerator<'a, 'b> { ExpressionCodeGenerator::new( self.llvm, @@ -758,6 +763,18 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { //Generate if statement content builder.position_at_end(conditional_block); + + // Generate counter increment + if let Some(instr_builder) = self.instrumentation { + if let Some(first_ast) = block.body.first() { + // instr_builder.emit_branch_increment( + // builder, + // current_function.get_name().to_str().unwrap(), + // first_ast.id, + // ); + } + } + self.generate_body(&block.body)?; builder.build_unconditional_branch(continue_block); } @@ -765,6 +782,7 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { if let Some(else_block) = else_block { builder.position_at_end(else_block); + // TODO - if else block[0] exists, then generate a ctr increment real quick self.generate_body(else_body)?; builder.build_unconditional_branch(continue_block); } diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index 69ab5257ccf..cd70c0a214f 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -3,8 +3,9 @@ use inkwell::builder::Builder; use inkwell::context::Context; use inkwell::module::Module; use inkwell::values::{FunctionValue, GlobalValue}; -use plc_ast::ast::{CompilationUnit, LinkageType}; -use plc_source::source_location::CodeSpan; +use plc_ast::ast::{AstId, AstNode, AstStatement, CompilationUnit, Implementation, LinkageType}; +use plc_ast::control_statements::AstControlStatement; +use plc_source::source_location::{CodeSpan, SourceLocation}; use rustc_llvm_coverage::types::{Counter, CounterId, CounterMappingRegion}; use rustc_llvm_coverage::*; use std::collections::HashMap; @@ -12,14 +13,58 @@ use std::ffi::CString; pub struct CoverageInstrumentationBuilder<'ink> { context: &'ink Context, + // module: &'ink Module<'ink>, files: Vec, cov_mapping_header: Option, function_pgos: HashMap)>, + ast_counter_lookup: HashMap, +} + +/// Manages the creation of mapping regions for a given function +#[derive(Debug)] +struct MappingRegionGenerator { + // TODO - verify that counter ids are PER function + pub mapping_regions: Vec, + next_counter_id: u32, + file_id: u32, +} + +impl MappingRegionGenerator { + pub fn new(file_id: u32) -> Self { + Self { mapping_regions: Vec::new(), next_counter_id: 0, file_id } + } + + /// Returns the index of the counter id added + pub fn add_mapping_region(&mut self, source: &SourceLocation) -> u32 { + let (start_line, start_col, end_line, end_col) = source.get_start_end(); + let counter_id = self.next_counter_id; + let counter = Counter::counter_value_reference(CounterId::new(counter_id)); + let mapping_region = CounterMappingRegion::code_region( + counter, + self.file_id, + start_line.try_into().unwrap(), + start_col.try_into().unwrap(), + end_line.try_into().unwrap(), + end_col.try_into().unwrap(), + ); + self.mapping_regions.push(mapping_region); + self.next_counter_id += 1; + + // Return the index of the counter id added + counter_id + } } impl<'ink> CoverageInstrumentationBuilder<'ink> { - pub fn new(context: &'ink Context, files: Vec) -> Self { - Self { context, files, cov_mapping_header: None, function_pgos: HashMap::new() } + pub fn new(context: &'ink Context, /* module: &'ink Module<'ink>,*/ files: Vec) -> Self { + Self { + context, + // module, + files, + cov_mapping_header: None, + function_pgos: HashMap::new(), + ast_counter_lookup: HashMap::new(), + } } pub fn initialize(&mut self, module: &Module<'ink>) { @@ -44,51 +89,13 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { continue; } // Skip no-definition functions + // TODO - investigate which functions don't have definitions and why if module.get_function(&implementation.name).is_none() { println!("Skipping undefined function: {}", &implementation.name); continue; } - let func_name = implementation.name.clone(); - // TODO - hash strucutrally - let struct_hash = rustc_llvm_coverage::interfaces::hash_str(&func_name); - // TODO - modify this? - let func_filenames = vec![unit.file_name.clone()]; - // TODO - use expression counters - let expressions = Vec::new(); - // TODO - file mapping table - let file_id = 1; - - // TODO - loop through dfs - let mut mapping_regions = Vec::new(); - { - let (start_line, start_col, end_line, end_col) = implementation.location.get_start_end(); - let counter = Counter::counter_value_reference(CounterId::new(0)); - let mapping_region = CounterMappingRegion::code_region( - counter, - file_id, - start_line.try_into().unwrap(), - start_col.try_into().unwrap(), - end_line.try_into().unwrap(), - end_col.try_into().unwrap(), - ); - mapping_regions.push(mapping_region); - } - - let is_used = true; - - let written_coverage_header = &self.cov_mapping_header.as_mut().unwrap(); - - let func = FunctionRecord::new( - func_name, - struct_hash, - func_filenames, - expressions, - mapping_regions, - is_used, - &written_coverage_header, - ); - + let func = self.generate_function_record(implementation); func.write_to_module(module); function_records.push(func); @@ -98,24 +105,6 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { for function_record in function_records { let func_name = function_record.name.clone(); - // Check if not there: - // if module.get_function(&func_name).is_none() { - // println!("Skipping undefined function: {}", &func_name); - // continue; - // // if let Some(llvm_index_func) = llvm_index.find_associated_implementation(&func_name) { - // // println!( - // // "Func: {} not in module, but found in LLVM index func: {:?}", - // // &func_name, llvm_index_func - // // ); - // // } - // } - - // TODO - decide whether or not this needs to come from the module - // Theoretically, they should both be identical - // let func = llvm_index - // .find_associated_implementation(&func_name) - // .expect("Function not found in LLVM index"); - let func = module .get_function(&func_name) .expect(&format!("Function not found in module: {}", func_name)); @@ -149,7 +138,125 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { ); } + pub fn emit_branch_increment<'ctx>( + &self, + builder: &Builder<'ink>, + module: &Module<'ctx>, + func_name: &str, + ast_id: AstId, + ) { + let (func_record, func_pgo_var) = self.function_pgos.get(func_name).unwrap(); + + let pgo_pointer = func_pgo_var.as_pointer_value(); + let num_counters = func_record.mapping_regions.len(); + + let counter_index = + self.ast_counter_lookup.get(&ast_id).expect(&format!("Counter not found for AstId {}", ast_id)); + rustc_llvm_coverage::emit_counter_increment( + builder, + module, + &pgo_pointer, + func_record.structural_hash, + num_counters.try_into().unwrap(), + *counter_index as u64, + ); + } + pub fn finalize(&mut self, module: &Module<'ink>) { run_instrumentation_lowering_pass(module); } + + /// Internal function to generate for a function: + /// - FunctionRecord + /// - MappingRegions + fn generate_function_record(&mut self, implementation: &Implementation) -> FunctionRecord { + // Gather function information + + let func_name = implementation.name.clone(); + // TODO - hash strucutrally + let struct_hash = rustc_llvm_coverage::interfaces::hash_str(&func_name); + let func_filenames = vec![implementation.location.get_file_name().unwrap().to_string()]; + // TODO - use expression counters + let expressions = Vec::new(); + // TODO - file mapping table + let file_id = 1; + + // Map entire function + let mut mapping_region_generator = MappingRegionGenerator::new(file_id); + let func_ctr_id = mapping_region_generator.add_mapping_region(&implementation.location); + assert!(func_ctr_id == 0); + + // DFS function statements + self.generate_coverage_records(&implementation.statements, &mut mapping_region_generator); + + // TODO - determine if function is used + let is_used = true; + let written_coverage_header = &self.cov_mapping_header.as_mut().unwrap(); + + FunctionRecord::new( + func_name, + struct_hash, + func_filenames, + expressions, + mapping_region_generator.mapping_regions, + is_used, + &written_coverage_header, + ) + } + + fn generate_coverage_records( + &mut self, + ast_node_list: &Vec, + mapping_region_generator: &mut MappingRegionGenerator, + ) { + for ast_node in ast_node_list { + // Only generate coverage records for control statements + let control_statement = match &ast_node.stmt { + AstStatement::ControlStatement(statement) => statement, + _ => continue, + }; + + // + match control_statement { + AstControlStatement::If(statement) => { + // Loop through if/elif blocks + for block in &statement.blocks { + // Setup ast->id mapping, store region location + self.register_ast_list_as_region(&block.body, mapping_region_generator); + // Recurse + self.generate_coverage_records(&block.body, mapping_region_generator); + } + + // Else block + self.register_ast_list_as_region(&statement.else_block, mapping_region_generator); + self.generate_coverage_records(&statement.else_block, mapping_region_generator); + } + AstControlStatement::ForLoop(statement) => (), + AstControlStatement::WhileLoop(statement) => (), + AstControlStatement::RepeatLoop(statement) => (), + AstControlStatement::Case(statement) => (), + } + } + } + + // TODO - find me a better name + /// Registers a Vec as a region, spanning first and last + fn register_ast_list_as_region( + &mut self, + ast_list: &Vec, + mapping_region_generator: &mut MappingRegionGenerator, + ) { + if ast_list.is_empty() { + return; + } + + // Create a span from first_block -> last_block + let first_block = ast_list.first().unwrap(); + let last_block = ast_list.last().unwrap(); + let span = first_block.location.span(&last_block.location); + + // Map the span, store the counter id in the lookup table (key at first_block.ast_id) + let ctr_id = mapping_region_generator.add_mapping_region(&span); + self.ast_counter_lookup.insert(first_block.id, ctr_id.try_into().unwrap()); + } } From 001ed97c647ac2f25f5fa6945e261966901a4c3f Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Tue, 30 Jan 2024 12:41:09 +0000 Subject: [PATCH 63/70] added branch instrumentation --- compiler/rustc_llvm_coverage/src/lib.rs | 26 +++++++++++++++ scripts/build.sh | 2 +- src/codegen.rs | 7 ++++ src/codegen/generators/pou_generator.rs | 3 ++ src/codegen/generators/statement_generator.rs | 33 ++++++++++++++++--- src/codegen/instrument.rs | 17 ++++++++-- 6 files changed, 79 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index d01d0d200dc..1f638850de6 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -19,6 +19,7 @@ use inkwell::intrinsics::Intrinsic; use inkwell::module::Module; use inkwell::passes::PassBuilderOptions; use inkwell::targets::{CodeModel, InitializationConfig, RelocMode, Target, TargetTriple}; +use inkwell::values::FunctionValue; use inkwell::values::PointerValue; use inkwell::OptimizationLevel; use std::ffi::CString; @@ -267,5 +268,30 @@ pub fn emit_counter_increment<'ink, 'ctx>( ); } +/// TODO - merge with function from above +pub fn emit_counter_increment_with_function<'ink>( + builder: &Builder<'ink>, + context: &Context, + increment_intrinsic_func: &FunctionValue<'ink>, + pgo_function_var: &PointerValue<'ink>, + structural_hash: u64, + num_counters: u32, + counter_idx: u64, +) { + // Create types + let i64_type = context.i64_type(); + let i32_type = context.i32_type(); + + let i64_hash = i64_type.const_int(structural_hash, false); + let i32_num_counters = i32_type.const_int(num_counters.into(), false); + let i64_counter_idx = i64_type.const_int(counter_idx, false); + + builder.build_call( + *increment_intrinsic_func, + &[(*pgo_function_var).into(), i64_hash.into(), i32_num_counters.into(), i64_counter_idx.into()], + "increment_call", + ); +} + // TODO // - investigate codegen diffs for function/function blocks/programs diff --git a/scripts/build.sh b/scripts/build.sh index 866fccea2bd..0f353394681 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -520,7 +520,7 @@ if [[ $build -ne 0 ]]; then # Show coverage echo "Coverage" llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT - llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT -show-regions + llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT -show-regions echo "-----------------------------------" echo "Export" diff --git a/src/codegen.rs b/src/codegen.rs index 5e4001d61dd..e897277e3d3 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -199,6 +199,13 @@ impl<'ink> CodeGen<'ink> { index.associate_utf16_literal(literal, literal_variable); } + // Add the increment intrinsic to the index + if let Some(instr_builder) = &mut self.instrument { + let increment_function = instr_builder.get_increment_function(&self.module); + let increment_function_name = increment_function.get_name().to_str().unwrap(); + index.associate_implementation(increment_function_name, increment_function); + } + Ok(index) } diff --git a/src/codegen/generators/pou_generator.rs b/src/codegen/generators/pou_generator.rs index a81b2cd9421..f5ab292c0ed 100644 --- a/src/codegen/generators/pou_generator.rs +++ b/src/codegen/generators/pou_generator.rs @@ -322,6 +322,8 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { self.llvm.builder.position_at_end(block); blocks.insert("entry".into(), block); + // TODO - don't hardcode this name + let increment_function = local_index.find_associated_implementation("llvm.instrprof.increment"); let function_context = FunctionContext { linking_context: self.index.find_implementation_by_name(&implementation.name).ok_or_else( || { @@ -333,6 +335,7 @@ impl<'ink, 'cg> PouGenerator<'ink, 'cg> { )?, function: current_function, blocks, + increment_function, }; let mut param_index = 0; diff --git a/src/codegen/generators/statement_generator.rs b/src/codegen/generators/statement_generator.rs index b7f98c9bf2d..88db21541f4 100644 --- a/src/codegen/generators/statement_generator.rs +++ b/src/codegen/generators/statement_generator.rs @@ -38,6 +38,9 @@ pub struct FunctionContext<'ink, 'b> { pub function: FunctionValue<'ink>, /// The blocks/labels this function can use pub blocks: HashMap>, + // TODO - there may be a better spot for this, but it works for now + /// Increment intrinsic + pub increment_function: Option>, } /// the StatementCodeGenerator is used to generate statements (For, If, etc.) or expressions (references, literals, etc.) @@ -767,11 +770,13 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { // Generate counter increment if let Some(instr_builder) = self.instrumentation { if let Some(first_ast) = block.body.first() { - // instr_builder.emit_branch_increment( - // builder, - // current_function.get_name().to_str().unwrap(), - // first_ast.id, - // ); + instr_builder.emit_branch_increment( + builder, + context, + &self.get_increment_function(), + current_function.get_name().to_str().unwrap(), + first_ast.id, + ); } } @@ -783,6 +788,20 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { if let Some(else_block) = else_block { builder.position_at_end(else_block); // TODO - if else block[0] exists, then generate a ctr increment real quick + + // Generate counter increment + if let Some(instr_builder) = self.instrumentation { + if let Some(first_ast) = else_body.first() { + instr_builder.emit_branch_increment( + builder, + context, + &self.get_increment_function(), + current_function.get_name().to_str().unwrap(), + first_ast.id, + ); + } + } + self.generate_body(else_body)?; builder.build_unconditional_branch(continue_block); } @@ -859,6 +878,10 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { fn get_llvm_deps(&self) -> (&Builder, FunctionValue, &Context) { (&self.llvm.builder, self.function_context.function, self.llvm.context) } + + fn get_increment_function(&self) -> FunctionValue { + self.function_context.increment_function.expect("Increment function not set") + } } /// when generating an assignment to a direct-access (e.g. a.b.c.%W3.%X2 := 2;) diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index cd70c0a214f..b9daccf2c4d 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -1,6 +1,7 @@ use super::LlvmTypedIndex; use inkwell::builder::Builder; use inkwell::context::Context; +use inkwell::intrinsics::Intrinsic; use inkwell::module::Module; use inkwell::values::{FunctionValue, GlobalValue}; use plc_ast::ast::{AstId, AstNode, AstStatement, CompilationUnit, Implementation, LinkageType}; @@ -141,7 +142,8 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { pub fn emit_branch_increment<'ctx>( &self, builder: &Builder<'ink>, - module: &Module<'ctx>, + context: &Context, + increment_intrinsic_func: &FunctionValue<'ink>, func_name: &str, ast_id: AstId, ) { @@ -152,9 +154,11 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { let counter_index = self.ast_counter_lookup.get(&ast_id).expect(&format!("Counter not found for AstId {}", ast_id)); - rustc_llvm_coverage::emit_counter_increment( + + rustc_llvm_coverage::emit_counter_increment_with_function( builder, - module, + context, + increment_intrinsic_func, &pgo_pointer, func_record.structural_hash, num_counters.try_into().unwrap(), @@ -259,4 +263,11 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { let ctr_id = mapping_region_generator.add_mapping_region(&span); self.ast_counter_lookup.insert(first_block.id, ctr_id.try_into().unwrap()); } + + pub fn get_increment_function(&self, module: &Module<'ink>) -> FunctionValue<'ink> { + let increment_intrinsic = Intrinsic::find("llvm.instrprof.increment").unwrap(); + let increment_intrinsic_func = increment_intrinsic.get_declaration(module, &[]).unwrap(); + + increment_intrinsic_func + } } From bd03a0ab548c1adf8c0a330a59ae1ebac81d3691 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Tue, 30 Jan 2024 14:17:53 +0000 Subject: [PATCH 64/70] pass for non registered ast --- src/codegen/instrument.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index b9daccf2c4d..491c9eb4944 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -152,8 +152,13 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { let pgo_pointer = func_pgo_var.as_pointer_value(); let num_counters = func_record.mapping_regions.len(); - let counter_index = - self.ast_counter_lookup.get(&ast_id).expect(&format!("Counter not found for AstId {}", ast_id)); + let counter_index = match self.ast_counter_lookup.get(&ast_id) { + Some(counter_index) => counter_index, + None => { + println!("Ast Not Registered: {} (from function {})", ast_id, func_name); + return; + } + }; rustc_llvm_coverage::emit_counter_increment_with_function( builder, From 8aa8377b60c3dc44640e43f7dc5854a72e7639c2 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Tue, 30 Jan 2024 19:08:36 +0000 Subject: [PATCH 65/70] branching type --- compiler/rustc_llvm_coverage/src/types.rs | 9 ++-- src/codegen/generators/statement_generator.rs | 28 ++++++++++++ src/codegen/instrument.rs | 43 ++++++++++++++++--- 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/types.rs b/compiler/rustc_llvm_coverage/src/types.rs index 2ee55556654..a13aa3a05a4 100644 --- a/compiler/rustc_llvm_coverage/src/types.rs +++ b/compiler/rustc_llvm_coverage/src/types.rs @@ -166,10 +166,10 @@ impl CounterMappingRegion { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. - #[allow(dead_code)] - pub(crate) fn branch_region( + // #[allow(dead_code)] + pub fn branch_region( counter: Counter, - false_counter: Counter, + // false_counter: Counter, file_id: u32, start_line: u32, start_col: u32, @@ -178,7 +178,8 @@ impl CounterMappingRegion { ) -> Self { Self { counter, - false_counter, + // false_counter, + false_counter: Counter::ZERO, file_id, expanded_file_id: 0, start_line, diff --git a/src/codegen/generators/statement_generator.rs b/src/codegen/generators/statement_generator.rs index 88db21541f4..1532f694c37 100644 --- a/src/codegen/generators/statement_generator.rs +++ b/src/codegen/generators/statement_generator.rs @@ -565,12 +565,40 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { } //generate the case's body builder.position_at_end(case_block); + + // Generate counter increment + if let Some(instr_builder) = self.instrumentation { + if let Some(first_ast) = conditional_block.body.first() { + instr_builder.emit_branch_increment( + builder, + context, + &self.get_increment_function(), + current_function.get_name().to_str().unwrap(), + first_ast.id, + ); + } + } + self.generate_body(&conditional_block.body)?; // skiop all other case-bodies builder.build_unconditional_branch(continue_block); } // current-else is the last else-block generated by the range-expressions builder.position_at_end(current_else_block); + + // Generate counter increment + if let Some(instr_builder) = self.instrumentation { + if let Some(first_ast) = else_body.first() { + instr_builder.emit_branch_increment( + builder, + context, + &self.get_increment_function(), + current_function.get_name().to_str().unwrap(), + first_ast.id, + ); + } + } + self.generate_body(else_body)?; builder.build_unconditional_branch(continue_block); continue_block.move_after(current_else_block).expect(INTERNAL_LLVM_ERROR); diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index 491c9eb4944..2101e0a15f4 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -35,8 +35,8 @@ impl MappingRegionGenerator { Self { mapping_regions: Vec::new(), next_counter_id: 0, file_id } } - /// Returns the index of the counter id added - pub fn add_mapping_region(&mut self, source: &SourceLocation) -> u32 { + // Returns the index of the counter id added + pub fn add_code_mapping_region(&mut self, source: &SourceLocation) -> u32 { let (start_line, start_col, end_line, end_col) = source.get_start_end(); let counter_id = self.next_counter_id; let counter = Counter::counter_value_reference(CounterId::new(counter_id)); @@ -54,6 +54,26 @@ impl MappingRegionGenerator { // Return the index of the counter id added counter_id } + + /// Returns the index of the counter id added + pub fn add_branch_mapping_region(&mut self, source: &SourceLocation) -> u32 { + let (start_line, start_col, end_line, end_col) = source.get_start_end(); + let counter_id = self.next_counter_id; + let counter = Counter::counter_value_reference(CounterId::new(counter_id)); + let mapping_region = CounterMappingRegion::branch_region( + counter, + self.file_id, + start_line.try_into().unwrap(), + start_col.try_into().unwrap(), + end_line.try_into().unwrap(), + end_col.try_into().unwrap(), + ); + self.mapping_regions.push(mapping_region); + self.next_counter_id += 1; + + // Return the index of the counter id added + counter_id + } } impl<'ink> CoverageInstrumentationBuilder<'ink> { @@ -155,6 +175,7 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { let counter_index = match self.ast_counter_lookup.get(&ast_id) { Some(counter_index) => counter_index, None => { + // TODO - figure out why this happens println!("Ast Not Registered: {} (from function {})", ast_id, func_name); return; } @@ -192,7 +213,7 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { // Map entire function let mut mapping_region_generator = MappingRegionGenerator::new(file_id); - let func_ctr_id = mapping_region_generator.add_mapping_region(&implementation.location); + let func_ctr_id = mapping_region_generator.add_code_mapping_region(&implementation.location); assert!(func_ctr_id == 0); // DFS function statements @@ -243,7 +264,19 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { AstControlStatement::ForLoop(statement) => (), AstControlStatement::WhileLoop(statement) => (), AstControlStatement::RepeatLoop(statement) => (), - AstControlStatement::Case(statement) => (), + AstControlStatement::Case(statement) => { + // Loop through case blocks + for block in &statement.case_blocks { + // Setup ast->id mapping, store region location + self.register_ast_list_as_region(&block.body, mapping_region_generator); + // Recurse + self.generate_coverage_records(&block.body, mapping_region_generator); + } + + // Else block + self.register_ast_list_as_region(&statement.else_block, mapping_region_generator); + self.generate_coverage_records(&statement.else_block, mapping_region_generator); + } } } } @@ -265,7 +298,7 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { let span = first_block.location.span(&last_block.location); // Map the span, store the counter id in the lookup table (key at first_block.ast_id) - let ctr_id = mapping_region_generator.add_mapping_region(&span); + let ctr_id = mapping_region_generator.add_branch_mapping_region(&span); self.ast_counter_lookup.insert(first_block.id, ctr_id.try_into().unwrap()); } From e80ce78d7f2e5abecc54519fbb2503ce06c9f8a9 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 31 Jan 2024 18:40:46 +0000 Subject: [PATCH 66/70] added branching support w/ loops and expressions --- compiler/rustc_llvm_coverage/src/types.rs | 15 +- src/codegen/generators/statement_generator.rs | 48 ++++ src/codegen/instrument.rs | 267 +++++++++++++++--- 3 files changed, 287 insertions(+), 43 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/types.rs b/compiler/rustc_llvm_coverage/src/types.rs index a13aa3a05a4..d663f12d34a 100644 --- a/compiler/rustc_llvm_coverage/src/types.rs +++ b/compiler/rustc_llvm_coverage/src/types.rs @@ -110,11 +110,11 @@ pub enum RegionKind { #[repr(C)] pub struct CounterMappingRegion { /// The counter type and type-dependent counter data, if any. - counter: Counter, + pub counter: Counter, /// If the `RegionKind` is a `BranchRegion`, this represents the counter /// for the false branch of the region. - false_counter: Counter, + pub false_counter: Counter, /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes @@ -139,7 +139,7 @@ pub struct CounterMappingRegion { /// mapping region is a gap area. end_col: u32, - kind: RegionKind, + pub kind: RegionKind, } impl CounterMappingRegion { @@ -169,7 +169,7 @@ impl CounterMappingRegion { // #[allow(dead_code)] pub fn branch_region( counter: Counter, - // false_counter: Counter, + false_counter: Counter, file_id: u32, start_line: u32, start_col: u32, @@ -178,8 +178,7 @@ impl CounterMappingRegion { ) -> Self { Self { counter, - // false_counter, - false_counter: Counter::ZERO, + false_counter, file_id, expanded_file_id: 0, start_line, @@ -286,12 +285,12 @@ pub enum CounterKind { pub struct Counter { // Important: The layout (order and types of fields) must match its C++ counterpart. pub kind: CounterKind, - id: u32, + pub id: u32, } impl Counter { /// A `Counter` of kind `Zero`. For this counter kind, the `id` is not used. - pub(crate) const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 }; + pub const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 }; /// Constructs a new `Counter` of kind `CounterValueReference`. pub fn counter_value_reference(counter_id: CounterId) -> Self { diff --git a/src/codegen/generators/statement_generator.rs b/src/codegen/generators/statement_generator.rs index 1532f694c37..daa25019a4a 100644 --- a/src/codegen/generators/statement_generator.rs +++ b/src/codegen/generators/statement_generator.rs @@ -428,6 +428,20 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { load_suffix: self.load_suffix.clone(), ..*self }; + + // Generate counter increment + if let Some(instr_builder) = self.instrumentation { + if let Some(first_ast) = body.first() { + instr_builder.emit_branch_increment( + builder, + context, + &self.get_increment_function(), + current_function.get_name().to_str().unwrap(), + first_ast.id, + ); + } + } + body_generator.generate_body(body)?; builder.build_unconditional_branch(increment_block); @@ -463,6 +477,17 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { //Continue builder.position_at_end(continue_block); + // Increment false counter + if let Some(instr_builder) = self.instrumentation { + instr_builder.emit_branch_increment( + builder, + context, + &self.get_increment_function(), + current_function.get_name().to_str().unwrap(), + end.id, + ); + } + Ok(()) } @@ -739,12 +764,35 @@ impl<'a, 'b> StatementCodeGenerator<'a, 'b> { load_suffix: self.load_suffix.clone(), ..*self }; + if let Some(instr_builder) = self.instrumentation { + if let Some(first_ast) = body.first() { + instr_builder.emit_branch_increment( + builder, + context, + &self.get_increment_function(), + current_function.get_name().to_str().unwrap(), + first_ast.id, + ); + } + } body_generator.generate_body(body)?; //Loop back builder.build_unconditional_branch(condition_check); //Continue builder.position_at_end(continue_block); + + // Increment false counter + if let Some(instr_builder) = self.instrumentation { + instr_builder.emit_branch_increment( + builder, + context, + &self.get_increment_function(), + current_function.get_name().to_str().unwrap(), + condition.id, + ); + } + Ok((condition_check, while_body)) } diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index 2101e0a15f4..898555bfdac 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -7,7 +7,9 @@ use inkwell::values::{FunctionValue, GlobalValue}; use plc_ast::ast::{AstId, AstNode, AstStatement, CompilationUnit, Implementation, LinkageType}; use plc_ast::control_statements::AstControlStatement; use plc_source::source_location::{CodeSpan, SourceLocation}; -use rustc_llvm_coverage::types::{Counter, CounterId, CounterMappingRegion}; +use rustc_llvm_coverage::types::{ + Counter, CounterExpression, CounterId, CounterMappingRegion, ExprKind, ExpressionId, +}; use rustc_llvm_coverage::*; use std::collections::HashMap; use std::ffi::CString; @@ -18,6 +20,13 @@ pub struct CoverageInstrumentationBuilder<'ink> { files: Vec, cov_mapping_header: Option, function_pgos: HashMap)>, + // TODO - better counter datastructures + // ast_counter_lookup: + // - if statements: map first body block -> true branch counter + // - case statements: map first body block -> true branch counter + // - for statements: map first body block -> true branch counter AND end block -> false branch counter + // - while statements: map first body block -> true branch counter AND condition block -> false branch counter + // - repeat statements: map first body block -> true branch counter AND condition block -> false branch counter ast_counter_lookup: HashMap, } @@ -26,20 +35,31 @@ pub struct CoverageInstrumentationBuilder<'ink> { struct MappingRegionGenerator { // TODO - verify that counter ids are PER function pub mapping_regions: Vec, + pub expressions: Vec, next_counter_id: u32, + next_expression_id: u32, file_id: u32, } impl MappingRegionGenerator { pub fn new(file_id: u32) -> Self { - Self { mapping_regions: Vec::new(), next_counter_id: 0, file_id } + Self { + mapping_regions: Vec::new(), + expressions: Vec::new(), + next_counter_id: 0, + next_expression_id: 0, + file_id, + } } - // Returns the index of the counter id added - pub fn add_code_mapping_region(&mut self, source: &SourceLocation) -> u32 { + /// Adds to internal index and returns for convenience + pub fn add_code_mapping_region(&mut self, source: &SourceLocation) -> CounterMappingRegion { let (start_line, start_col, end_line, end_col) = source.get_start_end(); + let counter_id = self.next_counter_id; let counter = Counter::counter_value_reference(CounterId::new(counter_id)); + self.next_counter_id += 1; + let mapping_region = CounterMappingRegion::code_region( counter, self.file_id, @@ -48,31 +68,74 @@ impl MappingRegionGenerator { end_line.try_into().unwrap(), end_col.try_into().unwrap(), ); - self.mapping_regions.push(mapping_region); + self.mapping_regions.push(mapping_region.clone()); + + mapping_region + } + + // TODO - consolidate the two below functions + /// Adds to internal index and returns for convenience + /// Specific to if statements and case statements + pub fn add_branch_mapping_region( + &mut self, + source: &SourceLocation, + last_false_counter: Counter, + ) -> CounterMappingRegion { + let (start_line, start_col, end_line, end_col) = source.get_start_end(); + + // Counts branch executions + let counter_id = self.next_counter_id; + let counter = Counter::counter_value_reference(CounterId::new(counter_id)); self.next_counter_id += 1; + // Count the branch skips (when cond evalutes to false using a_{n-1} - Counter) + let false_counter_id = self.next_expression_id; + let false_counter = Counter::expression(ExpressionId::new(false_counter_id)); + let false_counter_expression = + CounterExpression::new(last_false_counter, ExprKind::Subtract, counter); + self.expressions.push(false_counter_expression); + self.next_expression_id += 1; + + let mapping_region = CounterMappingRegion::branch_region( + counter, + false_counter, + self.file_id, + start_line.try_into().unwrap(), + start_col.try_into().unwrap(), + end_line.try_into().unwrap(), + end_col.try_into().unwrap(), + ); + self.mapping_regions.push(mapping_region.clone()); + // Return the index of the counter id added - counter_id + mapping_region } - /// Returns the index of the counter id added - pub fn add_branch_mapping_region(&mut self, source: &SourceLocation) -> u32 { + pub fn add_loop_branch_mapping_region( + &mut self, + source: &SourceLocation, + false_counter: Counter, + ) -> CounterMappingRegion { let (start_line, start_col, end_line, end_col) = source.get_start_end(); + + // Counts branch executions let counter_id = self.next_counter_id; let counter = Counter::counter_value_reference(CounterId::new(counter_id)); + self.next_counter_id += 1; + let mapping_region = CounterMappingRegion::branch_region( counter, + false_counter, self.file_id, start_line.try_into().unwrap(), start_col.try_into().unwrap(), end_line.try_into().unwrap(), end_col.try_into().unwrap(), ); - self.mapping_regions.push(mapping_region); - self.next_counter_id += 1; + self.mapping_regions.push(mapping_region.clone()); // Return the index of the counter id added - counter_id + mapping_region } } @@ -206,18 +269,20 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { // TODO - hash strucutrally let struct_hash = rustc_llvm_coverage::interfaces::hash_str(&func_name); let func_filenames = vec![implementation.location.get_file_name().unwrap().to_string()]; - // TODO - use expression counters - let expressions = Vec::new(); // TODO - file mapping table let file_id = 1; // Map entire function let mut mapping_region_generator = MappingRegionGenerator::new(file_id); - let func_ctr_id = mapping_region_generator.add_code_mapping_region(&implementation.location); - assert!(func_ctr_id == 0); + let func_map_region = mapping_region_generator.add_code_mapping_region(&implementation.location); + assert!(func_map_region.counter.id == 0); // DFS function statements - self.generate_coverage_records(&implementation.statements, &mut mapping_region_generator); + self.generate_coverage_records( + &implementation.statements, + &mut mapping_region_generator, + func_map_region.counter, + ); // TODO - determine if function is used let is_used = true; @@ -227,17 +292,23 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { func_name, struct_hash, func_filenames, - expressions, + mapping_region_generator.expressions, mapping_region_generator.mapping_regions, is_used, &written_coverage_header, ) } + /// DFS algorithm to parse AST and generate coverage records for all branching + /// `parent_counter_id` is the counter id of the parent node, used for calculating "false" branches under if/case statements + /// TODO - explain or diagram what's going on here with + /// - last_false_counter: for chain branches + /// - last_true_counter: for recursing fn generate_coverage_records( &mut self, ast_node_list: &Vec, mapping_region_generator: &mut MappingRegionGenerator, + parent_counter: Counter, ) { for ast_node in ast_node_list { // Only generate coverage records for control statements @@ -246,36 +317,120 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { _ => continue, }; + // Track last counter (a_{n-1}) - useful for false branch calculations + // Must be initialized to parent counter in first iteration + let mut last_true_counter = parent_counter; + let mut last_false_counter = parent_counter; + // match control_statement { AstControlStatement::If(statement) => { // Loop through if/elif blocks for block in &statement.blocks { // Setup ast->id mapping, store region location - self.register_ast_list_as_region(&block.body, mapping_region_generator); - // Recurse - self.generate_coverage_records(&block.body, mapping_region_generator); + (last_true_counter, last_false_counter) = self.register_ast_list_as_branch_region( + &block.body, + mapping_region_generator, + last_true_counter, + last_false_counter, + ); + // Recurse into child blocks + self.generate_coverage_records( + &block.body, + mapping_region_generator, + last_true_counter, + ); } - // Else block - self.register_ast_list_as_region(&statement.else_block, mapping_region_generator); - self.generate_coverage_records(&statement.else_block, mapping_region_generator); + // Else block ast->id mapping + (last_true_counter, last_false_counter) = self.register_ast_list_as_branch_region( + &statement.else_block, + mapping_region_generator, + last_true_counter, + last_false_counter, + ); + // Recurse into child blocks + self.generate_coverage_records( + &statement.else_block, + mapping_region_generator, + last_true_counter, + ); + } + AstControlStatement::ForLoop(statement) => { + // Loop through for loop body + (last_true_counter, last_false_counter) = self.register_ast_list_as_loop_branch_region( + &statement.body, + &statement.end, + mapping_region_generator, + last_true_counter, + last_false_counter, + ); + self.generate_coverage_records( + &statement.body, + mapping_region_generator, + last_true_counter, + ); + } + AstControlStatement::WhileLoop(statement) => { + // Loop through while loop body + (last_true_counter, last_false_counter) = self.register_ast_list_as_loop_branch_region( + &statement.body, + &statement.condition, + mapping_region_generator, + last_true_counter, + last_false_counter, + ); + self.generate_coverage_records( + &statement.body, + mapping_region_generator, + last_true_counter, + ); + } + AstControlStatement::RepeatLoop(statement) => { + // Loop through while loop body + (last_true_counter, last_false_counter) = self.register_ast_list_as_loop_branch_region( + &statement.body, + &statement.condition, + mapping_region_generator, + last_true_counter, + last_false_counter, + ); + self.generate_coverage_records( + &statement.body, + mapping_region_generator, + last_true_counter, + ); } - AstControlStatement::ForLoop(statement) => (), - AstControlStatement::WhileLoop(statement) => (), - AstControlStatement::RepeatLoop(statement) => (), AstControlStatement::Case(statement) => { // Loop through case blocks for block in &statement.case_blocks { // Setup ast->id mapping, store region location - self.register_ast_list_as_region(&block.body, mapping_region_generator); + (last_true_counter, last_false_counter) = self.register_ast_list_as_branch_region( + &block.body, + mapping_region_generator, + last_true_counter, + last_false_counter, + ); // Recurse - self.generate_coverage_records(&block.body, mapping_region_generator); + self.generate_coverage_records( + &block.body, + mapping_region_generator, + last_true_counter, + ); } // Else block - self.register_ast_list_as_region(&statement.else_block, mapping_region_generator); - self.generate_coverage_records(&statement.else_block, mapping_region_generator); + (last_true_counter, last_false_counter) = self.register_ast_list_as_branch_region( + &statement.else_block, + mapping_region_generator, + last_true_counter, + last_false_counter, + ); + self.generate_coverage_records( + &statement.else_block, + mapping_region_generator, + last_true_counter, + ); } } } @@ -283,13 +438,16 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { // TODO - find me a better name /// Registers a Vec as a region, spanning first and last - fn register_ast_list_as_region( + /// Returns the true and false counters for DFS + fn register_ast_list_as_branch_region( &mut self, ast_list: &Vec, mapping_region_generator: &mut MappingRegionGenerator, - ) { + last_true_counter: Counter, + last_false_counter: Counter, + ) -> (Counter, Counter) { if ast_list.is_empty() { - return; + return (last_true_counter, last_false_counter); } // Create a span from first_block -> last_block @@ -298,8 +456,47 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { let span = first_block.location.span(&last_block.location); // Map the span, store the counter id in the lookup table (key at first_block.ast_id) - let ctr_id = mapping_region_generator.add_branch_mapping_region(&span); - self.ast_counter_lookup.insert(first_block.id, ctr_id.try_into().unwrap()); + let mapping_region = mapping_region_generator.add_branch_mapping_region(&span, last_false_counter); + self.ast_counter_lookup.insert(first_block.id, mapping_region.counter.id.try_into().unwrap()); + + (mapping_region.counter, mapping_region.false_counter) + } + + // TODO - find a better name + fn register_ast_list_as_loop_branch_region( + &mut self, + loop_body_ast_list: &Vec, + loop_condition_ast: &AstNode, + mapping_region_generator: &mut MappingRegionGenerator, + last_true_counter: Counter, + last_false_counter: Counter, + ) -> (Counter, Counter) { + if loop_body_ast_list.is_empty() { + return (last_true_counter, last_false_counter); + } + + // Create a counter for the condition ast + // this is a temporary hack, because false_counter will not actually create + // a counter that doesn't exist, only reference once + let condition_mapping_region = + mapping_region_generator.add_code_mapping_region(&loop_condition_ast.location); + + // Create a span from first_block -> last_block + let first_block = loop_body_ast_list.first().unwrap(); + let last_block = loop_body_ast_list.last().unwrap(); + let span = first_block.location.span(&last_block.location); + + // Map the span, store the counter id in the lookup table (key at first_block.ast_id) + let mapping_region = + mapping_region_generator.add_loop_branch_mapping_region(&span, condition_mapping_region.counter); + + // Map loop body -> true branch counter + // Map loop condition -> false branch counter + self.ast_counter_lookup.insert(first_block.id, mapping_region.counter.id.try_into().unwrap()); + self.ast_counter_lookup + .insert(loop_condition_ast.id, mapping_region.false_counter.id.try_into().unwrap()); + + (mapping_region.counter, mapping_region.false_counter) } pub fn get_increment_function(&self, module: &Module<'ink>) -> FunctionValue<'ink> { From a292e7af83079a4191cb1b6014ef4f4260edeb4d Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Wed, 31 Jan 2024 18:41:13 +0000 Subject: [PATCH 67/70] update examples --- examples/hello_world.st | 73 ++++++++++++++++++++++++++++++++++++++++- scripts/build.sh | 7 +++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/examples/hello_world.st b/examples/hello_world.st index 3868c2c3c27..5d157841061 100644 --- a/examples/hello_world.st +++ b/examples/hello_world.st @@ -6,12 +6,18 @@ END_VAR END_FUNCTION FUNCTION main : DINT + VAR + i : INT; + j : INT; + END_VAR puts('hello, world!$N'); if (1) then puts('true$N'); - if (2) then + if (0) then puts('truetrue$N'); + elsif (0) then + puts('truefalse$N'); else puts('truefalse$N'); end_if; @@ -19,4 +25,69 @@ FUNCTION main : DINT else puts('false$N'); end_if; + + CASE 1 OF + 1: + puts('Case 1$N'); + 2: + puts('Case 2$N'); + 3: + puts('Case 3$N'); + ELSE + puts('Default case$N'); + END_CASE; + + CASE 1 OF + 2: + puts('Case 2$N'); + ELSE + puts('Default case$N'); + END_CASE; + + if (0) then + puts('true$N'); + elsif (0) then + puts('false$N'); + elsif (1) then + if (1) then + puts('true$N'); + elsif (0) then + puts('false$N'); + else + puts('false$N'); + end_if + puts('true$N'); + else + // puts('false$N'); + end_if; + + FOR i := 1 TO 10 DO + FOR j := 2 TO 3 DO + puts('This is an inner loop iteration'); + END_FOR; + puts('This is a loop iteration $N'); + END_FOR; + + i := 0; + j := 0; + + WHILE i < 10 DO + puts('This is while loop iteration $N'); + i := i + 1; + + + WHILE j < 1 DO + puts('This is an inner while loop iteration $N'); + j := j + 1; + END_WHILE; + + END_WHILE; + + i := 0; + + REPEAT + puts('This is a repeat loop iteration $N'); + i := i + 1; + UNTIL i >= 10 END_REPEAT; + END_FUNCTION diff --git a/scripts/build.sh b/scripts/build.sh index 0f353394681..6f1e0054fd6 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -517,10 +517,15 @@ if [[ $build -ne 0 ]]; then llvm-profdata-14 merge $PROFRAW -o $PROFEXPORT echo "-----------------------------------" + # Generate report + echo "Generating profdata" + llvm-cov-14 report -instr-profile=$PROFEXPORT hello_world.st.out + echo "-----------------------------------" + # Show coverage echo "Coverage" llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT - llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT -show-regions + llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT -show-regions -show-branches=count --show-expansions echo "-----------------------------------" echo "Export" From be1ec4ef63f0867da5acedb99e7f823616c10d87 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Sat, 10 Feb 2024 12:08:14 +0000 Subject: [PATCH 68/70] address sanitizer and cleanup --- compiler/rustc_llvm_coverage/src/lib.rs | 9 +++++++- scripts/build.sh | 4 +++- src/codegen.rs | 3 +++ src/codegen/instrument.rs | 30 +++++++++++++++++++++++-- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_llvm_coverage/src/lib.rs b/compiler/rustc_llvm_coverage/src/lib.rs index 1f638850de6..e9325f6bd86 100644 --- a/compiler/rustc_llvm_coverage/src/lib.rs +++ b/compiler/rustc_llvm_coverage/src/lib.rs @@ -232,7 +232,14 @@ pub fn run_instrumentation_lowering_pass<'ctx>(module: &Module<'ctx>) { .unwrap(); // Run pass (uses new pass manager) - let _ = module.run_passes("instrprof", &machine, PassBuilderOptions::create()); + // When compiling as IR, run this: + let passes = "instrprof"; + // When compiling as object, run this: + // let passes = "instrprof,asan-module,function(asan)"; + match module.run_passes(passes, &machine, PassBuilderOptions::create()) { + Ok(_) => (), + Err(e) => panic!("Failed to run instrprof pass: {}", e), + } } /// Emits a increment counter call at the current builder position. diff --git a/scripts/build.sh b/scripts/build.sh index 6f1e0054fd6..d7d1d2cdafe 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -484,6 +484,8 @@ if [[ $build -ne 0 ]]; then echo "Running on example program:" # ./target/debug/plc --ir ./examples/simple_program.st # ./target/debug/plc --ir hello_world.st + + # export ASAN_OPTIONS=detect_odr_violation=0 ./target/debug/plc --ir ./examples/hello_world.st # echo "-----------------------------------" # cat ./hello_world.st.ll @@ -497,7 +499,7 @@ if [[ $build -ne 0 ]]; then # # Compile echo "Compiling hello_world.st.ll to hello_world.st.out" - clang++-14 -fprofile-instr-generate -fcoverage-mapping -Wl,-u,__llvm_profile_runtime -O0 hello_world.st.ll -o hello_world.st.out + clang++-14 -fprofile-instr-generate -fcoverage-mapping -fsanitize=address -Wl,-u,__llvm_profile_runtime -O0 hello_world.st.ll wrapper.cpp -o hello_world.st.out echo "Done!" echo "-----------------------------------" diff --git a/src/codegen.rs b/src/codegen.rs index e897277e3d3..58306346730 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -223,6 +223,9 @@ impl<'ink> CodeGen<'ink> { let pou_generator = PouGenerator::new(llvm, global_index, annotations, llvm_index); if let Some(instr_builder) = &mut self.instrument { + // Mark functions as sanitize + instr_builder.sanitize_functions(unit, llvm_index, &self.module); + // Generate mapping header instr_builder.initialize(&self.module); diff --git a/src/codegen/instrument.rs b/src/codegen/instrument.rs index 898555bfdac..6f180856e45 100644 --- a/src/codegen/instrument.rs +++ b/src/codegen/instrument.rs @@ -1,4 +1,5 @@ use super::LlvmTypedIndex; +use inkwell::attributes::{Attribute, AttributeLoc}; use inkwell::builder::Builder; use inkwell::context::Context; use inkwell::intrinsics::Intrinsic; @@ -33,7 +34,6 @@ pub struct CoverageInstrumentationBuilder<'ink> { /// Manages the creation of mapping regions for a given function #[derive(Debug)] struct MappingRegionGenerator { - // TODO - verify that counter ids are PER function pub mapping_regions: Vec, pub expressions: Vec, next_counter_id: u32, @@ -208,7 +208,6 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { ) { let (func_record, func_pgo_var) = self.function_pgos.get(func_name).unwrap(); - // TODO - see if expressions need different num_counters let pgo_pointer = func_pgo_var.as_pointer_value(); let num_counters = func_record.mapping_regions.len(); @@ -505,4 +504,31 @@ impl<'ink> CoverageInstrumentationBuilder<'ink> { increment_intrinsic_func } + + pub fn sanitize_functions( + &mut self, + unit: &CompilationUnit, + llvm_index: &LlvmTypedIndex, + module: &Module<'ink>, + ) { + for implementation in &unit.implementations { + // Skip non-internal functions (external links + built-ins) + if implementation.linkage != LinkageType::Internal { + continue; + } + // Skip no-definition functions + // TODO - investigate which functions don't have definitions and why + if module.get_function(&implementation.name).is_none() { + println!("Skipping undefined function: {}", &implementation.name); + continue; + } + + let func = llvm_index.find_associated_implementation(&implementation.name).unwrap(); + + let context = module.get_context(); + let sanitizer_attribute_id = Attribute::get_named_enum_kind_id("sanitize_address"); + let sanitizer_attribute = context.create_enum_attribute(sanitizer_attribute_id, 0); + func.add_attribute(AttributeLoc::Function, sanitizer_attribute); + } + } } From f61b6939e9ce80b5c321801c99cd30704de74c66 Mon Sep 17 00:00:00 2001 From: Corban Villa Date: Fri, 5 Apr 2024 21:22:54 +0000 Subject: [PATCH 69/70] fix hardcoded paths --- src/codegen.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/codegen.rs b/src/codegen.rs index 58306346730..12f5afa143d 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -1,6 +1,7 @@ // Copyright (c) 2020 Ghaith Hachem and Mathias Rieder use std::{ cell::RefCell, + env, fs, ops::Deref, path::{Path, PathBuf}, }; @@ -100,8 +101,9 @@ impl<'ink> CodeGen<'ink> { let debug = debug::DebugBuilderEnum::new(context, &module, root, optimization_level, debug_level); // TODO - disable instr here - // TODO - local path shouldn't be hardcoded - let filenames = vec!["/workspaces/corbanvilla_rusty".to_string(), module_location.to_string()]; + let current_dir = env::current_dir().expect("Failed to get current directory"); + + let filenames = vec![current_dir.to_str().unwrap().to_string(), module_location.to_string()]; let instr_builder = instrument::CoverageInstrumentationBuilder::new(context, filenames); // instrument::CoverageInstrumentationBuilder::new(/*context, */ &module, filenames); From 5cd15bf3f2860380ff6b8772205da429c20addca Mon Sep 17 00:00:00 2001 From: Animcogn Date: Wed, 20 Nov 2024 15:11:18 +0400 Subject: [PATCH 70/70] Update build.sh --- scripts/build.sh | 90 ++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index d7d1d2cdafe..3a7ec3bfb58 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -480,62 +480,62 @@ fi if [[ $build -ne 0 ]]; then run_build # Test a program - echo "-----------------------------------" - echo "Running on example program:" - # ./target/debug/plc --ir ./examples/simple_program.st - # ./target/debug/plc --ir hello_world.st - - # export ASAN_OPTIONS=detect_odr_violation=0 - ./target/debug/plc --ir ./examples/hello_world.st # echo "-----------------------------------" - # cat ./hello_world.st.ll - echo "-----------------------------------" + # echo "Running on example program:" + # # ./target/debug/plc --ir ./examples/simple_program.st + # # ./target/debug/plc --ir hello_world.st + + # # export ASAN_OPTIONS=detect_odr_violation=0 + # #./target/debug/plc --ir ./examples/hello_world.st + # # echo "-----------------------------------" + # # cat ./hello_world.st.ll + # echo "-----------------------------------" - # Cleanup prof - PROFRAW=./default.profraw - PROFEXPORT=./default.profexport - rm -f $PROFRAW $PROFEXPORT - rm -f hello_world.st.out + # # Cleanup prof + # PROFRAW=./default.profraw + # PROFEXPORT=./default.profexport + # rm -f $PROFRAW $PROFEXPORT + # rm -f hello_world.st.out - # # Compile - echo "Compiling hello_world.st.ll to hello_world.st.out" - clang++-14 -fprofile-instr-generate -fcoverage-mapping -fsanitize=address -Wl,-u,__llvm_profile_runtime -O0 hello_world.st.ll wrapper.cpp -o hello_world.st.out - echo "Done!" - echo "-----------------------------------" + # # # Compile + # echo "Compiling hello_world.st.ll to hello_world.st.out" + # clang++-14 -fprofile-instr-generate -fcoverage-mapping -fsanitize=address -Wl,-u,__llvm_profile_runtime -O0 hello_world.st.ll wrapper.cpp -o hello_world.st.out + # echo "Done!" + # echo "-----------------------------------" - # # Run - echo "Running program" - ./hello_world.st.out - echo "Done!" - echo "-----------------------------------" + # # # Run + # echo "Running program" + # ./hello_world.st.out + # echo "Done!" + # echo "-----------------------------------" - # Show profdata - echo "Showing profdata" - llvm-profdata-14 show -all-functions $PROFRAW - echo "-----------------------------------" + # # Show profdata + # echo "Showing profdata" + # llvm-profdata-14 show -all-functions $PROFRAW + # echo "-----------------------------------" - # Generate profdata - echo "Generating profdata" - llvm-profdata-14 merge $PROFRAW -o $PROFEXPORT - echo "-----------------------------------" + # # Generate profdata + # echo "Generating profdata" + # llvm-profdata-14 merge $PROFRAW -o $PROFEXPORT + # echo "-----------------------------------" - # Generate report - echo "Generating profdata" - llvm-cov-14 report -instr-profile=$PROFEXPORT hello_world.st.out - echo "-----------------------------------" + # # Generate report + # echo "Generating profdata" + # llvm-cov-14 report -instr-profile=$PROFEXPORT hello_world.st.out + # echo "-----------------------------------" - # Show coverage - echo "Coverage" - llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT - llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT -show-regions -show-branches=count --show-expansions - echo "-----------------------------------" + # # Show coverage + # echo "Coverage" + # llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT + # llvm-cov-14 show ./hello_world.st.out -instr-profile=$PROFEXPORT -show-regions -show-branches=count --show-expansions + # echo "-----------------------------------" - echo "Export" - # llvm-cov-14 export -instr-profile=$PROFEXPORT hello_world.st.out - echo "-----------------------------------" + # echo "Export" + # # llvm-cov-14 export -instr-profile=$PROFEXPORT hello_world.st.out + # echo "-----------------------------------" #Build the standard functions - # run_std_build + run_std_build fi if [[ $package -ne 0 ]]; then