Rollup merge of #101339 - the8472:ci-randomize-debug, r=Mark-Simulacrum

enable -Zrandomize-layout in debug CI builds

This builds rustc/libs/tools with `-Zrandomize-layout` on *-debug CI runners.

Only a handful of tests and asserts break with that enabled, which is promising. One test was fixable, the rest is dealt with by disabling them through new cargo features or compiletest directives.

The config.toml flag `rust.randomize-layout` defaults to false, so it has to be explicitly enabled for now.

Author: matthiaskrgr

Cargo.lock

@@ -3569,6 +3569,7 @@ dependencies = [
  "rustc_hir_pretty",
  "rustc_hir_typeck",
  "rustc_incremental",
+ "rustc_index",
  "rustc_infer",
  "rustc_interface",
  "rustc_lint",

compiler/rustc/Cargo.toml

@@ -30,5 +30,6 @@ features = ['unprefixed_malloc_on_supported_platforms']
 jemalloc = ['dep:jemalloc-sys']
 llvm = ['rustc_driver_impl/llvm']
 max_level_info = ['rustc_driver_impl/max_level_info']
+rustc_randomized_layouts = ['rustc_driver_impl/rustc_randomized_layouts']
 rustc_use_parallel_compiler = ['rustc_driver_impl/rustc_use_parallel_compiler']
 # tidy-alphabetical-end

compiler/rustc_abi/src/layout.rs

@@ -968,8 +968,8 @@ fn univariant<
     let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
     let mut max_repr_align = repr.align;
     let mut inverse_memory_index: IndexVec<u32, FieldIdx> = fields.indices().collect();
-    let optimize = !repr.inhibit_struct_field_reordering();
-    if optimize && fields.len() > 1 {
+    let optimize_field_order = !repr.inhibit_struct_field_reordering();
+    if optimize_field_order && fields.len() > 1 {
         let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
         let optimizing = &mut inverse_memory_index.raw[..end];
         let fields_excluding_tail = &fields.raw[..end];
@@ -1176,7 +1176,7 @@ fn univariant<
     // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
     // Field 5 would be the first element, so memory_index is i:
     // Note: if we didn't optimize, it's already right.
-    let memory_index = if optimize {
+    let memory_index = if optimize_field_order {
         inverse_memory_index.invert_bijective_mapping()
     } else {
         debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices()));
@@ -1189,6 +1189,9 @@ fn univariant<
     }
     let mut layout_of_single_non_zst_field = None;
     let mut abi = Abi::Aggregate { sized };
+
+    let optimize_abi = !repr.inhibit_newtype_abi_optimization();
+
     // Try to make this a Scalar/ScalarPair.
     if sized && size.bytes() > 0 {
         // We skip *all* ZST here and later check if we are good in terms of alignment.
@@ -1205,7 +1208,7 @@ fn univariant<
                     match field.abi {
                         // For plain scalars, or vectors of them, we can't unpack
                         // newtypes for `#[repr(C)]`, as that affects C ABIs.
-                        Abi::Scalar(_) | Abi::Vector { .. } if optimize => {
+                        Abi::Scalar(_) | Abi::Vector { .. } if optimize_abi => {
                             abi = field.abi;
                         }
                         // But scalar pairs are Rust-specific and get

compiler/rustc_abi/src/lib.rs

@@ -43,14 +43,17 @@ bitflags! {
         const IS_SIMD            = 1 << 1;
         const IS_TRANSPARENT     = 1 << 2;
         // Internal only for now. If true, don't reorder fields.
+        // On its own it does not prevent ABI optimizations.
         const IS_LINEAR          = 1 << 3;
-        // If true, the type's layout can be randomized using
-        // the seed stored in `ReprOptions.field_shuffle_seed`
+        // If true, the type's crate has opted into layout randomization.
+        // Other flags can still inhibit reordering and thus randomization.
+        // The seed stored in `ReprOptions.field_shuffle_seed`.
         const RANDOMIZE_LAYOUT   = 1 << 4;
         // Any of these flags being set prevent field reordering optimisation.
-        const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits()
+        const FIELD_ORDER_UNOPTIMIZABLE   = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
                                  | ReprFlags::IS_LINEAR.bits();
+        const ABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();
     }
 }
 
@@ -139,10 +142,14 @@ impl ReprOptions {
         self.c() || self.int.is_some()
     }
 
+    pub fn inhibit_newtype_abi_optimization(&self) -> bool {
+        self.flags.intersects(ReprFlags::ABI_UNOPTIMIZABLE)
+    }
+
     /// Returns `true` if this `#[repr()]` guarantees a fixed field order,
     /// e.g. `repr(C)` or `repr(<int>)`.
     pub fn inhibit_struct_field_reordering(&self) -> bool {
-        self.flags.intersects(ReprFlags::IS_UNOPTIMISABLE) || self.int.is_some()
+        self.flags.intersects(ReprFlags::FIELD_ORDER_UNOPTIMIZABLE) || self.int.is_some()
     }
 
     /// Returns `true` if this type is valid for reordering and `-Z randomize-layout`

compiler/rustc_driver_impl/Cargo.toml

@@ -23,6 +23,7 @@ rustc_hir_analysis = { path = "../rustc_hir_analysis" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_hir_typeck = { path = "../rustc_hir_typeck" }
 rustc_incremental = { path = "../rustc_incremental" }
+rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_interface = { path = "../rustc_interface" }
 rustc_lint = { path = "../rustc_lint" }
@@ -72,6 +73,10 @@ ctrlc = "3.4.4"
 # tidy-alphabetical-start
 llvm = ['rustc_interface/llvm']
 max_level_info = ['rustc_log/max_level_info']
+rustc_randomized_layouts = [
+    'rustc_index/rustc_randomized_layouts',
+    'rustc_middle/rustc_randomized_layouts'
+]
 rustc_use_parallel_compiler = [
     'rustc_data_structures/rustc_use_parallel_compiler',
     'rustc_interface/rustc_use_parallel_compiler',

compiler/rustc_index/Cargo.toml

@@ -20,4 +20,5 @@ nightly = [
     "dep:rustc_macros",
     "rustc_index_macros/nightly",
 ]
+rustc_randomized_layouts = []
 # tidy-alphabetical-end

compiler/rustc_index/src/lib.rs

@@ -33,8 +33,19 @@ pub use vec::IndexVec;
 ///
 /// </div>
 #[macro_export]
+#[cfg(not(feature = "rustc_randomized_layouts"))]
 macro_rules! static_assert_size {
     ($ty:ty, $size:expr) => {
         const _: [(); $size] = [(); ::std::mem::size_of::<$ty>()];
     };
 }
+
+#[macro_export]
+#[cfg(feature = "rustc_randomized_layouts")]
+macro_rules! static_assert_size {
+    ($ty:ty, $size:expr) => {
+        // no effect other than using the statements.
+        // struct sizes are not deterministic under randomized layouts
+        const _: (usize, usize) = ($size, ::std::mem::size_of::<$ty>());
+    };
+}

compiler/rustc_middle/Cargo.toml

@@ -40,5 +40,6 @@ tracing = "0.1"
 
 [features]
 # tidy-alphabetical-start
+rustc_randomized_layouts = []
 rustc_use_parallel_compiler = ["dep:rustc-rayon-core"]
 # tidy-alphabetical-end

compiler/rustc_middle/src/query/plumbing.rs

@@ -337,6 +337,7 @@ macro_rules! define_callbacks {
                 // Ensure that values grow no larger than 64 bytes by accident.
                 // Increase this limit if necessary, but do try to keep the size low if possible
                 #[cfg(target_pointer_width = "64")]
+                #[cfg(not(feature = "rustc_randomized_layouts"))]
                 const _: () = {
                     if mem::size_of::<Value<'static>>() > 64 {
                         panic!("{}", concat!(

compiler/rustc_middle/src/ty/mod.rs

@@ -35,6 +35,7 @@ use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_errors::{Diag, ErrorGuaranteed, StashKey};
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap};
+use rustc_hir::LangItem;
 use rustc_index::IndexVec;
 use rustc_macros::{
     extension, Decodable, Encodable, HashStable, TyDecodable, TyEncodable, TypeFoldable,
@@ -1570,8 +1571,15 @@ impl<'tcx> TyCtxt<'tcx> {
             flags.insert(ReprFlags::RANDOMIZE_LAYOUT);
         }
 
+        // box is special, on the one hand the compiler assumes an ordered layout, with the pointer
+        // always at offset zero. On the other hand we want scalar abi optimizations.
+        let is_box = self.is_lang_item(did.to_def_id(), LangItem::OwnedBox);
+
         // This is here instead of layout because the choice must make it into metadata.
-        if !self.consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did))) {
+        if is_box
+            || !self
+                .consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did)))
+        {
             flags.insert(ReprFlags::IS_LINEAR);
         }