Prototype VaList proposal

Author: beetrees

compiler/rustc_abi/src/layout/ty.rs

@@ -182,6 +182,9 @@ pub trait TyAbiInterface<'a, C>: Sized + std::fmt::Debug {
     fn is_tuple(this: TyAndLayout<'a, Self>) -> bool;
     fn is_unit(this: TyAndLayout<'a, Self>) -> bool;
     fn is_transparent(this: TyAndLayout<'a, Self>) -> bool;
+    /// Returns `true` if the type is always passed indirectly. Currently only
+    /// used for `VaList`s.
+    fn is_pass_indirectly(this: TyAndLayout<'a, Self>) -> bool;
 }
 
 impl<'a, Ty> TyAndLayout<'a, Ty> {
@@ -279,6 +282,13 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         Ty::is_transparent(self)
     }
 
+    pub fn is_pass_indirectly<C>(self) -> bool
+    where
+        Ty: TyAbiInterface<'a, C>,
+    {
+        Ty::is_pass_indirectly(self)
+    }
+
     /// Finds the one field that is not a 1-ZST.
     /// Returns `None` if there are multiple non-1-ZST fields or only 1-ZST-fields.
     pub fn non_1zst_field<C>(&self, cx: &C) -> Option<(FieldIdx, Self)>

compiler/rustc_abi/src/lib.rs

@@ -93,6 +93,9 @@ bitflags! {
         // Other flags can still inhibit reordering and thus randomization.
         // The seed stored in `ReprOptions.field_shuffle_seed`.
         const RANDOMIZE_LAYOUT   = 1 << 4;
+        // If true, the type is always passed indirectly in C-like ABIs.
+        // Currently only used for `VaList`s.
+        const PASS_INDIRECTLY    = 1 << 5;
         // Any of these flags being set prevent field reordering optimisation.
         const FIELD_ORDER_UNOPTIMIZABLE   = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()

compiler/rustc_codegen_ssa/src/traits/intrinsic.rs

@@ -30,10 +30,10 @@ pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes {
         vtable_byte_offset: u64,
         typeid: Self::Metadata,
     ) -> Self::Value;
-    /// Trait method used to inject `va_start` on the "spoofed" `VaListImpl` in
+    /// Trait method used to inject `va_start` on the "spoofed" `VaList` in
     /// Rust defined C-variadic functions.
     fn va_start(&mut self, val: Self::Value) -> Self::Value;
-    /// Trait method used to inject `va_end` on the "spoofed" `VaListImpl` before
+    /// Trait method used to inject `va_end` on the "spoofed" `VaList` before
     /// Rust defined C-variadic functions return.
     fn va_end(&mut self, val: Self::Value) -> Self::Value;
 }

compiler/rustc_middle/src/ty/layout.rs

@@ -3,7 +3,7 @@ use std::{cmp, fmt};
 
 use rustc_abi::{
     AddressSpace, Align, ExternAbi, FieldIdx, FieldsShape, HasDataLayout, LayoutData, PointeeInfo,
-    PointerKind, Primitive, ReprOptions, Scalar, Size, TagEncoding, TargetDataLayout,
+    PointerKind, Primitive, ReprFlags, ReprOptions, Scalar, Size, TagEncoding, TargetDataLayout,
     TyAbiInterface, VariantIdx, Variants,
 };
 use rustc_error_messages::DiagMessage;
@@ -1138,6 +1138,10 @@ where
     fn is_transparent(this: TyAndLayout<'tcx>) -> bool {
         matches!(this.ty.kind(), ty::Adt(def, _) if def.repr().transparent())
     }
+
+    fn is_pass_indirectly(this: TyAndLayout<'tcx>) -> bool {
+        matches!(this.ty.kind(), ty::Adt(def, _) if def.repr().flags.contains(ReprFlags::PASS_INDIRECTLY))
+    }
 }
 
 /// Calculates whether a function's ABI can unwind or not.

compiler/rustc_middle/src/ty/mod.rs

@@ -1585,6 +1585,12 @@ impl<'tcx> TyCtxt<'tcx> {
             flags.insert(ReprFlags::IS_LINEAR);
         }
 
+        if self.is_lang_item(did.to_def_id(), LangItem::VaList)
+            && !flags.contains(ReprFlags::IS_TRANSPARENT)
+        {
+            flags.insert(ReprFlags::PASS_INDIRECTLY);
+        }
+
         ReprOptions { int: size, align: max_align, pack: min_pack, flags, field_shuffle_seed }
     }
 

compiler/rustc_target/src/callconv/aarch64.rs

@@ -114,6 +114,12 @@ where
         // Not touching this...
         return;
     }
+    // `is_pass_indirectly` is only `true` for `VaList`, which would be passed indirectly by the
+    // logic below anyway, so this is just here to make it explicit that this case is handled.
+    if arg.layout.is_pass_indirectly() {
+        arg.make_indirect();
+        return;
+    }
     if !arg.layout.is_aggregate() {
         if kind == AbiKind::DarwinPCS {
             // On Darwin, when passing an i8/i16, it must be sign-extended to 32 bits,

compiler/rustc_target/src/callconv/powerpc.rs

@@ -1,3 +1,5 @@
+use rustc_abi::TyAbiInterface;
+
 use crate::callconv::{ArgAbi, FnAbi};
 use crate::spec::HasTargetSpec;
 
@@ -9,7 +11,10 @@ fn classify_ret<Ty>(ret: &mut ArgAbi<'_, Ty>) {
     }
 }
 
-fn classify_arg<Ty>(cx: &impl HasTargetSpec, arg: &mut ArgAbi<'_, Ty>) {
+fn classify_arg<'a, Ty, C>(cx: &impl HasTargetSpec, arg: &mut ArgAbi<'a, Ty>)
+where
+    Ty: TyAbiInterface<'a, C> + Copy,
+{
     if arg.is_ignore() {
         // powerpc-unknown-linux-{gnu,musl,uclibc} doesn't ignore ZSTs.
         if cx.target_spec().os == "linux"
@@ -20,14 +25,19 @@ fn classify_arg<Ty>(cx: &impl HasTargetSpec, arg: &mut ArgAbi<'_, Ty>) {
         }
         return;
     }
-    if arg.layout.is_aggregate() {
+    // `is_pass_indirectly` is only `true` for `VaList` which is already an aggregate, so the
+    // `.is_pass_indirectly()` call is just to make it explicit that this case is handled.
+    if arg.layout.is_aggregate() || arg.layout.is_pass_indirectly() {
         arg.make_indirect();
     } else {
         arg.extend_integer_width_to(32);
     }
 }
 
-pub(crate) fn compute_abi_info<Ty>(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'_, Ty>) {
+pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'a, Ty>)
+where
+    Ty: TyAbiInterface<'a, C> + Copy,
+{
     if !fn_abi.ret.is_ignore() {
         classify_ret(&mut fn_abi.ret);
     }

compiler/rustc_target/src/callconv/s390x.rs

@@ -37,6 +37,12 @@ where
         }
         return;
     }
+    // `is_pass_indirectly` is only `true` for `VaList`, which would be passed indirectly by the
+    // logic below anyway, so this is just here to make it explicit that this case is handled.
+    if arg.layout.is_pass_indirectly() {
+        arg.make_indirect();
+        return;
+    }
 
     let size = arg.layout.size;
     if size.bits() <= 128 {

compiler/rustc_target/src/callconv/x86_64.rs

@@ -186,6 +186,11 @@ where
             // Not touching this...
             return;
         }
+        if is_arg && arg.layout.is_pass_indirectly() {
+            int_regs = int_regs.saturating_sub(1);
+            arg.make_indirect();
+            return;
+        }
         let mut cls_or_mem = classify_arg(cx, arg);
 
         if is_arg {

compiler/rustc_target/src/callconv/x86_win64.rs

@@ -1,11 +1,14 @@
-use rustc_abi::{BackendRepr, Float, Integer, Primitive, RegKind, Size};
+use rustc_abi::{BackendRepr, Float, Integer, Primitive, RegKind, Size, TyAbiInterface};
 
 use crate::callconv::{ArgAbi, FnAbi, Reg};
 use crate::spec::{HasTargetSpec, RustcAbi};
 
 // Win64 ABI: https://docs.microsoft.com/en-us/cpp/build/parameter-passing
 
-pub(crate) fn compute_abi_info<Ty>(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'_, Ty>) {
+pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'a, Ty>)
+where
+    Ty: TyAbiInterface<'a, C> + Copy,
+{
     let fixup = |a: &mut ArgAbi<'_, Ty>, is_ret: bool| {
         match a.layout.backend_repr {
             BackendRepr::Memory { sized: false } => {}
@@ -59,6 +62,14 @@ pub(crate) fn compute_abi_info<Ty>(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'
             arg.make_indirect_from_ignore();
             continue;
         }
+        // The `win64` ABI can be used on non-Windows targets which set `PASS_INDIRECTLY` on
+        // `VaList`, so that case is handled here. `is_pass_indirectly` is only `true` for the
+        // System V `VaList`, which would be passed indirectly by `fixup` anyway, so this is just
+        // here to make it explicit that this case is handled.
+        if arg.layout.is_pass_indirectly() {
+            arg.make_indirect();
+            continue;
+        }
         fixup(arg, false);
     }
     // FIXME: We should likely also do something about ZST return types, similar to above.