Allow SVE types to be taken by reference.

Ensures that an SVE type can have a reference taken to it. This
currently emits a `dereferenceable` attribute for the ptr using the
element size as the number of bytes. While not perfect this is correct
as a vector will always have a least one primitive.

Author: JamieCunliffe

compiler/rustc_codegen_llvm/src/builder.rs

@@ -538,10 +538,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             }
         }
 
-        assert_eq!(
-            place.val.llextra.is_some(),
-            place.layout.is_unsized() && !place.layout.is_runtime_sized()
-        );
+        if !place.layout.is_runtime_sized() {
+            assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
+        }
 
         if place.layout.is_zst() {
             return OperandRef::zero_sized(place.layout);

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -248,7 +248,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
             if memory_locals.contains(local) {
                 debug!("alloc: {:?} -> place", local);
-                if layout.is_unsized() {
+                if layout.is_unsized() && !layout.is_runtime_sized() {
                     LocalRef::UnsizedPlace(PlaceRef::alloca_unsized_indirect(&mut start_bx, layout))
                 } else {
                     LocalRef::Place(PlaceRef::alloca(&mut start_bx, layout))

compiler/rustc_codegen_ssa/src/traits/type_.rs

@@ -87,7 +87,7 @@ pub trait DerivedTypeMethods<'tcx>: BaseTypeMethods<'tcx> + MiscMethods<'tcx> {
 
     fn type_has_metadata(&self, ty: Ty<'tcx>) -> bool {
         let param_env = ty::ParamEnv::reveal_all();
-        if ty.is_sized(self.tcx(), param_env) {
+        if ty.is_sized(self.tcx(), param_env) || ty.is_scalable_simd() {
             return false;
         }
 

compiler/rustc_mir_build/src/build/expr/as_operand.rs

@@ -166,11 +166,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             let ty = expr.ty;
             let param_env = this.param_env;
 
-            if !ty.is_sized(tcx, param_env) {
-                // !sized means !copy, so this is an unsized move unless it's a scalable SIMD type.
-                if !ty.is_scalable_simd() {
-                    assert!(!ty.is_copy_modulo_regions(tcx, param_env));
-                }
+            if !ty.is_sized(tcx, param_env) && !ty.is_scalable_simd() {
+                // !sized means !copy, so this is an unsized move.
+                assert!(!ty.is_copy_modulo_regions(tcx, param_env));
 
                 // As described above, detect the case where we are passing a value of unsized
                 // type, and that value is coming from the deref of a box.

compiler/rustc_monomorphize/src/collector.rs

@@ -1017,7 +1017,7 @@ fn find_vtable_types_for_unsizing<'tcx>(
     let ptr_vtable = |inner_source: Ty<'tcx>, inner_target: Ty<'tcx>| {
         let param_env = ty::ParamEnv::reveal_all();
         let type_has_metadata = |ty: Ty<'tcx>| -> bool {
-            if ty.is_sized(tcx.tcx, param_env) {
+            if ty.is_sized(tcx.tcx, param_env) || ty.is_scalable_simd() {
                 return false;
             }
             let tail = tcx.struct_tail_erasing_lifetimes(ty, param_env);

compiler/rustc_ty_utils/src/layout.rs

@@ -201,6 +201,9 @@ fn layout_of_uncached<'tcx>(
             }
 
             let pointee = tcx.normalize_erasing_regions(param_env, pointee);
+            if pointee.is_scalable_simd() {
+                return Ok(tcx.mk_layout(LayoutS::scalar(cx, data_ptr)));
+            }
             if pointee.is_sized(tcx, param_env) {
                 return Ok(tcx.mk_layout(LayoutS::scalar(cx, data_ptr)));
             }

library/core/src/lib.rs

@@ -56,6 +56,7 @@
 #![cfg(not(test))]
 //
 #![stable(feature = "core", since = "1.6.0")]
+#![cfg_attr(not(bootstrap), feature(repr_scalable))]
 #![doc(
     html_playground_url = "https://play.rust-lang.org/",
     issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/",

tests/codegen/simd/allow-scalable-references.rs

@@ -0,0 +1,58 @@
+//@ only-aarch64
+//@ compile-flags: -C opt-level=2 --edition=2021
+
+#![crate_type = "lib"]
+#![allow(incomplete_features, internal_features, improper_ctypes)]
+#![feature(
+    repr_simd,
+    repr_scalable,
+    simd_ffi,
+    unsized_locals,
+    unsized_fn_params,
+    link_llvm_intrinsics
+)]
+
+#[repr(simd, scalable(4))]
+#[allow(non_camel_case_types)]
+pub struct svint32_t {
+    _ty: [i32],
+}
+
+#[inline(never)]
+#[target_feature(enable = "sve")]
+pub unsafe fn svdup_n_s32(op: i32) -> svint32_t {
+    extern "C" {
+        #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.dup.x.nxv4i32")]
+        fn _svdup_n_s32(op: i32) -> svint32_t;
+    }
+    unsafe { _svdup_n_s32(op) }
+}
+
+#[inline]
+#[target_feature(enable = "sve,sve2")]
+pub unsafe fn svxar_n_s32<const IMM3: i32>(op1: svint32_t, op2: svint32_t) -> svint32_t {
+    extern "C" {
+        #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.xar.nxv4i32")]
+        fn _svxar_n_s32(op1: svint32_t, op2: svint32_t, imm3: i32) -> svint32_t;
+    }
+    unsafe { _svxar_n_s32(op1, op2, IMM3) }
+}
+
+#[inline(never)]
+#[no_mangle]
+#[target_feature(enable = "sve,sve2")]
+// CHECK: define <vscale x 4 x i32> @pass_as_ref(ptr noalias nocapture noundef readonly align 4 dereferenceable(4) %a, <vscale x 4 x i32> %b)
+pub unsafe fn pass_as_ref(a: &svint32_t, b: svint32_t) -> svint32_t {
+    // CHECK: load <vscale x 4 x i32>, ptr %a, align 4
+    svxar_n_s32::<1>(*a, b)
+}
+
+#[no_mangle]
+#[target_feature(enable = "sve,sve2")]
+// CHECK: define <vscale x 4 x i32> @test()
+pub unsafe fn test() -> svint32_t {
+    let a = svdup_n_s32(1);
+    let b = svdup_n_s32(2);
+    // CHECK: call <vscale x 4 x i32> @pass_as_ref(ptr noalias noundef nonnull readonly align 4 dereferenceable(4) %a, <vscale x 4 x i32> %b)
+    pass_as_ref(&a, b)
+}