Skip to content

Commit 74f61f4

Browse files
davidtwcoJamieCunliffe
davidtwco
and
JamieCunliffe
committed
codegen: implement repr(scalable)
Introduces `BackendRepr::ScalableVector` corresponding to scalable vector types annotated with `repr(scalable)` which lowers to a scalable vector type in LLVM. Co-authored-by: Jamie Cunliffe <Jamie.Cunliffe@arm.com>
1 parent 73f7524 commit 74f61f4

File tree

39 files changed

+490
-47
lines changed

39 files changed

+490
-47
lines changed

‎compiler/rustc_abi/src/callconv.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
8282
}))
8383
}
8484

85+
BackendRepr::ScalableVector { .. } => Err(Heterogeneous),
86+
8587
BackendRepr::ScalarPair(..) | BackendRepr::Memory { sized: true } => {
8688
// Helper for computing `homogeneous_aggregate`, allowing a custom
8789
// starting offset (used below for handling variants).

‎compiler/rustc_abi/src/layout.rs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
151151
element: F,
152152
count: u64,
153153
repr_packed: bool,
154+
scalable: Option<u32>,
154155
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
155156
let elt = element.as_ref();
156157
if count == 0 {
@@ -169,7 +170,12 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
169170
let dl = self.cx.data_layout();
170171
let size =
171172
elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
172-
let (repr, align) = if repr_packed && !count.is_power_of_two() {
173+
let (repr, align) = if let Some(elt) = scalable {
174+
(
175+
BackendRepr::ScalableVector { element: e_repr, count: elt as u64 },
176+
dl.llvmlike_vector_align(size),
177+
)
178+
} else if repr_packed && !count.is_power_of_two() {
173179
// Non-power-of-two vectors have padding up to the next power-of-two.
174180
// If we're a packed repr, remove the padding while keeping the alignment as close
175181
// to a vector as possible.
@@ -461,6 +467,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
461467
BackendRepr::Scalar(..)
462468
| BackendRepr::ScalarPair(..)
463469
| BackendRepr::SimdVector { .. }
470+
| BackendRepr::ScalableVector { .. }
464471
| BackendRepr::Memory { .. } => repr,
465472
},
466473
};
@@ -532,7 +539,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
532539
hide_niches(a);
533540
hide_niches(b);
534541
}
535-
BackendRepr::SimdVector { element, count: _ } => hide_niches(element),
542+
BackendRepr::SimdVector { element, .. }
543+
| BackendRepr::ScalableVector { element, .. } => hide_niches(element),
536544
BackendRepr::Memory { sized: _ } => {}
537545
}
538546
st.largest_niche = None;

‎compiler/rustc_abi/src/lib.rs

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1449,6 +1449,10 @@ impl AddressSpace {
14491449
pub enum BackendRepr {
14501450
Scalar(Scalar),
14511451
ScalarPair(Scalar, Scalar),
1452+
ScalableVector {
1453+
element: Scalar,
1454+
count: u64,
1455+
},
14521456
SimdVector {
14531457
element: Scalar,
14541458
count: u64,
@@ -1467,6 +1471,9 @@ impl BackendRepr {
14671471
match *self {
14681472
BackendRepr::Scalar(_)
14691473
| BackendRepr::ScalarPair(..)
1474+
// FIXME(repr_scalable): Scalable vectors are forced to be `Sized` while the
1475+
// `sized_hierarchy` feature is not yet fully implemented
1476+
| BackendRepr::ScalableVector { .. }
14701477
| BackendRepr::SimdVector { .. } => false,
14711478
BackendRepr::Memory { sized } => !sized,
14721479
}
@@ -1507,7 +1514,9 @@ impl BackendRepr {
15071514
BackendRepr::Scalar(s) => Some(s.align(cx).abi),
15081515
BackendRepr::ScalarPair(s1, s2) => Some(s1.align(cx).max(s2.align(cx)).abi),
15091516
// The align of a Vector can vary in surprising ways
1510-
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
1517+
BackendRepr::SimdVector { .. }
1518+
| BackendRepr::Memory { .. }
1519+
| BackendRepr::ScalableVector { .. } => None,
15111520
}
15121521
}
15131522

@@ -1529,7 +1538,9 @@ impl BackendRepr {
15291538
Some(size)
15301539
}
15311540
// The size of a Vector can vary in surprising ways
1532-
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
1541+
BackendRepr::SimdVector { .. }
1542+
| BackendRepr::Memory { .. }
1543+
| BackendRepr::ScalableVector { .. } => None,
15331544
}
15341545
}
15351546

@@ -1544,6 +1555,9 @@ impl BackendRepr {
15441555
BackendRepr::SimdVector { element: element.to_union(), count }
15451556
}
15461557
BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },
1558+
BackendRepr::ScalableVector { element, count } => {
1559+
BackendRepr::ScalableVector { element: element.to_union(), count }
1560+
}
15471561
}
15481562
}
15491563

@@ -1784,7 +1798,9 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
17841798
/// Returns `true` if this is an aggregate type (including a ScalarPair!)
17851799
pub fn is_aggregate(&self) -> bool {
17861800
match self.backend_repr {
1787-
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => false,
1801+
BackendRepr::Scalar(_)
1802+
| BackendRepr::SimdVector { .. }
1803+
| BackendRepr::ScalableVector { .. } => false,
17881804
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => true,
17891805
}
17901806
}
@@ -1878,6 +1894,19 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
18781894
self.is_sized() && self.size.bytes() == 0 && self.align.abi.bytes() == 1
18791895
}
18801896

1897+
/// Returns `true` if the size of the type is only known at runtime.
1898+
pub fn is_runtime_sized(&self) -> bool {
1899+
matches!(self.backend_repr, BackendRepr::ScalableVector { .. })
1900+
}
1901+
1902+
/// Returns the elements count of a scalable vector.
1903+
pub fn scalable_vector_element_count(&self) -> Option<u64> {
1904+
match self.backend_repr {
1905+
BackendRepr::ScalableVector { count, .. } => Some(count),
1906+
_ => None,
1907+
}
1908+
}
1909+
18811910
/// Returns `true` if the type is a ZST and not unsized.
18821911
///
18831912
/// Note that this does *not* imply that the type is irrelevant for layout! It can still have
@@ -1886,6 +1915,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
18861915
match self.backend_repr {
18871916
BackendRepr::Scalar(_)
18881917
| BackendRepr::ScalarPair(..)
1918+
| BackendRepr::ScalableVector { .. }
18891919
| BackendRepr::SimdVector { .. } => false,
18901920
BackendRepr::Memory { sized } => sized && self.size.bytes() == 0,
18911921
}

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -536,7 +536,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
536536
let layout = self.layout_of(tp_ty).layout;
537537
let _use_integer_compare = match layout.backend_repr() {
538538
Scalar(_) | ScalarPair(_, _) => true,
539-
SimdVector { .. } => false,
539+
SimdVector { .. } | ScalableVector { .. } => false,
540540
Memory { .. } => {
541541
// For rusty ABIs, small aggregates are actually passed
542542
// as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),

‎compiler/rustc_codegen_gcc/src/type_of.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(
8585
);
8686
}
8787
BackendRepr::Memory { .. } => {}
88+
BackendRepr::ScalableVector { .. } => todo!(),
8889
}
8990

9091
let name = match *layout.ty.kind() {
@@ -178,7 +179,7 @@ pub trait LayoutGccExt<'tcx> {
178179
impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
179180
fn is_gcc_immediate(&self) -> bool {
180181
match self.backend_repr {
181-
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => true,
182+
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } | ScalableVector { .. } => true,
182183
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => false,
183184
}
184185
}
@@ -188,6 +189,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
188189
BackendRepr::ScalarPair(..) => true,
189190
BackendRepr::Scalar(_)
190191
| BackendRepr::SimdVector { .. }
192+
| BackendRepr::ScalableVector { .. }
191193
| BackendRepr::Memory { .. } => false,
192194
}
193195
}

‎compiler/rustc_codegen_llvm/src/builder.rs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
548548
}
549549
}
550550

551+
fn scalable_alloca(&mut self, elt: u64, align: Align, element_ty: Ty<'_>) -> Self::Value {
552+
let mut bx = Builder::with_cx(self.cx);
553+
bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
554+
let llvm_ty = match element_ty.kind() {
555+
ty::Bool => bx.type_i1(),
556+
ty::Int(int_ty) => self.cx.type_int_from_ty(*int_ty),
557+
ty::Uint(uint_ty) => self.cx.type_uint_from_ty(*uint_ty),
558+
ty::Float(float_ty) => self.cx.type_float_from_ty(*float_ty),
559+
_ => unreachable!("A simd type can only contain a bool, int, uint, or float"),
560+
};
561+
562+
unsafe {
563+
let ty = llvm::LLVMScalableVectorType(llvm_ty, elt.try_into().unwrap());
564+
let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, ty, UNNAMED);
565+
llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
566+
alloca
567+
}
568+
}
569+
551570
fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
552571
unsafe {
553572
let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);

‎compiler/rustc_codegen_llvm/src/intrinsic.rs

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
452452
let use_integer_compare = match layout.backend_repr() {
453453
Scalar(_) | ScalarPair(_, _) => true,
454454
SimdVector { .. } => false,
455+
ScalableVector { .. } => {
456+
tcx.dcx().emit_err(InvalidMonomorphization::NonScalableType {
457+
span,
458+
name: sym::raw_eq,
459+
ty: tp_ty,
460+
});
461+
return Ok(());
462+
}
455463
Memory { .. } => {
456464
// For rusty ABIs, small aggregates are actually passed
457465
// as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
@@ -1250,7 +1258,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
12501258
bx.bitcast(val, llret_ty)
12511259
}
12521260
OperandValue::ZeroSized => bx.const_undef(llret_ty),
1253-
OperandValue::Pair(_, _) => todo!(),
1261+
OperandValue::Pair(_, _) => {
1262+
return_error!(InvalidMonomorphization::NonScalableType { span, name, ty: ret_ty })
1263+
}
12541264
});
12551265
}
12561266

@@ -1449,11 +1459,27 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
14491459
m_len == v_len,
14501460
InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
14511461
);
1452-
let in_elem_bitwidth = require_int_or_uint_ty!(
1453-
m_elem_ty.kind(),
1454-
InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
1455-
);
1456-
let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len);
1462+
1463+
let m_i1s = if args[1].layout.ty.is_scalable_simd() {
1464+
match m_elem_ty.kind() {
1465+
ty::Bool => {}
1466+
_ => return_error!(InvalidMonomorphization::MaskWrongElementType {
1467+
span,
1468+
name,
1469+
ty: m_elem_ty
1470+
}),
1471+
};
1472+
let i1 = bx.type_i1();
1473+
let i1xn = bx.type_scalable_vector(i1, m_len as u64);
1474+
bx.trunc(args[0].immediate(), i1xn)
1475+
} else {
1476+
let in_elem_bitwidth = require_int_or_uint_ty!(
1477+
m_elem_ty.kind(),
1478+
InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
1479+
);
1480+
vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len)
1481+
};
1482+
14571483
return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
14581484
}
14591485

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1072,6 +1072,7 @@ unsafe extern "C" {
10721072
// Operations on array, pointer, and vector types (sequence types)
10731073
pub(crate) fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type;
10741074
pub(crate) fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
1075+
pub(crate) fn LLVMScalableVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
10751076

10761077
pub(crate) fn LLVMGetElementType(Ty: &Type) -> &Type;
10771078
pub(crate) fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint;

‎compiler/rustc_codegen_llvm/src/type_.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
6868
unsafe { llvm::LLVMVectorType(ty, len as c_uint) }
6969
}
7070

71+
pub(crate) fn type_scalable_vector(&self, ty: &'ll Type, count: u64) -> &'ll Type {
72+
unsafe { llvm::LLVMScalableVectorType(ty, count as c_uint) }
73+
}
74+
7175
pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> {
7276
unsafe {
7377
let n_args = llvm::LLVMCountParamTypes(ty) as usize;

‎compiler/rustc_codegen_llvm/src/type_of.rs

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,15 @@ fn uncached_llvm_type<'a, 'tcx>(
2323
let element = layout.scalar_llvm_type_at(cx, element);
2424
return cx.type_vector(element, count);
2525
}
26+
BackendRepr::ScalableVector { ref element, count } => {
27+
let element = if element.is_bool() {
28+
cx.type_i1()
29+
} else {
30+
layout.scalar_llvm_type_at(cx, *element)
31+
};
32+
33+
return cx.type_scalable_vector(element, count);
34+
}
2635
BackendRepr::Memory { .. } | BackendRepr::ScalarPair(..) => {}
2736
}
2837

@@ -171,7 +180,9 @@ pub(crate) trait LayoutLlvmExt<'tcx> {
171180
impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
172181
fn is_llvm_immediate(&self) -> bool {
173182
match self.backend_repr {
174-
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => true,
183+
BackendRepr::Scalar(_)
184+
| BackendRepr::SimdVector { .. }
185+
| BackendRepr::ScalableVector { .. } => true,
175186
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => false,
176187
}
177188
}
@@ -181,6 +192,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
181192
BackendRepr::ScalarPair(..) => true,
182193
BackendRepr::Scalar(_)
183194
| BackendRepr::SimdVector { .. }
195+
| BackendRepr::ScalableVector { .. }
184196
| BackendRepr::Memory { .. } => false,
185197
}
186198
}

0 commit comments

Comments
 (0)