Skip to content

Commit 78cab36

Browse files
Mark-SimulacrumMark-Simulacrum
authored
Rollup merge of rust-lang#55633 - nikic:memcpy-align, r=nagisa
Support memcpy/memmove with differing src/dst alignment If LLVM 7 is used, generate memcpy/memmove with differing src/dst alignment. I've added new FFI functions to construct these through the builder API, which is more convenient than dealing with differing intrinsic signatures depending on the LLVM version. Fixes rust-lang#49740.
2 parents 6153ce1 + 463ad90 commit 78cab36

File tree

11 files changed

+98
-51
lines changed

11 files changed

+98
-51
lines changed

‎src/librustc_codegen_llvm/abi.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,9 +225,10 @@ impl ArgTypeExt<'ll, 'tcx> for ArgType<'tcx, Ty<'tcx>> {
225225
// ...and then memcpy it to the intended destination.
226226
base::call_memcpy(bx,
227227
bx.pointercast(dst.llval, Type::i8p(cx)),
228+
self.layout.align,
228229
bx.pointercast(llscratch, Type::i8p(cx)),
230+
scratch_align,
229231
C_usize(cx, self.layout.size.bytes()),
230-
self.layout.align.min(scratch_align),
231232
MemFlags::empty());
232233

233234
bx.lifetime_end(llscratch, scratch_size);

‎src/librustc_codegen_llvm/base.rs

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ use mir::place::PlaceRef;
5353
use attributes;
5454
use builder::{Builder, MemFlags};
5555
use callee;
56-
use common::{C_bool, C_bytes_in_context, C_i32, C_usize};
56+
use common::{C_bool, C_bytes_in_context, C_usize};
5757
use rustc_mir::monomorphize::item::DefPathBasedNames;
5858
use common::{C_struct_in_context, C_array, val_ty};
5959
use consts;
@@ -77,7 +77,6 @@ use rustc_data_structures::sync::Lrc;
7777
use std::any::Any;
7878
use std::cmp;
7979
use std::ffi::CString;
80-
use std::i32;
8180
use std::ops::{Deref, DerefMut};
8281
use std::sync::mpsc;
8382
use std::time::{Instant, Duration};
@@ -319,8 +318,8 @@ pub fn coerce_unsized_into(
319318
}
320319

321320
if src_f.layout.ty == dst_f.layout.ty {
322-
memcpy_ty(bx, dst_f.llval, src_f.llval, src_f.layout,
323-
src_f.align.min(dst_f.align), MemFlags::empty());
321+
memcpy_ty(bx, dst_f.llval, dst_f.align, src_f.llval, src_f.align,
322+
src_f.layout, MemFlags::empty());
324323
} else {
325324
coerce_unsized_into(bx, src_f, dst_f);
326325
}
@@ -420,44 +419,42 @@ pub fn to_immediate_scalar(
420419
pub fn call_memcpy(
421420
bx: &Builder<'_, 'll, '_>,
422421
dst: &'ll Value,
422+
dst_align: Align,
423423
src: &'ll Value,
424+
src_align: Align,
424425
n_bytes: &'ll Value,
425-
align: Align,
426426
flags: MemFlags,
427427
) {
428428
if flags.contains(MemFlags::NONTEMPORAL) {
429429
// HACK(nox): This is inefficient but there is no nontemporal memcpy.
430-
let val = bx.load(src, align);
430+
let val = bx.load(src, src_align);
431431
let ptr = bx.pointercast(dst, val_ty(val).ptr_to());
432-
bx.store_with_flags(val, ptr, align, flags);
432+
bx.store_with_flags(val, ptr, dst_align, flags);
433433
return;
434434
}
435435
let cx = bx.cx;
436-
let ptr_width = &cx.sess().target.target.target_pointer_width;
437-
let key = format!("llvm.memcpy.p0i8.p0i8.i{}", ptr_width);
438-
let memcpy = cx.get_intrinsic(&key);
439436
let src_ptr = bx.pointercast(src, Type::i8p(cx));
440437
let dst_ptr = bx.pointercast(dst, Type::i8p(cx));
441438
let size = bx.intcast(n_bytes, cx.isize_ty, false);
442-
let align = C_i32(cx, align.abi() as i32);
443-
let volatile = C_bool(cx, flags.contains(MemFlags::VOLATILE));
444-
bx.call(memcpy, &[dst_ptr, src_ptr, size, align, volatile], None);
439+
let volatile = flags.contains(MemFlags::VOLATILE);
440+
bx.memcpy(dst_ptr, dst_align.abi(), src_ptr, src_align.abi(), size, volatile);
445441
}
446442

447443
pub fn memcpy_ty(
448444
bx: &Builder<'_, 'll, 'tcx>,
449445
dst: &'ll Value,
446+
dst_align: Align,
450447
src: &'ll Value,
448+
src_align: Align,
451449
layout: TyLayout<'tcx>,
452-
align: Align,
453450
flags: MemFlags,
454451
) {
455452
let size = layout.size.bytes();
456453
if size == 0 {
457454
return;
458455
}
459456

460-
call_memcpy(bx, dst, src, C_usize(bx.cx, size), align, flags);
457+
call_memcpy(bx, dst, dst_align, src, src_align, C_usize(bx.cx, size), flags);
461458
}
462459

463460
pub fn call_memset(

‎src/librustc_codegen_llvm/builder.rs

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -781,6 +781,24 @@ impl Builder<'a, 'll, 'tcx> {
781781
}
782782
}
783783

784+
pub fn memcpy(&self, dst: &'ll Value, dst_align: u64,
785+
src: &'ll Value, src_align: u64,
786+
size: &'ll Value, is_volatile: bool) -> &'ll Value {
787+
unsafe {
788+
llvm::LLVMRustBuildMemCpy(self.llbuilder, dst, dst_align as c_uint,
789+
src, src_align as c_uint, size, is_volatile)
790+
}
791+
}
792+
793+
pub fn memmove(&self, dst: &'ll Value, dst_align: u64,
794+
src: &'ll Value, src_align: u64,
795+
size: &'ll Value, is_volatile: bool) -> &'ll Value {
796+
unsafe {
797+
llvm::LLVMRustBuildMemMove(self.llbuilder, dst, dst_align as c_uint,
798+
src, src_align as c_uint, size, is_volatile)
799+
}
800+
}
801+
784802
pub fn minnum(&self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
785803
self.count_insn("minnum");
786804
unsafe {

‎src/librustc_codegen_llvm/context.rs

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -530,12 +530,6 @@ fn declare_intrinsic(cx: &CodegenCx<'ll, '_>, key: &str) -> Option<&'ll Value> {
530530
let t_v4f64 = Type::vector(t_f64, 4);
531531
let t_v8f64 = Type::vector(t_f64, 8);
532532

533-
ifn!("llvm.memcpy.p0i8.p0i8.i16", fn(i8p, i8p, t_i16, t_i32, i1) -> void);
534-
ifn!("llvm.memcpy.p0i8.p0i8.i32", fn(i8p, i8p, t_i32, t_i32, i1) -> void);
535-
ifn!("llvm.memcpy.p0i8.p0i8.i64", fn(i8p, i8p, t_i64, t_i32, i1) -> void);
536-
ifn!("llvm.memmove.p0i8.p0i8.i16", fn(i8p, i8p, t_i16, t_i32, i1) -> void);
537-
ifn!("llvm.memmove.p0i8.p0i8.i32", fn(i8p, i8p, t_i32, t_i32, i1) -> void);
538-
ifn!("llvm.memmove.p0i8.p0i8.i64", fn(i8p, i8p, t_i64, t_i32, i1) -> void);
539533
ifn!("llvm.memset.p0i8.i16", fn(i8p, t_i8, t_i16, t_i32, i1) -> void);
540534
ifn!("llvm.memset.p0i8.i32", fn(i8p, t_i8, t_i32, t_i32, i1) -> void);
541535
ifn!("llvm.memset.p0i8.i64", fn(i8p, t_i8, t_i64, t_i32, i1) -> void);

‎src/librustc_codegen_llvm/intrinsic.rs

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use glue;
2323
use type_::Type;
2424
use type_of::LayoutLlvmExt;
2525
use rustc::ty::{self, Ty};
26-
use rustc::ty::layout::{HasDataLayout, LayoutOf};
26+
use rustc::ty::layout::LayoutOf;
2727
use rustc::hir;
2828
use syntax::ast;
2929
use syntax::symbol::Symbol;
@@ -690,28 +690,14 @@ fn copy_intrinsic(
690690
let cx = bx.cx;
691691
let (size, align) = cx.size_and_align_of(ty);
692692
let size = C_usize(cx, size.bytes());
693-
let align = C_i32(cx, align.abi() as i32);
694-
695-
let operation = if allow_overlap {
696-
"memmove"
697-
} else {
698-
"memcpy"
699-
};
700-
701-
let name = format!("llvm.{}.p0i8.p0i8.i{}", operation,
702-
cx.data_layout().pointer_size.bits());
703-
693+
let align = align.abi();
704694
let dst_ptr = bx.pointercast(dst, Type::i8p(cx));
705695
let src_ptr = bx.pointercast(src, Type::i8p(cx));
706-
let llfn = cx.get_intrinsic(&name);
707-
708-
bx.call(llfn,
709-
&[dst_ptr,
710-
src_ptr,
711-
bx.mul(size, count),
712-
align,
713-
C_bool(cx, volatile)],
714-
None)
696+
if allow_overlap {
697+
bx.memmove(dst_ptr, align, src_ptr, align, bx.mul(size, count), volatile)
698+
} else {
699+
bx.memcpy(dst_ptr, align, src_ptr, align, bx.mul(size, count), volatile)
700+
}
715701
}
716702

717703
fn memset_intrinsic(

‎src/librustc_codegen_llvm/llvm/ffi.rs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -998,6 +998,22 @@ extern "C" {
998998
Bundle: Option<&OperandBundleDef<'a>>,
999999
Name: *const c_char)
10001000
-> &'a Value;
1001+
pub fn LLVMRustBuildMemCpy(B: &Builder<'a>,
1002+
Dst: &'a Value,
1003+
DstAlign: c_uint,
1004+
Src: &'a Value,
1005+
SrcAlign: c_uint,
1006+
Size: &'a Value,
1007+
IsVolatile: bool)
1008+
-> &'a Value;
1009+
pub fn LLVMRustBuildMemMove(B: &Builder<'a>,
1010+
Dst: &'a Value,
1011+
DstAlign: c_uint,
1012+
Src: &'a Value,
1013+
SrcAlign: c_uint,
1014+
Size: &'a Value,
1015+
IsVolatile: bool)
1016+
-> &'a Value;
10011017
pub fn LLVMBuildSelect(B: &Builder<'a>,
10021018
If: &'a Value,
10031019
Then: &'a Value,

‎src/librustc_codegen_llvm/mir/block.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -784,7 +784,8 @@ impl FunctionCx<'a, 'll, 'tcx> {
784784
// have scary latent bugs around.
785785

786786
let scratch = PlaceRef::alloca(bx, arg.layout, "arg");
787-
base::memcpy_ty(bx, scratch.llval, llval, op.layout, align, MemFlags::empty());
787+
base::memcpy_ty(bx, scratch.llval, scratch.align, llval, align,
788+
op.layout, MemFlags::empty());
788789
(scratch.llval, scratch.align, true)
789790
} else {
790791
(llval, align, true)

‎src/librustc_codegen_llvm/mir/operand.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -282,8 +282,8 @@ impl OperandValue<'ll> {
282282
}
283283
match self {
284284
OperandValue::Ref(r, None, source_align) => {
285-
base::memcpy_ty(bx, dest.llval, r, dest.layout,
286-
source_align.min(dest.align), flags)
285+
base::memcpy_ty(bx, dest.llval, dest.align, r, source_align,
286+
dest.layout, flags)
287287
}
288288
OperandValue::Ref(_, Some(_), _) => {
289289
bug!("cannot directly store unsized values");
@@ -324,7 +324,7 @@ impl OperandValue<'ll> {
324324
// Allocate an appropriate region on the stack, and copy the value into it
325325
let (llsize, _) = glue::size_and_align_of_dst(&bx, unsized_ty, Some(llextra));
326326
let lldst = bx.array_alloca(Type::i8(bx.cx), llsize, "unsized_tmp", max_align);
327-
base::call_memcpy(&bx, lldst, llptr, llsize, min_align, flags);
327+
base::call_memcpy(&bx, lldst, max_align, llptr, min_align, llsize, flags);
328328

329329
// Store the allocated region and the extra to the indirect place.
330330
let indirect_operand = OperandValue::Pair(lldst, llextra);

‎src/rustllvm/RustWrapper.cpp

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1237,6 +1237,40 @@ extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMValueRef Fn,
12371237
unwrap(Fn), makeArrayRef(unwrap(Args), NumArgs), Bundles, Name));
12381238
}
12391239

1240+
extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B,
1241+
LLVMValueRef Dst, unsigned DstAlign,
1242+
LLVMValueRef Src, unsigned SrcAlign,
1243+
LLVMValueRef Size, bool IsVolatile) {
1244+
#if LLVM_VERSION_GE(7, 0)
1245+
return wrap(unwrap(B)->CreateMemCpy(
1246+
unwrap(Dst), DstAlign,
1247+
unwrap(Src), SrcAlign,
1248+
unwrap(Size), IsVolatile));
1249+
#else
1250+
unsigned Align = std::min(DstAlign, SrcAlign);
1251+
return wrap(unwrap(B)->CreateMemCpy(
1252+
unwrap(Dst), unwrap(Src),
1253+
unwrap(Size), Align, IsVolatile));
1254+
#endif
1255+
}
1256+
1257+
extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B,
1258+
LLVMValueRef Dst, unsigned DstAlign,
1259+
LLVMValueRef Src, unsigned SrcAlign,
1260+
LLVMValueRef Size, bool IsVolatile) {
1261+
#if LLVM_VERSION_GE(7, 0)
1262+
return wrap(unwrap(B)->CreateMemMove(
1263+
unwrap(Dst), DstAlign,
1264+
unwrap(Src), SrcAlign,
1265+
unwrap(Size), IsVolatile));
1266+
#else
1267+
unsigned Align = std::min(DstAlign, SrcAlign);
1268+
return wrap(unwrap(B)->CreateMemMove(
1269+
unwrap(Dst), unwrap(Src),
1270+
unwrap(Size), Align, IsVolatile));
1271+
#endif
1272+
}
1273+
12401274
extern "C" LLVMValueRef
12411275
LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn, LLVMValueRef *Args,
12421276
unsigned NumArgs, LLVMBasicBlockRef Then,

‎src/test/codegen/packed.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ pub struct BigPacked2 {
6565
pub fn call_pkd1(f: fn() -> Array) -> BigPacked1 {
6666
// CHECK: [[ALLOCA:%[_a-z0-9]+]] = alloca %Array
6767
// CHECK: call void %{{.*}}(%Array* noalias nocapture sret dereferenceable(32) [[ALLOCA]])
68-
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 %{{.*}}, i8* align 1 %{{.*}}, i{{[0-9]+}} 32, i1 false)
68+
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 %{{.*}}, i8* align 4 %{{.*}}, i{{[0-9]+}} 32, i1 false)
6969
// check that calls whose destination is a field of a packed struct
7070
// go through an alloca rather than calling the function with an
7171
// unaligned destination.
@@ -77,7 +77,7 @@ pub fn call_pkd1(f: fn() -> Array) -> BigPacked1 {
7777
pub fn call_pkd2(f: fn() -> Array) -> BigPacked2 {
7878
// CHECK: [[ALLOCA:%[_a-z0-9]+]] = alloca %Array
7979
// CHECK: call void %{{.*}}(%Array* noalias nocapture sret dereferenceable(32) [[ALLOCA]])
80-
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 2 %{{.*}}, i8* align 2 %{{.*}}, i{{[0-9]+}} 32, i1 false)
80+
// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 2 %{{.*}}, i8* align 4 %{{.*}}, i{{[0-9]+}} 32, i1 false)
8181
// check that calls whose destination is a field of a packed struct
8282
// go through an alloca rather than calling the function with an
8383
// unaligned destination.

0 commit comments

Comments
 (0)