Skip to content

Commit ee92ffc

Browse files
saethlinsaethlin
committed
Add OperandValue::Uninit
1 parent fd30be0 commit ee92ffc

File tree

5 files changed

+91
-43
lines changed

5 files changed

+91
-43
lines changed

‎compiler/rustc_codegen_ssa/src/base.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,7 @@ pub(crate) fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
298298
let (base, info) = match bx.load_operand(src).val {
299299
OperandValue::Pair(base, info) => unsize_ptr(bx, base, src_ty, dst_ty, Some(info)),
300300
OperandValue::Immediate(base) => unsize_ptr(bx, base, src_ty, dst_ty, None),
301-
OperandValue::Ref(..) | OperandValue::ZeroSized => bug!(),
301+
OperandValue::Ref(..) | OperandValue::ZeroSized | OperandValue::Uninit => bug!(),
302302
};
303303
OperandValue::Pair(base, info).store(bx, dst);
304304
}

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi, PassMode};
1717
use tracing::{debug, info};
1818

1919
use super::operand::OperandRef;
20-
use super::operand::OperandValue::{Immediate, Pair, Ref, ZeroSized};
20+
use super::operand::OperandValue::{Immediate, Pair, Ref, Uninit, ZeroSized};
2121
use super::place::{PlaceRef, PlaceValue};
2222
use super::{CachedLlbb, FunctionCx, LocalRef};
2323
use crate::base::{self, is_call_from_compiler_builtins_to_upstream_monomorphization};
@@ -527,10 +527,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
527527

528528
PassMode::Direct(_) | PassMode::Pair(..) => {
529529
let op = self.codegen_consume(bx, mir::Place::return_place().as_ref());
530-
if let Ref(place_val) = op.val {
531-
bx.load_from_place(bx.backend_type(op.layout), place_val)
532-
} else {
533-
op.immediate_or_packed_pair(bx)
530+
match op.val {
531+
Uninit => bx.const_undef(bx.backend_type(op.layout)),
532+
Ref(place_val) => bx.load_from_place(bx.backend_type(op.layout), place_val),
533+
_ => op.immediate_or_packed_pair(bx),
534534
}
535535
}
536536

@@ -557,6 +557,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
557557
place_val.llval
558558
}
559559
ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"),
560+
Uninit => {
561+
bx.ret_void();
562+
return;
563+
}
560564
};
561565
load_cast(bx, cast_ty, llslot, self.fn_abi.ret.layout.align.abi)
562566
}
@@ -1587,6 +1591,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
15871591
}
15881592
_ => bug!("ZST {op:?} wasn't ignored, but was passed with abi {arg:?}"),
15891593
},
1594+
Uninit => {
1595+
let scratch = PlaceRef::alloca(bx, arg.layout);
1596+
(scratch.val.llval, scratch.val.align, true)
1597+
}
15901598
};
15911599

15921600
if by_ref && !arg.is_indirect() {

‎compiler/rustc_codegen_ssa/src/mir/debuginfo.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
341341
OperandValue::ZeroSized => {
342342
// These never have a value to talk about
343343
}
344+
OperandValue::Uninit => {
345+
// Better not have a useful name
346+
}
344347
},
345348
LocalRef::PendingOperand => {}
346349
}

‎compiler/rustc_codegen_ssa/src/mir/operand.rs

Lines changed: 65 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,14 @@ pub enum OperandValue<V> {
6767
/// `is_zst` on its `Layout` returns `true`. Note however that
6868
/// these values can still require alignment.
6969
ZeroSized,
70+
Uninit,
7071
}
7172

7273
impl<V: CodegenObject> OperandValue<V> {
74+
pub(crate) fn is_uninit(&self) -> bool {
75+
matches!(self, OperandValue::Uninit)
76+
}
77+
7378
/// Treat this value as a pointer and return the data pointer and
7479
/// optional metadata as backend values.
7580
///
@@ -100,6 +105,7 @@ impl<V: CodegenObject> OperandValue<V> {
100105
ty: TyAndLayout<'tcx>,
101106
) -> bool {
102107
match self {
108+
OperandValue::Uninit => true,
103109
OperandValue::ZeroSized => ty.is_zst(),
104110
OperandValue::Immediate(_) => cx.is_backend_immediate(ty),
105111
OperandValue::Pair(_, _) => cx.is_backend_scalar_pair(ty),
@@ -144,6 +150,10 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
144150
) -> Self {
145151
let layout = bx.layout_of(ty);
146152

153+
if val.all_bytes_uninit(bx.tcx()) {
154+
return OperandRef { val: OperandValue::Uninit, layout };
155+
}
156+
147157
let val = match val {
148158
ConstValue::Scalar(x) => {
149159
let BackendRepr::Scalar(scalar) = layout.backend_repr else {
@@ -442,6 +452,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
442452

443453
// Read the tag/niche-encoded discriminant from memory.
444454
let tag_op = match self.val {
455+
OperandValue::Uninit => bug!("shouldn't load from uninit"),
445456
OperandValue::ZeroSized => bug!(),
446457
OperandValue::Immediate(_) | OperandValue::Pair(_, _) => {
447458
self.extract_field(fx, bx, tag_field.as_usize())
@@ -591,6 +602,28 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
591602
}
592603

593604
impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, Result<V, abi::Scalar>> {
605+
fn update_uninit<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
606+
bx: &mut Bx,
607+
tgt: &mut Result<V, abi::Scalar>,
608+
) {
609+
let to_scalar = tgt.unwrap_err();
610+
let bty = bx.cx().type_from_scalar(to_scalar);
611+
*tgt = Ok(bx.const_undef(bty));
612+
}
613+
614+
fn update<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
615+
bx: &mut Bx,
616+
tgt: &mut Result<V, abi::Scalar>,
617+
src: V,
618+
from_scalar: rustc_abi::Scalar,
619+
) {
620+
let from_bty = bx.cx().type_from_scalar(from_scalar);
621+
let to_scalar = tgt.unwrap_err();
622+
let to_bty = bx.cx().type_from_scalar(to_scalar);
623+
let imm = transmute_immediate(bx, src, from_scalar, from_bty, to_scalar, to_bty);
624+
*tgt = Ok(imm);
625+
}
626+
594627
pub(crate) fn insert_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
595628
&mut self,
596629
bx: &mut Bx,
@@ -614,37 +647,48 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, Result<V, abi::Scalar>> {
614647
(field_layout.is_zst(), field_offset == Size::ZERO)
615648
};
616649

617-
let mut update = |tgt: &mut Result<V, abi::Scalar>, src, from_scalar| {
618-
let from_bty = bx.cx().type_from_scalar(from_scalar);
619-
let to_scalar = tgt.unwrap_err();
620-
let to_bty = bx.cx().type_from_scalar(to_scalar);
621-
let imm = transmute_immediate(bx, src, from_scalar, from_bty, to_scalar, to_bty);
622-
*tgt = Ok(imm);
623-
};
624-
625650
match (operand.val, operand.layout.backend_repr) {
626651
(OperandValue::ZeroSized, _) if expect_zst => {}
627652
(OperandValue::Immediate(v), BackendRepr::Scalar(from_scalar)) => match &mut self.val {
628653
OperandValue::Immediate(val @ Err(_)) if is_zero_offset => {
629-
update(val, v, from_scalar);
654+
Self::update(bx, val, v, from_scalar);
630655
}
631656
OperandValue::Pair(fst @ Err(_), _) if is_zero_offset => {
632-
update(fst, v, from_scalar);
657+
Self::update(bx, fst, v, from_scalar);
633658
}
634659
OperandValue::Pair(_, snd @ Err(_)) if !is_zero_offset => {
635-
update(snd, v, from_scalar);
660+
Self::update(bx, snd, v, from_scalar);
661+
}
662+
_ => bug!("Tried to insert {operand:?} into {v:?}.{f:?} of {self:?}"),
663+
},
664+
(OperandValue::Uninit, BackendRepr::Scalar(_)) => match &mut self.val {
665+
OperandValue::Immediate(val @ Err(_)) if is_zero_offset => {
666+
Self::update_uninit(bx, val);
667+
}
668+
OperandValue::Pair(fst @ Err(_), _) if is_zero_offset => {
669+
Self::update_uninit(bx, fst);
670+
}
671+
OperandValue::Pair(_, snd @ Err(_)) if !is_zero_offset => {
672+
Self::update_uninit(bx, snd);
636673
}
637674
_ => bug!("Tried to insert {operand:?} into {v:?}.{f:?} of {self:?}"),
638675
},
639676
(OperandValue::Pair(a, b), BackendRepr::ScalarPair(from_sa, from_sb)) => {
640677
match &mut self.val {
641678
OperandValue::Pair(fst @ Err(_), snd @ Err(_)) => {
642-
update(fst, a, from_sa);
643-
update(snd, b, from_sb);
679+
Self::update(bx, fst, a, from_sa);
680+
Self::update(bx, snd, b, from_sb);
644681
}
645682
_ => bug!("Tried to insert {operand:?} into {v:?}.{f:?} of {self:?}"),
646683
}
647684
}
685+
(OperandValue::Uninit, BackendRepr::ScalarPair(..)) => match &mut self.val {
686+
OperandValue::Pair(fst @ Err(_), snd @ Err(_)) => {
687+
Self::update_uninit(bx, fst);
688+
Self::update_uninit(bx, snd);
689+
}
690+
_ => bug!("Tried to insert {operand:?} into {v:?}.{f:?} of {self:?}"),
691+
},
648692
_ => bug!("Unsupported operand {operand:?} inserting into {v:?}.{f:?} of {self:?}"),
649693
}
650694
}
@@ -663,6 +707,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, Result<V, abi::Scalar>> {
663707
};
664708

665709
let val = match val {
710+
OperandValue::Uninit => OperandValue::Uninit,
666711
OperandValue::ZeroSized => OperandValue::ZeroSized,
667712
OperandValue::Immediate(v) => OperandValue::Immediate(unwrap(v)),
668713
OperandValue::Pair(a, b) => OperandValue::Pair(unwrap(a), unwrap(b)),
@@ -739,6 +784,13 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
739784
) {
740785
debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest);
741786
match self {
787+
OperandValue::Uninit => {
788+
// Ideally we'd hint to the backend that the destination is deinitialized by the
789+
// store. But in practice the destination is almost always uninit already because
790+
// OperandValue::Uninit is pretty much only produced by MaybeUninit::uninit.
791+
// Attempting to generate a hint by calling memset with undef mostly seems to
792+
// confuse LLVM.
793+
}
742794
OperandValue::ZeroSized => {
743795
// Avoid generating stores of zero-sized values, because the only way to have a
744796
// zero-sized value is through `undef`/`poison`, and the store itself is useless.

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

Lines changed: 9 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
2626
) {
2727
match *rvalue {
2828
mir::Rvalue::Use(ref operand) => {
29-
if let mir::Operand::Constant(const_op) = operand {
30-
let val = self.eval_mir_constant(const_op);
31-
if val.all_bytes_uninit(self.cx.tcx()) {
32-
return;
33-
}
34-
}
35-
3629
let cg_operand = self.codegen_operand(bx, operand);
3730
// FIXME: consider not copying constants through stack. (Fixable by codegen'ing
3831
// constants into `OperandValue::Ref`; why don’t we do that yet if we don’t?)
@@ -74,6 +67,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
7467
base::coerce_unsized_into(bx, scratch, dest);
7568
scratch.storage_dead(bx);
7669
}
70+
OperandValue::Uninit => {}
7771
OperandValue::Ref(val) => {
7872
if val.llextra.is_some() {
7973
bug!("unsized coercion on an unsized rvalue");
@@ -97,16 +91,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
9791
return;
9892
}
9993

100-
// When the element is a const with all bytes uninit, emit a single memset that
101-
// writes undef to the entire destination.
102-
if let mir::Operand::Constant(const_op) = elem {
103-
let val = self.eval_mir_constant(const_op);
104-
if val.all_bytes_uninit(self.cx.tcx()) {
105-
return;
106-
}
107-
}
108-
10994
let cg_elem = self.codegen_operand(bx, elem);
95+
// Normally the check for uninit is handled inside the operand helpers, but in this
96+
// one case we want to bail early so that we don't generate the loop form with an
97+
// empty body.
98+
if cg_elem.val.is_uninit() {
99+
return;
100+
}
110101

111102
let try_init_all_same = |bx: &mut Bx, v| {
112103
let start = dest.val.llval;
@@ -165,13 +156,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
165156
assert_eq!(operands.len(), 1);
166157
}
167158
for (i, operand) in operands.iter_enumerated() {
168-
if let mir::Operand::Constant(const_op) = operand {
169-
let val = self.eval_mir_constant(const_op);
170-
if val.all_bytes_uninit(self.cx.tcx()) {
171-
continue;
172-
}
173-
}
174-
175159
let op = self.codegen_operand(bx, operand);
176160
// Do not generate stores and GEPis for zero-sized fields.
177161
if !op.layout.is_zst() {
@@ -212,7 +196,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
212196
}
213197

214198
match src.val {
215-
OperandValue::Ref(..) | OperandValue::ZeroSized => {
199+
OperandValue::Ref(..) | OperandValue::ZeroSized | OperandValue::Uninit => {
216200
span_bug!(
217201
self.mir.span,
218202
"Operand path should have handled transmute \
@@ -257,6 +241,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
257241
let cast_kind = self.value_kind(cast);
258242

259243
match operand.val {
244+
OperandValue::Uninit => Some(OperandValue::Uninit),
260245
OperandValue::Ref(source_place_val) => {
261246
assert_eq!(source_place_val.llextra, None);
262247
assert_matches!(operand_kind, OperandValueKind::Ref);

0 commit comments

Comments
 (0)