Skip to content

Commit b741351

Browse files
oli-obkoli-obk
committed
Generalize the Assume intrinsic statement to a general Intrinsic statement
1 parent 3f07645 commit b741351

File tree

28 files changed

+166
-143
lines changed

28 files changed

+166
-143
lines changed

‎compiler/rustc_borrowck/src/dataflow.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -391,8 +391,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
391391
| mir::StatementKind::Retag { .. }
392392
| mir::StatementKind::AscribeUserType(..)
393393
| mir::StatementKind::Coverage(..)
394-
| mir::StatementKind::CopyNonOverlapping(..)
395-
| mir::StatementKind::Assume(..)
394+
| mir::StatementKind::Intrinsic(..)
396395
| mir::StatementKind::Nop => {}
397396
}
398397
}

‎compiler/rustc_borrowck/src/invalidation.rs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use rustc_data_structures::graph::dominators::Dominators;
22
use rustc_middle::mir::visit::Visitor;
3-
use rustc_middle::mir::{BasicBlock, Body, Location, Place, Rvalue};
3+
use rustc_middle::mir::{self, BasicBlock, Body, Location, NonDivergingIntrinsic, Place, Rvalue};
44
use rustc_middle::mir::{BorrowKind, Mutability, Operand};
55
use rustc_middle::mir::{InlineAsmOperand, Terminator, TerminatorKind};
66
use rustc_middle::mir::{Statement, StatementKind};
@@ -63,11 +63,14 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
6363
StatementKind::FakeRead(box (_, _)) => {
6464
// Only relevant for initialized/liveness/safety checks.
6565
}
66-
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
66+
StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
67+
self.consume_operand(location, op);
68+
}
69+
StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
6770
ref src,
6871
ref dst,
6972
ref count,
70-
}) => {
73+
})) => {
7174
self.consume_operand(location, src);
7275
self.consume_operand(location, dst);
7376
self.consume_operand(location, count);
@@ -76,8 +79,6 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
7679
StatementKind::AscribeUserType(..)
7780
// Doesn't have any language semantics
7881
| StatementKind::Coverage(..)
79-
// Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck
80-
| StatementKind::Assume(..)
8182
// Does not actually affect borrowck
8283
| StatementKind::StorageLive(..) => {}
8384
StatementKind::StorageDead(local) => {

‎compiler/rustc_borrowck/src/lib.rs

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ use rustc_index::bit_set::ChunkedBitSet;
2626
use rustc_index::vec::IndexVec;
2727
use rustc_infer::infer::{DefiningAnchor, InferCtxt, TyCtxtInferExt};
2828
use rustc_middle::mir::{
29-
traversal, Body, ClearCrossCrate, Local, Location, Mutability, Operand, Place, PlaceElem,
30-
PlaceRef, VarDebugInfoContents,
29+
traversal, Body, ClearCrossCrate, Local, Location, Mutability, NonDivergingIntrinsic, Operand,
30+
Place, PlaceElem, PlaceRef, VarDebugInfoContents,
3131
};
3232
use rustc_middle::mir::{AggregateKind, BasicBlock, BorrowCheckResult, BorrowKind};
3333
use rustc_middle::mir::{Field, ProjectionElem, Promoted, Rvalue, Statement, StatementKind};
@@ -591,10 +591,10 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
591591
flow_state,
592592
);
593593
}
594-
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
595-
..
596-
}) => {
597-
span_bug!(
594+
StatementKind::Intrinsic(box ref kind) => match kind {
595+
// Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck
596+
NonDivergingIntrinsic::Assume(..) => {},
597+
NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
598598
span,
599599
"Unexpected CopyNonOverlapping, should only appear after lower_intrinsics",
600600
)
@@ -603,8 +603,6 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
603603
StatementKind::AscribeUserType(..)
604604
// Doesn't have any language semantics
605605
| StatementKind::Coverage(..)
606-
// Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck
607-
| StatementKind::Assume(..)
608606
// Does not actually affect borrowck
609607
| StatementKind::StorageLive(..) => {}
610608
StatementKind::StorageDead(local) => {

‎compiler/rustc_borrowck/src/type_check/mod.rs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1302,14 +1302,14 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
13021302
);
13031303
}
13041304
}
1305-
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
1306-
..
1307-
}) => span_bug!(
1308-
stmt.source_info.span,
1309-
"Unexpected StatementKind::CopyNonOverlapping, should only appear after lowering_intrinsics",
1310-
),
1305+
StatementKind::Intrinsic(box ref kind) => match kind {
1306+
NonDivergingIntrinsic::Assume(..) => {}
1307+
NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
1308+
stmt.source_info.span,
1309+
"Unexpected NonDivergingIntrinsic::CopyNonOverlapping, should only appear after lowering_intrinsics",
1310+
),
1311+
},
13111312
StatementKind::FakeRead(..)
1312-
| StatementKind::Assume(..)
13131313
| StatementKind::StorageLive(..)
13141314
| StatementKind::StorageDead(..)
13151315
| StatementKind::Retag { .. }

‎compiler/rustc_codegen_cranelift/src/base.rs

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -791,25 +791,34 @@ fn codegen_stmt<'tcx>(
791791
| StatementKind::Nop
792792
| StatementKind::FakeRead(..)
793793
| StatementKind::Retag { .. }
794-
// We ignore `assume` intrinsics, they are only useful for optimizations
795-
| StatementKind::Assume(..)
796794
| StatementKind::AscribeUserType(..) => {}
797795

798796
StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
799-
StatementKind::CopyNonOverlapping(inner) => {
800-
let dst = codegen_operand(fx, &inner.dst);
801-
let pointee = dst
802-
.layout()
803-
.pointee_info_at(fx, rustc_target::abi::Size::ZERO)
804-
.expect("Expected pointer");
805-
let dst = dst.load_scalar(fx);
806-
let src = codegen_operand(fx, &inner.src).load_scalar(fx);
807-
let count = codegen_operand(fx, &inner.count).load_scalar(fx);
808-
let elem_size: u64 = pointee.size.bytes();
809-
let bytes =
810-
if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
811-
fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
812-
}
797+
StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic {
798+
// We ignore `assume` intrinsics, they are only useful for optimizations
799+
NonDivergingIntrinsic::Assume(_) => {}
800+
NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
801+
src,
802+
dst,
803+
count,
804+
}) => {
805+
let dst = codegen_operand(fx, dst);
806+
let pointee = dst
807+
.layout()
808+
.pointee_info_at(fx, rustc_target::abi::Size::ZERO)
809+
.expect("Expected pointer");
810+
let dst = dst.load_scalar(fx);
811+
let src = codegen_operand(fx, src).load_scalar(fx);
812+
let count = codegen_operand(fx, count).load_scalar(fx);
813+
let elem_size: u64 = pointee.size.bytes();
814+
let bytes = if elem_size != 1 {
815+
fx.bcx.ins().imul_imm(count, elem_size as i64)
816+
} else {
817+
count
818+
};
819+
fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
820+
}
821+
},
813822
}
814823
}
815824

‎compiler/rustc_codegen_cranelift/src/constant.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -536,11 +536,12 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
536536
{
537537
return None;
538538
}
539-
StatementKind::CopyNonOverlapping(_) => {
540-
return None;
541-
} // conservative handling
539+
StatementKind::Intrinsic(ref intrinsic) => match **intrinsic {
540+
NonDivergingIntrinsic::CopyNonOverlapping(..) => return None,
541+
NonDivergingIntrinsic::Assume(..) => {}
542+
},
543+
// conservative handling
542544
StatementKind::Assign(_)
543-
| StatementKind::Assume(_)
544545
| StatementKind::FakeRead(_)
545546
| StatementKind::SetDiscriminant { .. }
546547
| StatementKind::Deinit(_)

‎compiler/rustc_codegen_ssa/src/mir/statement.rs

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use rustc_middle::mir;
2+
use rustc_middle::mir::NonDivergingIntrinsic;
23

34
use super::FunctionCx;
45
use super::LocalRef;
@@ -73,11 +74,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
7374
self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope);
7475
bx
7576
}
76-
mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
77-
ref src,
78-
ref dst,
79-
ref count,
80-
}) => {
77+
mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => {
78+
let op_val = self.codegen_operand(&mut bx, op);
79+
bx.assume(op_val.immediate());
80+
bx
81+
}
82+
mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
83+
mir::CopyNonOverlapping { ref count, ref src, ref dst },
84+
)) => {
8185
let dst_val = self.codegen_operand(&mut bx, dst);
8286
let src_val = self.codegen_operand(&mut bx, src);
8387
let count = self.codegen_operand(&mut bx, count).immediate();
@@ -93,11 +97,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
9397
bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty());
9498
bx
9599
}
96-
mir::StatementKind::Assume(box ref op) => {
97-
let op_val = self.codegen_operand(&mut bx, op);
98-
bx.assume(op_val.immediate());
99-
bx
100-
}
101100
mir::StatementKind::FakeRead(..)
102101
| mir::StatementKind::Retag { .. }
103102
| mir::StatementKind::AscribeUserType(..)

‎compiler/rustc_const_eval/src/interpret/step.rs

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
//!
33
//! The main entry point is the `step` method.
44
5-
use rustc_middle::mir;
65
use rustc_middle::mir::interpret::{InterpResult, Scalar};
6+
use rustc_middle::mir::{self, NonDivergingIntrinsic};
77
use rustc_middle::ty::layout::LayoutOf;
88

99
use super::{InterpCx, Machine};
@@ -114,22 +114,23 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
114114
M::retag(self, *kind, &dest)?;
115115
}
116116

117-
// Call CopyNonOverlapping
118-
CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { src, dst, count }) => {
119-
let src = self.eval_operand(src, None)?;
120-
let dst = self.eval_operand(dst, None)?;
121-
let count = self.eval_operand(count, None)?;
122-
self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?;
123-
}
124-
125-
// Call Assume
126-
Assume(box op) => {
117+
Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
127118
let op = self.eval_operand(op, None)?;
128119
let cond = self.read_scalar(&op)?.to_bool()?;
129120
if !cond {
130121
throw_ub_format!("`assume` called with `false`");
131122
}
132123
}
124+
Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
125+
ref count,
126+
ref src,
127+
ref dst,
128+
})) => {
129+
let src = self.eval_operand(src, None)?;
130+
let dst = self.eval_operand(dst, None)?;
131+
let count = self.eval_operand(count, None)?;
132+
self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?;
133+
}
133134

134135
// Statements we do not track.
135136
AscribeUserType(..) => {}

‎compiler/rustc_const_eval/src/transform/check_consts/check.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -678,8 +678,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
678678
| StatementKind::Retag { .. }
679679
| StatementKind::AscribeUserType(..)
680680
| StatementKind::Coverage(..)
681-
| StatementKind::CopyNonOverlapping(..)
682-
| StatementKind::Assume(..)
681+
| StatementKind::Intrinsic(..)
683682
| StatementKind::Nop => {}
684683
}
685684
}

‎compiler/rustc_const_eval/src/transform/validate.rs

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@ use rustc_middle::mir::interpret::Scalar;
77
use rustc_middle::mir::visit::NonUseContext::VarDebugInfo;
88
use rustc_middle::mir::visit::{PlaceContext, Visitor};
99
use rustc_middle::mir::{
10-
traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, Local, Location,
11-
MirPass, MirPhase, Operand, Place, PlaceElem, PlaceRef, ProjectionElem, RuntimePhase, Rvalue,
12-
SourceScope, Statement, StatementKind, Terminator, TerminatorKind, UnOp, START_BLOCK,
10+
traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping,
11+
Local, Location, MirPass, MirPhase, NonDivergingIntrinsic, Operand, Place, PlaceElem, PlaceRef,
12+
ProjectionElem, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind, Terminator,
13+
TerminatorKind, UnOp, START_BLOCK,
1314
};
1415
use rustc_middle::ty::fold::BottomUpFolder;
1516
use rustc_middle::ty::subst::Subst;
@@ -636,7 +637,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
636637
);
637638
}
638639
}
639-
StatementKind::Assume(box ref op) => {
640+
StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
640641
let ty = op.ty(&self.body.local_decls, self.tcx);
641642
if !ty.is_bool() {
642643
self.fail(
@@ -645,11 +646,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
645646
);
646647
}
647648
}
648-
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
649-
ref src,
650-
ref dst,
651-
ref count,
652-
}) => {
649+
StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
650+
CopyNonOverlapping { src, dst, count },
651+
)) => {
653652
let src_ty = src.ty(&self.body.local_decls, self.tcx);
654653
let op_src_ty = if let Some(src_deref) = src_ty.builtin_deref(true) {
655654
src_deref.ty

0 commit comments

Comments
 (0)