Skip to content

Commit 0bdf7b3

Browse files
committed
fix likely and unlikely intrinsics
1 parent d4f6f9e commit 0bdf7b3

File tree

29 files changed

+298
-50
lines changed

29 files changed

+298
-50
lines changed

compiler/rustc_borrowck/src/lib.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -623,7 +623,10 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
623623
);
624624
}
625625
StatementKind::Intrinsic(box kind) => match kind {
626-
NonDivergingIntrinsic::Assume(op) => self.consume_operand(location, (op, span), flow_state),
626+
NonDivergingIntrinsic::Assume(op)
627+
| NonDivergingIntrinsic::Expect(op, ..)
628+
=> self.consume_operand(location, (op, span), flow_state),
629+
627630
NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
628631
span,
629632
"Unexpected CopyNonOverlapping, should only appear after lower_intrinsics",

compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
5252
StatementKind::FakeRead(box (_, _)) => {
5353
// Only relevant for initialized/liveness/safety checks.
5454
}
55-
StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
55+
StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op))
56+
| StatementKind::Intrinsic(box NonDivergingIntrinsic::Expect(op, ..)) => {
5657
self.consume_operand(location, op);
5758
}
5859
StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {

compiler/rustc_borrowck/src/type_check/mod.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1367,7 +1367,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
13671367
}
13681368
}
13691369
StatementKind::Intrinsic(box kind) => match kind {
1370-
NonDivergingIntrinsic::Assume(op) => self.check_operand(op, location),
1370+
NonDivergingIntrinsic::Assume(op) | NonDivergingIntrinsic::Expect(op, ..) => {
1371+
self.check_operand(op, location)
1372+
}
13711373
NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
13721374
stmt.source_info.span,
13731375
"Unexpected NonDivergingIntrinsic::CopyNonOverlapping, should only appear after lowering_intrinsics",

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -438,11 +438,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
438438
fx.bcx.ins().trap(TrapCode::User(0));
439439
return;
440440
}
441-
sym::likely | sym::unlikely => {
442-
intrinsic_args!(fx, args => (a); intrinsic);
443-
444-
ret.write_cvalue(fx, a);
445-
}
446441
sym::is_val_statically_known => {
447442
intrinsic_args!(fx, args => (_a); intrinsic);
448443

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
117117
let func = unsafe { std::mem::transmute(simple.expect("simple")) };
118118
self.call(self.type_void(), None, None, func, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None)
119119
},
120-
sym::likely => {
121-
self.expect(args[0].immediate(), true)
122-
}
123-
sym::unlikely => {
124-
self.expect(args[0].immediate(), false)
125-
}
126120
sym::is_val_statically_known => {
127121
let a = args[0].immediate();
128122
let builtin = self.context.get_builtin_function("__builtin_constant_p");

compiler/rustc_codegen_llvm/src/builder.rs

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,49 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
197197
}
198198
}
199199

200+
fn cond_br_with_expect(
201+
&mut self,
202+
cond: &'ll Value,
203+
then_llbb: &'ll BasicBlock,
204+
else_llbb: &'ll BasicBlock,
205+
expect: ExpectKind,
206+
) {
207+
// emit the branch instruction
208+
let n = unsafe { llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb) };
209+
210+
// emit expectation metadata
211+
match expect {
212+
ExpectKind::None => {}
213+
ExpectKind::True | ExpectKind::False => unsafe {
214+
let s = "branch_weights";
215+
let v = [
216+
llvm::LLVMMDStringInContext(
217+
self.cx.llcx,
218+
s.as_ptr() as *const c_char,
219+
s.len() as c_uint,
220+
),
221+
// 'then' branch weight
222+
self.cx.const_u32(if expect == ExpectKind::True { 2000 } else { 1 }),
223+
// 'else' branch weight
224+
self.cx.const_u32(if expect == ExpectKind::True { 1 } else { 2000 }),
225+
];
226+
llvm::LLVMSetMetadata(
227+
n,
228+
llvm::MD_prof as c_uint,
229+
llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint),
230+
);
231+
},
232+
ExpectKind::Unpredictable => unsafe {
233+
let v: [&Value; 0] = [];
234+
llvm::LLVMSetMetadata(
235+
n,
236+
llvm::MD_unpredictable as c_uint,
237+
llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint),
238+
);
239+
},
240+
}
241+
}
242+
200243
fn switch(
201244
&mut self,
202245
v: &'ll Value,

compiler/rustc_codegen_llvm/src/intrinsic.rs

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,6 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
116116
None,
117117
)
118118
}
119-
sym::likely => {
120-
self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)])
121-
}
122119
sym::is_val_statically_known => {
123120
let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx);
124121
match self.type_kind(intrinsic_type) {
@@ -131,8 +128,6 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
131128
_ => self.const_bool(false),
132129
}
133130
}
134-
sym::unlikely => self
135-
.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]),
136131
kw::Try => {
137132
try_intrinsic(
138133
self,

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,7 @@ pub enum MetadataType {
427427
MD_nontemporal = 9,
428428
MD_mem_parallel_loop_access = 10,
429429
MD_nonnull = 11,
430+
MD_unpredictable = 15,
430431
MD_align = 17,
431432
MD_type = 19,
432433
MD_vcall_visibility = 28,

compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
315315
&mut self,
316316
helper: TerminatorCodegenHelper<'tcx>,
317317
bx: &mut Bx,
318+
bb: mir::BasicBlock,
318319
discr: &mir::Operand<'tcx>,
319320
targets: &SwitchTargets,
320321
) {
@@ -328,10 +329,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
328329
let lltrue = helper.llbb_with_cleanup(self, target);
329330
let llfalse = helper.llbb_with_cleanup(self, targets.otherwise());
330331
if switch_ty == bx.tcx().types.bool {
332+
let expect = if let Some(x) = self.mir[bb].statements.last()
333+
&& let mir::StatementKind::Intrinsic(box mir::NonDivergingIntrinsic::Expect(
334+
op,
335+
kind,
336+
)) = &x.kind
337+
&& self.codegen_operand(bx, op).immediate() == discr.immediate()
338+
{
339+
match kind {
340+
mir::ExpectKind::True => ExpectKind::True,
341+
mir::ExpectKind::False => ExpectKind::False,
342+
mir::ExpectKind::Unpredictable => ExpectKind::Unpredictable,
343+
}
344+
} else {
345+
ExpectKind::None
346+
};
347+
331348
// Don't generate trivial icmps when switching on bool.
332349
match test_value {
333-
0 => bx.cond_br(discr.immediate(), llfalse, lltrue),
334-
1 => bx.cond_br(discr.immediate(), lltrue, llfalse),
350+
0 => bx.cond_br_with_expect(discr.immediate(), llfalse, lltrue, expect),
351+
1 => bx.cond_br_with_expect(discr.immediate(), lltrue, llfalse, expect.not()),
335352
_ => bug!(),
336353
}
337354
} else {
@@ -1218,7 +1235,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
12181235
}
12191236

12201237
mir::TerminatorKind::SwitchInt { ref discr, ref targets } => {
1221-
self.codegen_switchint_terminator(helper, bx, discr, targets);
1238+
self.codegen_switchint_terminator(helper, bx, bb, discr, targets);
12221239
MergingSucc::False
12231240
}
12241241

compiler/rustc_codegen_ssa/src/mir/statement.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
7070
let op_val = self.codegen_operand(bx, op);
7171
bx.assume(op_val.immediate());
7272
}
73+
mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Expect(..)) => {}
7374
mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
7475
mir::CopyNonOverlapping { ref count, ref src, ref dst },
7576
)) => {

0 commit comments

Comments
 (0)