Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 88538ad

Browse files
jonas-schievinkjonas-schievink
committed
Record intra-statement/terminator conflicts
Some MIR statements and terminators have an (undocumented...) invariant that some of their input and outputs must not overlap. This records conflicts between locals used in these positions.
1 parent ddd6930 commit 88538ad

File tree

1 file changed

+206
-22
lines changed

1 file changed

+206
-22
lines changed

‎compiler/rustc_mir/src/transform/dest_prop.rs

Lines changed: 206 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,8 @@ use rustc_index::{
109109
use rustc_middle::mir::tcx::PlaceTy;
110110
use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
111111
use rustc_middle::mir::{
112-
traversal, Body, Local, LocalKind, Location, Operand, Place, PlaceElem, Rvalue, Statement,
113-
StatementKind, Terminator, TerminatorKind,
112+
traversal, Body, InlineAsmOperand, Local, LocalKind, Location, Operand, Place, PlaceElem,
113+
Rvalue, Statement, StatementKind, Terminator, TerminatorKind,
114114
};
115115
use rustc_middle::ty::{self, Ty, TyCtxt};
116116

@@ -397,7 +397,9 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> {
397397
}
398398
}
399399

400-
struct Conflicts {
400+
struct Conflicts<'a> {
401+
relevant_locals: &'a BitSet<Local>,
402+
401403
/// The conflict matrix. It is always symmetric and the adjacency matrix of the corresponding
402404
/// conflict graph.
403405
matrix: BitMatrix<Local, Local>,
@@ -406,30 +408,21 @@ struct Conflicts {
406408
unify_cache: BitSet<Local>,
407409
}
408410

409-
impl Conflicts {
411+
impl Conflicts<'a> {
410412
fn build<'tcx>(
411413
tcx: TyCtxt<'tcx>,
412414
body: &'_ Body<'tcx>,
413415
source: MirSource<'tcx>,
414-
relevant_locals: &BitSet<Local>,
416+
relevant_locals: &'a BitSet<Local>,
415417
) -> Self {
416418
// We don't have to look out for locals that have their address taken, since
417419
// `find_candidates` already takes care of that.
418420

419-
let mut conflicts = BitMatrix::from_row_n(
421+
let conflicts = BitMatrix::from_row_n(
420422
&BitSet::new_empty(body.local_decls.len()),
421423
body.local_decls.len(),
422424
);
423425

424-
let mut record_conflicts = |new_conflicts: &mut BitSet<_>| {
425-
// Remove all locals that are not candidates.
426-
new_conflicts.intersect(relevant_locals);
427-
428-
for local in new_conflicts.iter() {
429-
conflicts.union_row_with(&new_conflicts, local);
430-
}
431-
};
432-
433426
let def_id = source.def_id();
434427
let mut init = MaybeInitializedLocals
435428
.into_engine(tcx, body, def_id)
@@ -494,6 +487,12 @@ impl Conflicts {
494487
},
495488
);
496489

490+
let mut this = Self {
491+
relevant_locals,
492+
matrix: conflicts,
493+
unify_cache: BitSet::new_empty(body.local_decls.len()),
494+
};
495+
497496
let mut live_and_init_locals = Vec::new();
498497

499498
// Visit only reachable basic blocks. The exact order is not important.
@@ -511,14 +510,22 @@ impl Conflicts {
511510
BitSet::new_empty(body.local_decls.len())
512511
});
513512

514-
// First, go forwards for `MaybeInitializedLocals`.
515-
for statement_index in 0..=data.statements.len() {
516-
let loc = Location { block, statement_index };
513+
// First, go forwards for `MaybeInitializedLocals` and apply intra-statement/terminator
514+
// conflicts.
515+
for (i, statement) in data.statements.iter().enumerate() {
516+
this.record_statement_conflicts(statement);
517+
518+
let loc = Location { block, statement_index: i };
517519
init.seek_before_primary_effect(loc);
518520

519-
live_and_init_locals[statement_index].clone_from(init.get());
521+
live_and_init_locals[i].clone_from(init.get());
520522
}
521523

524+
this.record_terminator_conflicts(data.terminator());
525+
let term_loc = Location { block, statement_index: data.statements.len() };
526+
init.seek_before_primary_effect(term_loc);
527+
live_and_init_locals[term_loc.statement_index].clone_from(init.get());
528+
522529
// Now, go backwards and union with the liveness results.
523530
for statement_index in (0..=data.statements.len()).rev() {
524531
let loc = Location { block, statement_index };
@@ -528,7 +535,7 @@ impl Conflicts {
528535

529536
trace!("record conflicts at {:?}", loc);
530537

531-
record_conflicts(&mut live_and_init_locals[statement_index]);
538+
this.record_conflicts(&mut live_and_init_locals[statement_index]);
532539
}
533540

534541
init.seek_to_block_end(block);
@@ -537,10 +544,187 @@ impl Conflicts {
537544
conflicts.intersect(live.get());
538545
trace!("record conflicts at end of {:?}", block);
539546

540-
record_conflicts(&mut conflicts);
547+
this.record_conflicts(&mut conflicts);
548+
}
549+
550+
this
551+
}
552+
553+
fn record_conflicts(&mut self, new_conflicts: &mut BitSet<Local>) {
554+
// Remove all locals that are not candidates.
555+
new_conflicts.intersect(self.relevant_locals);
556+
557+
for local in new_conflicts.iter() {
558+
self.matrix.union_row_with(&new_conflicts, local);
559+
}
560+
}
561+
562+
/// Records locals that must not overlap during the evaluation of `stmt`. These locals conflict
563+
/// and must not be merged.
564+
fn record_statement_conflicts(&mut self, stmt: &Statement<'_>) {
565+
match &stmt.kind {
566+
// While the left and right sides of an assignment must not overlap, we do not mark
567+
// conflicts here as that would make this optimization useless. When we optimize, we
568+
// eliminate the resulting self-assignments automatically.
569+
StatementKind::Assign(_) => {}
570+
571+
StatementKind::LlvmInlineAsm(asm) => {
572+
// Inputs and outputs must not overlap.
573+
for (_, input) in &*asm.inputs {
574+
if let Some(in_place) = input.place() {
575+
if !in_place.is_indirect() {
576+
for out_place in &*asm.outputs {
577+
if !out_place.is_indirect() && !in_place.is_indirect() {
578+
self.matrix.insert(in_place.local, out_place.local);
579+
self.matrix.insert(out_place.local, in_place.local);
580+
}
581+
}
582+
}
583+
}
584+
}
585+
}
586+
587+
StatementKind::SetDiscriminant { .. }
588+
| StatementKind::StorageLive(_)
589+
| StatementKind::StorageDead(_)
590+
| StatementKind::Retag(_, _)
591+
| StatementKind::FakeRead(_, _)
592+
| StatementKind::AscribeUserType(_, _)
593+
| StatementKind::Nop => {}
541594
}
595+
}
542596

543-
Self { matrix: conflicts, unify_cache: BitSet::new_empty(body.local_decls.len()) }
597+
fn record_terminator_conflicts(&mut self, term: &Terminator<'_>) {
598+
match &term.kind {
599+
TerminatorKind::DropAndReplace { location, value, target: _, unwind: _ } => {
600+
if let Some(place) = value.place() {
601+
if !place.is_indirect() && !location.is_indirect() {
602+
self.matrix.insert(place.local, location.local);
603+
self.matrix.insert(location.local, place.local);
604+
}
605+
}
606+
}
607+
TerminatorKind::Yield { value, resume: _, resume_arg, drop: _ } => {
608+
if let Some(place) = value.place() {
609+
if !place.is_indirect() && !resume_arg.is_indirect() {
610+
self.matrix.insert(place.local, resume_arg.local);
611+
self.matrix.insert(resume_arg.local, place.local);
612+
}
613+
}
614+
}
615+
TerminatorKind::Call {
616+
func,
617+
args,
618+
destination: Some((dest_place, _)),
619+
cleanup: _,
620+
from_hir_call: _,
621+
} => {
622+
// No arguments may overlap with the destination.
623+
for arg in args.iter().chain(Some(func)) {
624+
if let Some(place) = arg.place() {
625+
if !place.is_indirect() && !dest_place.is_indirect() {
626+
self.matrix.insert(dest_place.local, place.local);
627+
self.matrix.insert(place.local, dest_place.local);
628+
}
629+
}
630+
}
631+
}
632+
TerminatorKind::InlineAsm {
633+
template: _,
634+
operands,
635+
options: _,
636+
line_spans: _,
637+
destination: _,
638+
} => {
639+
// The intended semantics here aren't documented, we just assume that nothing that
640+
// could be written to by the assembly may overlap with any other operands.
641+
for op in operands {
642+
match op {
643+
InlineAsmOperand::Out { reg: _, late: _, place: Some(dest_place) }
644+
| InlineAsmOperand::InOut {
645+
reg: _,
646+
late: _,
647+
in_value: _,
648+
out_place: Some(dest_place),
649+
} => {
650+
// For output place `place`, add all places accessed by the inline asm.
651+
for op in operands {
652+
match op {
653+
InlineAsmOperand::In { reg: _, value } => {
654+
if let Some(p) = value.place() {
655+
if !p.is_indirect() && !dest_place.is_indirect() {
656+
self.matrix.insert(p.local, dest_place.local);
657+
self.matrix.insert(dest_place.local, p.local);
658+
}
659+
}
660+
}
661+
InlineAsmOperand::Out {
662+
reg: _,
663+
late: _,
664+
place: Some(place),
665+
} => {
666+
if !place.is_indirect() && !dest_place.is_indirect() {
667+
self.matrix.insert(place.local, dest_place.local);
668+
self.matrix.insert(dest_place.local, place.local);
669+
}
670+
}
671+
InlineAsmOperand::InOut {
672+
reg: _,
673+
late: _,
674+
in_value,
675+
out_place,
676+
} => {
677+
if let Some(place) = in_value.place() {
678+
if !place.is_indirect() && !dest_place.is_indirect() {
679+
self.matrix.insert(place.local, dest_place.local);
680+
self.matrix.insert(dest_place.local, place.local);
681+
}
682+
}
683+
684+
if let Some(place) = out_place {
685+
if !place.is_indirect() && !dest_place.is_indirect() {
686+
self.matrix.insert(place.local, dest_place.local);
687+
self.matrix.insert(dest_place.local, place.local);
688+
}
689+
}
690+
}
691+
InlineAsmOperand::Out { reg: _, late: _, place: None }
692+
| InlineAsmOperand::Const { value: _ }
693+
| InlineAsmOperand::SymFn { value: _ }
694+
| InlineAsmOperand::SymStatic { value: _ } => {}
695+
}
696+
}
697+
}
698+
InlineAsmOperand::Const { value } => {
699+
assert!(value.place().is_none());
700+
}
701+
InlineAsmOperand::InOut {
702+
reg: _,
703+
late: _,
704+
in_value: _,
705+
out_place: None,
706+
}
707+
| InlineAsmOperand::In { reg: _, value: _ }
708+
| InlineAsmOperand::Out { reg: _, late: _, place: None }
709+
| InlineAsmOperand::SymFn { value: _ }
710+
| InlineAsmOperand::SymStatic { value: _ } => {}
711+
}
712+
}
713+
}
714+
715+
TerminatorKind::Goto { .. }
716+
| TerminatorKind::Call { destination: None, .. }
717+
| TerminatorKind::SwitchInt { .. }
718+
| TerminatorKind::Resume
719+
| TerminatorKind::Abort
720+
| TerminatorKind::Return
721+
| TerminatorKind::Unreachable
722+
| TerminatorKind::Drop { .. }
723+
| TerminatorKind::Assert { .. }
724+
| TerminatorKind::GeneratorDrop
725+
| TerminatorKind::FalseEdges { .. }
726+
| TerminatorKind::FalseUnwind { .. } => {}
727+
}
544728
}
545729

546730
fn contains(&self, a: Local, b: Local) -> bool {

0 commit comments

Comments
 (0)