Skip to content

Commit 8a9442c

Browse files
committed
Only save liveness for relevant locals.
1 parent 9298dfa commit 8a9442c

File tree

2 files changed

+59
-38
lines changed

2 files changed

+59
-38
lines changed

compiler/rustc_mir_dataflow/src/points.rs

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use rustc_index::bit_set::DenseBitSet;
22
use rustc_index::interval::SparseIntervalMatrix;
3-
use rustc_index::{Idx, IndexVec};
3+
use rustc_index::{Idx, IndexSlice, IndexVec};
44
use rustc_middle::mir::{self, BasicBlock, Body, Location};
55

66
use crate::framework::{Analysis, Direction, Results, ResultsVisitor, visit_results};
@@ -95,25 +95,30 @@ rustc_index::newtype_index! {
9595
}
9696

9797
/// Add points depending on the result of the given dataflow analysis.
98-
pub fn save_as_intervals<'tcx, N, A>(
98+
pub fn save_as_intervals<'tcx, N, M, A>(
9999
elements: &DenseLocationMap,
100100
body: &mir::Body<'tcx>,
101+
relevant: &IndexSlice<N, M>,
101102
mut analysis: A,
102103
results: Results<A::Domain>,
103104
) -> SparseIntervalMatrix<N, PointIndex>
104105
where
105106
N: Idx,
106-
A: Analysis<'tcx, Domain = DenseBitSet<N>>,
107+
M: Idx,
108+
A: Analysis<'tcx, Domain = DenseBitSet<M>>,
107109
{
108110
let mut values = SparseIntervalMatrix::new(elements.num_points());
109111
let reachable_blocks = mir::traversal::reachable_as_bitset(body);
110112
if A::Direction::IS_BACKWARD {
111113
// Iterate blocks in decreasing order, to visit locations in decreasing order. This
112114
// allows to use the more efficient `prepend` method to interval sets.
113-
let callback = |state: &DenseBitSet<N>, location| {
115+
let callback = |state: &DenseBitSet<M>, location| {
114116
let point = elements.point_from_location(location);
115-
// Use internal iterator manually as it is much more efficient.
116-
state.iter().for_each(|node| values.prepend(node, point));
117+
for (relevant, &original) in relevant.iter_enumerated() {
118+
if state.contains(original) {
119+
values.prepend(relevant, point);
120+
}
121+
}
117122
};
118123
let mut visitor = Visitor { callback };
119124
visit_results(
@@ -127,10 +132,13 @@ where
127132
} else {
128133
// Iterate blocks in increasing order, to visit locations in increasing order. This
129134
// allows to use the more efficient `append` method to interval sets.
130-
let callback = |state: &DenseBitSet<N>, location| {
135+
let callback = |state: &DenseBitSet<M>, location| {
131136
let point = elements.point_from_location(location);
132-
// Use internal iterator manually as it is much more efficient.
133-
state.iter().for_each(|node| values.append(node, point));
137+
for (relevant, &original) in relevant.iter_enumerated() {
138+
if state.contains(original) {
139+
values.append(relevant, point);
140+
}
141+
}
134142
};
135143
let mut visitor = Visitor { callback };
136144
visit_results(body, reachable_blocks.iter(), &mut analysis, &results, &mut visitor);

compiler/rustc_mir_transform/src/dest_prop.rs

Lines changed: 42 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -162,10 +162,19 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation {
162162
trace!(?def_id);
163163

164164
let borrowed = rustc_mir_dataflow::impls::borrowed_locals(body);
165+
let points = DenseLocationMap::new(body);
166+
167+
candidates.reset_and_find(body, &borrowed);
168+
trace!(?candidates);
169+
if candidates.c.is_empty() {
170+
return;
171+
}
172+
173+
let relevant = RelevantLocals::compute(&candidates, body.local_decls.len());
165174

166175
let live = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, Some("MaybeLiveLocals-DestProp"));
167-
let points = DenseLocationMap::new(body);
168-
let mut live = save_as_intervals(&points, body, live.analysis, live.results);
176+
let mut live =
177+
save_as_intervals(&points, body, &relevant.original, live.analysis, live.results);
169178

170179
// In order to avoid having to collect data for every single pair of locals in the body, we
171180
// do not allow doing more than one merge for places that are derived from the same local at
@@ -181,22 +190,14 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation {
181190
// https://github.com/rust-lang/regex/tree/b5372864e2df6a2f5e543a556a62197f50ca3650
182191
let mut round_count = 0;
183192
loop {
184-
// PERF: Can we do something smarter than recalculating the candidates and liveness
185-
// results?
186-
candidates.reset_and_find(body, &borrowed);
187-
trace!(?candidates);
188-
if candidates.c.is_empty() {
189-
break;
190-
}
191-
192-
dest_prop_mir_dump(tcx, body, &points, &live, round_count);
193+
dest_prop_mir_dump(tcx, body, &points, &live, &relevant, round_count);
193194

194195
let mut filter = FilterInformation::filter_liveness(
195196
&points,
196197
&live,
197198
&mut write_info,
198199
body,
199-
&candidates,
200+
&relevant,
200201
);
201202

202203
// This is the set of merges we will apply this round. It is a subset of the candidates.
@@ -216,6 +217,10 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation {
216217
merged_locals.insert(src);
217218
merged_locals.insert(dest);
218219

220+
// `find_non_conflicting` ensures this is true.
221+
let src = relevant.shrink[src].expect("merged locals are relevant");
222+
let dest = relevant.shrink[dest].expect("merged locals are relevant");
223+
219224
// Update liveness information based on the merge we just performed.
220225
// Every location where `src` was live, `dest` will be live.
221226
live.union_rows(src, dest);
@@ -232,6 +237,12 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation {
232237
round_count += 1;
233238

234239
apply_merges(body, tcx, merges, merged_locals);
240+
241+
candidates.reset_and_find(body, &borrowed);
242+
trace!(?candidates);
243+
if candidates.c.is_empty() {
244+
break;
245+
}
235246
}
236247

237248
trace!(round_count);
@@ -373,7 +384,7 @@ impl RelevantLocals {
373384
struct FilterInformation<'a, 'tcx> {
374385
body: &'a Body<'tcx>,
375386
points: &'a DenseLocationMap,
376-
relevant: RelevantLocals,
387+
relevant: &'a RelevantLocals,
377388
conflicts: BitMatrix<RelevantLocal, RelevantLocal>,
378389
write_info: &'a mut WriteInfo,
379390
at: Location,
@@ -418,13 +429,11 @@ impl<'a, 'tcx> FilterInformation<'a, 'tcx> {
418429
/// locals as also being read from.
419430
fn filter_liveness(
420431
points: &'a DenseLocationMap,
421-
live: &SparseIntervalMatrix<Local, PointIndex>,
432+
live: &SparseIntervalMatrix<RelevantLocal, PointIndex>,
422433
write_info: &'a mut WriteInfo,
423434
body: &'a Body<'tcx>,
424-
candidates: &Candidates,
435+
relevant: &'a RelevantLocals,
425436
) -> Self {
426-
let num_locals = body.local_decls.len();
427-
let relevant = RelevantLocals::compute(candidates, num_locals);
428437
let num_relevant = relevant.original.len();
429438
let mut this = FilterInformation {
430439
body,
@@ -441,7 +450,7 @@ impl<'a, 'tcx> FilterInformation<'a, 'tcx> {
441450
this
442451
}
443452

444-
fn internal_filter_liveness(&mut self, live: &SparseIntervalMatrix<Local, PointIndex>) {
453+
fn internal_filter_liveness(&mut self, live: &SparseIntervalMatrix<RelevantLocal, PointIndex>) {
445454
for (block, data) in traversal::preorder(self.body) {
446455
self.at = Location { block, statement_index: data.statements.len() };
447456
self.write_info.for_terminator(&data.terminator().kind);
@@ -457,7 +466,7 @@ impl<'a, 'tcx> FilterInformation<'a, 'tcx> {
457466
trace!(?self.conflicts, "initial conflicts");
458467
}
459468

460-
fn record_conflicts(&mut self, live: &SparseIntervalMatrix<Local, PointIndex>) {
469+
fn record_conflicts(&mut self, live: &SparseIntervalMatrix<RelevantLocal, PointIndex>) {
461470
let writes = self.write_info.writes.iter().filter_map(|&p| self.relevant.shrink[p]);
462471

463472
let skip_pair = self.write_info.skip_pair.and_then(|(p, q)| {
@@ -475,11 +484,11 @@ impl<'a, 'tcx> FilterInformation<'a, 'tcx> {
475484
}
476485
}
477486

478-
for (q, &original_q) in self.relevant.original.iter_enumerated() {
487+
for q in self.relevant.original.indices() {
479488
if p != q
480489
&& skip_pair != Some((p, q))
481490
&& skip_pair != Some((q, p))
482-
&& live.contains(original_q, at)
491+
&& live.contains(q, at)
483492
{
484493
self.conflicts.insert(p, q);
485494
}
@@ -488,21 +497,21 @@ impl<'a, 'tcx> FilterInformation<'a, 'tcx> {
488497
}
489498

490499
#[tracing::instrument(level = "trace", skip(self))]
491-
fn record_merge(&mut self, src: Local, dest: Local) {
500+
fn record_merge(&mut self, src: RelevantLocal, dest: RelevantLocal) {
492501
trace!(?self.conflicts, "pre");
493-
let src = self.relevant.shrink[src].expect("merged locals are relevant");
494-
let dest = self.relevant.shrink[dest].expect("merged locals are relevant");
495502
self.conflicts.union_rows(src, dest);
496503
self.conflicts.union_cols(src, dest);
497504
trace!(?self.conflicts, "post");
498505
}
499506

500507
#[tracing::instrument(level = "trace", skip(self), ret)]
501508
fn find_non_conflicting(&self, src: Local, candidates: &Vec<Local>) -> Option<Local> {
502-
let src = self.relevant.shrink[src].expect("merged locals are relevant");
509+
// Refuse to merge if the local is not marked relevant.
510+
let src = self.relevant.shrink[src]?;
503511
candidates.iter().copied().find(|&dest| {
504-
let dest = self.relevant.shrink[dest].expect("merged locals are relevant");
505-
!self.conflicts.contains(src, dest) && !self.conflicts.contains(dest, src)
512+
self.relevant.shrink[dest].is_some_and(|dest| {
513+
!self.conflicts.contains(src, dest) && !self.conflicts.contains(dest, src)
514+
})
506515
})
507516
}
508517
}
@@ -773,12 +782,16 @@ fn dest_prop_mir_dump<'tcx>(
773782
tcx: TyCtxt<'tcx>,
774783
body: &Body<'tcx>,
775784
points: &DenseLocationMap,
776-
live: &SparseIntervalMatrix<Local, PointIndex>,
785+
live: &SparseIntervalMatrix<RelevantLocal, PointIndex>,
786+
relevant: &RelevantLocals,
777787
round: usize,
778788
) {
779789
let locals_live_at = |location| {
780790
let location = points.point_from_location(location);
781-
live.rows().filter(|&r| live.contains(r, location)).collect::<Vec<_>>()
791+
live.rows()
792+
.filter(|&r| live.contains(r, location))
793+
.map(|rl| relevant.original[rl])
794+
.collect::<Vec<_>>()
782795
};
783796
dump_mir(tcx, false, "DestinationPropagation-dataflow", &round, body, |pass_where, w| {
784797
if let PassWhere::BeforeLocation(loc) = pass_where {

0 commit comments

Comments
 (0)