Skip to content

Commit 88e170a

Browse files
fhahnfhahn
committed
[SCEV] Try to re-use pointer LCSSA phis when expanding SCEVs.
Generalize the code added in llvm#147214 to also support re-using pointer LCSSA phis when expanding integer SCEVs with AddRecs. A common source of integer AddRecs with pointer bases are runtime checks emitted by LV based on the distance between 2 pointer AddRecs. This improves codegen in some cases when vectorizing and prevents regressions with llvm#142309, which turns some phis into single-entry ones, which SCEV will look through now (and expand the whole AddRec), whereas before it would have to treat the LCSSA phi as SCEVUnknown.
1 parent 2f17107 commit 88e170a

File tree

4 files changed

+37
-32
lines changed

4 files changed

+37
-32
lines changed

‎llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1224,18 +1224,33 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
12241224
}
12251225

12261226
Value *SCEVExpander::tryToReuseLCSSAPhi(const SCEVAddRecExpr *S) {
1227+
Type *STy = S->getType();
12271228
const Loop *L = S->getLoop();
12281229
BasicBlock *EB = L->getExitBlock();
12291230
if (!EB || !EB->getSinglePredecessor() ||
12301231
!SE.DT.dominates(EB, Builder.GetInsertBlock()))
12311232
return nullptr;
12321233

12331234
for (auto &PN : EB->phis()) {
1234-
if (!SE.isSCEVable(PN.getType()) || PN.getType() != S->getType())
1235+
if (!SE.isSCEVable(PN.getType()))
12351236
continue;
1236-
auto *ExitV = SE.getSCEV(&PN);
1237-
if (S == ExitV)
1238-
return &PN;
1237+
auto *ExitSCEV = SE.getSCEV(&PN);
1238+
Type *PhiTy = PN.getType();
1239+
if (STy->isIntegerTy() && PhiTy->isPointerTy())
1240+
ExitSCEV = SE.getPtrToIntExpr(ExitSCEV, STy);
1241+
else if (S->getType() != PN.getType())
1242+
continue;
1243+
const SCEV *Diff = SE.getMinusSCEV(S, ExitSCEV);
1244+
if (isa<SCEVCouldNotCompute>(Diff) ||
1245+
SCEVExprContains(Diff,
1246+
[](const SCEV *S) { return isa<SCEVAddRecExpr>(S); }))
1247+
continue;
1248+
1249+
Value *DiffV = expand(Diff);
1250+
Value *BaseV = &PN;
1251+
if (DiffV->getType()->isIntegerTy() && PhiTy->isPointerTy())
1252+
BaseV = Builder.CreatePtrToInt(BaseV, DiffV->getType());
1253+
return Builder.CreateAdd(BaseV, DiffV);
12391254
}
12401255

12411256
return nullptr;

‎llvm/test/Transforms/LoopLoadElim/invalidate-laa-after-versioning.ll

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,19 +59,16 @@ define void @test(ptr %arg, i64 %arg1) {
5959
; CHECK-NEXT: [[GEP_5:%.*]] = getelementptr inbounds double, ptr [[LCSSA_PTR_IV_1]], i64 1
6060
; CHECK-NEXT: br label [[INNER_2:%.*]]
6161
; CHECK: inner.2:
62-
; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[INNER_2]] ], [ 0, [[INNER_1_EXIT]] ]
6362
; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[GEP_5]], [[INNER_1_EXIT]] ], [ [[PTR_IV_2_NEXT:%.*]], [[INNER_2]] ]
6463
; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr inbounds double, ptr [[PTR_IV_2]], i64 1
65-
; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
6664
; CHECK-NEXT: br i1 false, label [[INNER_3_LVER_CHECK:%.*]], label [[INNER_2]]
6765
; CHECK: inner.3.lver.check:
68-
; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], [[INNER_2]] ]
6966
; CHECK-NEXT: [[LCSSA_PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2]], [[INNER_2]] ]
7067
; CHECK-NEXT: [[GEP_6:%.*]] = getelementptr inbounds double, ptr [[PTR_PHI]], i64 1
7168
; CHECK-NEXT: [[GEP_7:%.*]] = getelementptr inbounds double, ptr [[LCSSA_PTR_IV_2]], i64 1
72-
; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[INDVAR_LCSSA]], 3
73-
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 24
74-
; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP1]]
69+
; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[LCSSA_PTR_IV_2]] to i64
70+
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 16
71+
; CHECK-NEXT: [[SCEVGEP3:%.*]] = inttoptr i64 [[TMP1]] to ptr
7572
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[GEP_7]], [[GEP_1]]
7673
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[PTR_PHI]], [[SCEVGEP3]]
7774
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -104,7 +101,7 @@ define void @test(ptr %arg, i64 %arg1) {
104101
; CHECK-NEXT: br i1 [[C_2]], label [[OUTER_LATCH_LOOPEXIT4:%.*]], label [[INNER_3]]
105102
; CHECK: outer.latch.loopexit:
106103
; CHECK-NEXT: br label [[OUTER_LATCH]]
107-
; CHECK: outer.latch.loopexit4:
104+
; CHECK: outer.latch.loopexit3:
108105
; CHECK-NEXT: br label [[OUTER_LATCH]]
109106
; CHECK: outer.latch:
110107
; CHECK-NEXT: br label [[INNER_1_LVER_CHECK]]

‎llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -104,27 +104,23 @@ define void @runtime_checks_ptr_inductions(ptr %dst.1, ptr %dst.2, i1 %c) {
104104
; CHECK-LABEL: define void @runtime_checks_ptr_inductions(
105105
; CHECK-SAME: ptr [[DST_1:%.*]], ptr [[DST_2:%.*]], i1 [[C:%.*]]) {
106106
; CHECK-NEXT: [[ENTRY:.*]]:
107-
; CHECK-NEXT: [[DST_11:%.*]] = ptrtoint ptr [[DST_1]] to i64
108107
; CHECK-NEXT: br label %[[LOOP_1:.*]]
109108
; CHECK: [[LOOP_1]]:
110-
; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[LOOP_1]] ], [ 0, %[[ENTRY]] ]
111109
; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[DST_1]], %[[ENTRY]] ], [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP_1]] ]
112110
; CHECK-NEXT: [[CALL:%.*]] = call i32 @val()
113111
; CHECK-NEXT: [[SEL_DST:%.*]] = select i1 [[C]], ptr [[DST_1]], ptr [[DST_2]]
114112
; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 1
115113
; CHECK-NEXT: [[EC_1:%.*]] = icmp eq i32 [[CALL]], 0
116-
; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
117114
; CHECK-NEXT: br i1 [[EC_1]], label %[[LOOP_2_HEADER_PREHEADER:.*]], label %[[LOOP_1]]
118115
; CHECK: [[LOOP_2_HEADER_PREHEADER]]:
119-
; CHECK-NEXT: [[SEL_DST_LCSSA2:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ]
120-
; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], %[[LOOP_1]] ]
116+
; CHECK-NEXT: [[SEL_DST_LCSSA1:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ]
121117
; CHECK-NEXT: [[PTR_IV_1_LCSSA:%.*]] = phi ptr [ [[PTR_IV_1]], %[[LOOP_1]] ]
122118
; CHECK-NEXT: [[SEL_DST_LCSSA:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ]
123-
; CHECK-NEXT: [[SEL_DST_LCSSA23:%.*]] = ptrtoint ptr [[SEL_DST_LCSSA2]] to i64
119+
; CHECK-NEXT: [[SEL_DST_LCSSA12:%.*]] = ptrtoint ptr [[SEL_DST_LCSSA1]] to i64
124120
; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
125121
; CHECK: [[VECTOR_MEMCHECK]]:
126-
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDVAR_LCSSA]], [[DST_11]]
127-
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SEL_DST_LCSSA23]]
122+
; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[PTR_IV_1_LCSSA]] to i64
123+
; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SEL_DST_LCSSA12]]
128124
; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 2
129125
; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
130126
; CHECK: [[VECTOR_PH]]:
@@ -146,13 +142,13 @@ define void @runtime_checks_ptr_inductions(ptr %dst.1, ptr %dst.2, i1 %c) {
146142
; CHECK-NEXT: br label %[[SCALAR_PH]]
147143
; CHECK: [[SCALAR_PH]]:
148144
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1023, %[[MIDDLE_BLOCK]] ], [ 1, %[[LOOP_2_HEADER_PREHEADER]] ], [ 1, %[[VECTOR_MEMCHECK]] ]
149-
; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[PTR_IV_1_LCSSA]], %[[VECTOR_MEMCHECK]] ]
150-
; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SEL_DST_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[SEL_DST_LCSSA]], %[[VECTOR_MEMCHECK]] ]
145+
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[PTR_IV_1_LCSSA]], %[[VECTOR_MEMCHECK]] ]
146+
; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SEL_DST_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[SEL_DST_LCSSA]], %[[VECTOR_MEMCHECK]] ]
151147
; CHECK-NEXT: br label %[[LOOP_2_HEADER:.*]]
152148
; CHECK: [[LOOP_2_HEADER]]:
153149
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[DEC7:%.*]], %[[LOOP_2_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
154-
; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
155-
; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ]
150+
; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ]
151+
; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
156152
; CHECK-NEXT: [[EC_2:%.*]] = icmp eq i32 [[IV]], 1024
157153
; CHECK-NEXT: br i1 [[EC_2]], label %[[EXIT:.*]], label %[[LOOP_2_LATCH]]
158154
; CHECK: [[LOOP_2_LATCH]]:

‎llvm/test/Transforms/LoopVersioning/invalidate-laa-after-versioning.ll

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -56,19 +56,16 @@ define void @test(ptr %arg, i64 %arg1) {
5656
; CHECK-NEXT: [[GEP_5:%.*]] = getelementptr inbounds double, ptr [[LCSSA_PTR_IV_1]], i64 1
5757
; CHECK-NEXT: br label [[INNER_2:%.*]]
5858
; CHECK: inner.2:
59-
; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[INNER_2]] ], [ 0, [[INNER_1_EXIT]] ]
6059
; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[GEP_5]], [[INNER_1_EXIT]] ], [ [[PTR_IV_2_NEXT:%.*]], [[INNER_2]] ]
6160
; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr inbounds double, ptr [[PTR_IV_2]], i64 1
62-
; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
6361
; CHECK-NEXT: br i1 false, label [[INNER_3_LVER_CHECK:%.*]], label [[INNER_2]]
6462
; CHECK: inner.3.lver.check:
65-
; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], [[INNER_2]] ]
6663
; CHECK-NEXT: [[LCSSA_PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2]], [[INNER_2]] ]
6764
; CHECK-NEXT: [[GEP_6:%.*]] = getelementptr inbounds double, ptr [[PTR_PHI]], i64 1
6865
; CHECK-NEXT: [[GEP_7:%.*]] = getelementptr inbounds double, ptr [[LCSSA_PTR_IV_2]], i64 1
69-
; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[INDVAR_LCSSA]], 3
70-
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 24
71-
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP1]]
66+
; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[LCSSA_PTR_IV_2]] to i64
67+
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 16
68+
; CHECK-NEXT: [[SCEVGEP:%.*]] = inttoptr i64 [[TMP1]] to ptr
7269
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[GEP_7]], [[GEP_1]]
7370
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[PTR_PHI]], [[SCEVGEP]]
7471
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
@@ -90,10 +87,10 @@ define void @test(ptr %arg, i64 %arg1) {
9087
; CHECK: inner.3:
9188
; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ 0, [[INNER_3_PH]] ], [ [[IV_2_NEXT:%.*]], [[INNER_3]] ]
9289
; CHECK-NEXT: [[GEP_8:%.*]] = getelementptr inbounds double, ptr [[GEP_6]], i64 [[IV_2]]
93-
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_7]], align 8, !alias.scope !0, !noalias !3
94-
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_8]], align 8, !alias.scope !3
90+
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_7]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
91+
; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_8]], align 8, !alias.scope [[META3]]
9592
; CHECK-NEXT: [[GEP_9:%.*]] = getelementptr double, ptr [[PTR_PHI]], i64 [[IV_2]]
96-
; CHECK-NEXT: [[TMP18:%.*]] = load double, ptr [[GEP_9]], align 8, !alias.scope !3
93+
; CHECK-NEXT: [[TMP18:%.*]] = load double, ptr [[GEP_9]], align 8, !alias.scope [[META3]]
9794
; CHECK-NEXT: [[IV_2_NEXT]] = add nuw nsw i64 [[IV_2]], 1
9895
; CHECK-NEXT: [[C_2:%.*]] = icmp eq i64 [[IV_2]], 1
9996
; CHECK-NEXT: br i1 [[C_2]], label [[OUTER_LATCH_LOOPEXIT3:%.*]], label [[INNER_3]]

0 commit comments

Comments
 (0)