Skip to content

Commit 0129d1d

Browse files
fhahnfhahn
committed
[VPlan] Materialize constant vector trip counts before final opts.
Materialize constant vector trip counts before ::execute, if the trip count can be computed as Original (TC / (VF * UF)) * (VF * UF). For now this excludes when the tail is folded or scalar epilogues are required. This enables removing a number of redundant branches from the middle block. For now this is also only done when not vectorizing the epilogue, as the simplification complicates stitching the 2 plans together.
1 parent aa09dbb commit 0129d1d

File tree

165 files changed

+1417
-1416
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

165 files changed

+1417
-1416
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7560,6 +7560,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
75607560
VPlanTransforms::runPass(VPlanTransforms::materializeBroadcasts, BestVPlan);
75617561
VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
75627562
VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType());
7563+
VPlanTransforms::removeBranchOnConst(BestVPlan);
75637564
VPlanTransforms::narrowInterleaveGroups(
75647565
BestVPlan, BestVF,
75657566
TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector));
@@ -10536,6 +10537,25 @@ bool LoopVectorizePass::processLoop(Loop *L) {
1053610537
InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width,
1053710538
VF.MinProfitableTripCount, IC, &CM, BFI, PSI,
1053810539
Checks, BestPlan);
10540+
10541+
// Materialize vector trip counts for constants early if it can simply
10542+
// be computed as (Original TC / VF * UF) * VF * UF.
10543+
if (BestPlan.hasScalarTail() &&
10544+
!CM.requiresScalarEpilogue(VF.Width.isVector())) {
10545+
VPValue *TC = BestPlan.getTripCount();
10546+
if (TC->isLiveIn()) {
10547+
ScalarEvolution &SE = *PSE.getSE();
10548+
auto *TCScev = SE.getSCEV(TC->getLiveInIRValue());
10549+
const SCEV *VFxUF =
10550+
SE.getElementCount(TCScev->getType(), VF.Width * IC);
10551+
auto VecTCScev =
10552+
SE.getMulExpr(SE.getUDivExpr(TCScev, VFxUF), VFxUF);
10553+
if (auto *NewC = dyn_cast<SCEVConstant>(VecTCScev))
10554+
BestPlan.getVectorTripCount().setUnderlyingValue(
10555+
NewC->getValue());
10556+
}
10557+
}
10558+
1053910559
LVP.executePlan(VF.Width, IC, BestPlan, LB, DT, false);
1054010560
++LoopsVectorized;
1054110561

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -941,7 +941,8 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
941941
BackedgeTakenCount->setUnderlyingValue(TCMO);
942942
}
943943

944-
VectorTripCount.setUnderlyingValue(VectorTripCountV);
944+
if (!VectorTripCount.getUnderlyingValue())
945+
VectorTripCount.setUnderlyingValue(VectorTripCountV);
945946

946947
IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
947948
// FIXME: Model VF * UF computation completely in VPlan.

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1842,9 +1842,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
18421842
}
18431843
}
18441844

1845-
/// Remove BranchOnCond recipes with true or false conditions together with
1846-
/// removing dead edges to their successors.
1847-
static void removeBranchOnConst(VPlan &Plan) {
1845+
void VPlanTransforms::removeBranchOnConst(VPlan &Plan) {
18481846
using namespace llvm::VPlanPatternMatch;
18491847
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
18501848
vp_depth_first_shallow(Plan.getEntry()))) {

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,10 @@ struct VPlanTransforms {
202202
/// CanonicalIVTy as type for all un-typed live-ins in VPTypeAnalysis.
203203
static void simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy);
204204

205+
/// Remove BranchOnCond recipes with true or false conditions together with
206+
/// removing dead edges to their successors.
207+
static void removeBranchOnConst(VPlan &Plan);
208+
205209
/// If there's a single exit block, optimize its phi recipes that use exiting
206210
/// IV values by feeding them precomputed end values instead, possibly taken
207211
/// one step backwards.

‎llvm/test/Transforms/LoopVectorize/AArch64/blend-costs.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
368368
; CHECK-NEXT: [[TMP71:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
369369
; CHECK-NEXT: br i1 [[TMP71]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
370370
; CHECK: [[MIDDLE_BLOCK]]:
371-
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
371+
; CHECK-NEXT: br label %[[SCALAR_PH]]
372372
; CHECK: [[SCALAR_PH]]:
373373
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 96, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
374374
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
@@ -388,7 +388,7 @@ define void @test_blend_feeding_replicated_store_2(ptr noalias %src, ptr %dst, i
388388
; CHECK: [[LOOP_LATCH]]:
389389
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV1]], 1
390390
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 100
391-
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
391+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
392392
; CHECK: [[EXIT]]:
393393
; CHECK-NEXT: ret void
394394
;

‎llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ define void @fshl_operand_first_order_recurrence(ptr %dst, ptr noalias %src) {
3232
; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3333
; CHECK: [[MIDDLE_BLOCK]]:
3434
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i64> [[WIDE_LOAD1]], i32 1
35-
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
35+
; CHECK-NEXT: br label %[[SCALAR_PH]]
3636
; CHECK: [[SCALAR_PH]]:
3737
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
3838
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
@@ -47,7 +47,7 @@ define void @fshl_operand_first_order_recurrence(ptr %dst, ptr noalias %src) {
4747
; CHECK-NEXT: store i64 [[OR]], ptr [[GEP_DST]], align 8
4848
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
4949
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 100
50-
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
50+
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
5151
; CHECK: [[EXIT]]:
5252
; CHECK-NEXT: ret void
5353
;
@@ -86,9 +86,9 @@ define void @powi_call(ptr %P) {
8686
; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[TMP4]], align 8
8787
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
8888
; CHECK: [[MIDDLE_BLOCK]]:
89-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
89+
; CHECK-NEXT: br label %[[EXIT:.*]]
9090
; CHECK: [[SCALAR_PH]]:
91-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 2, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
91+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
9292
; CHECK-NEXT: br label %[[LOOP:.*]]
9393
; CHECK: [[LOOP]]:
9494
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -680,7 +680,7 @@ define void @multiple_exit_conditions(ptr %src, ptr noalias %dst) #1 {
680680
; DEFAULT-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
681681
; DEFAULT-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
682682
; DEFAULT: [[MIDDLE_BLOCK]]:
683-
; DEFAULT-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
683+
; DEFAULT-NEXT: br label %[[SCALAR_PH]]
684684
; DEFAULT: [[SCALAR_PH]]:
685685
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
686686
; DEFAULT-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ 512, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
@@ -696,7 +696,7 @@ define void @multiple_exit_conditions(ptr %src, ptr noalias %dst) #1 {
696696
; DEFAULT-NEXT: [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 8
697697
; DEFAULT-NEXT: [[IV_CLAMP:%.*]] = and i64 [[IV]], 4294967294
698698
; DEFAULT-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_CLAMP]], 512
699-
; DEFAULT-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
699+
; DEFAULT-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
700700
; DEFAULT: [[EXIT]]:
701701
; DEFAULT-NEXT: ret void
702702
;
@@ -1492,7 +1492,7 @@ define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) {
14921492
; DEFAULT-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
14931493
; DEFAULT-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
14941494
; DEFAULT: [[MIDDLE_BLOCK]]:
1495-
; DEFAULT-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
1495+
; DEFAULT-NEXT: br label %[[SCALAR_PH]]
14961496
; DEFAULT: [[SCALAR_PH]]:
14971497
; DEFAULT-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
14981498
; DEFAULT-NEXT: br label %[[LOOP_HEADER:.*]]
@@ -1506,7 +1506,7 @@ define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) {
15061506
; DEFAULT-NEXT: [[T:%.*]] = trunc nuw nsw i64 [[IV_NEXT]] to i32
15071507
; DEFAULT-NEXT: store i32 [[T]], ptr [[DST]], align 4
15081508
; DEFAULT-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 21
1509-
; DEFAULT-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP29:![0-9]+]]
1509+
; DEFAULT-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP29:![0-9]+]]
15101510
; DEFAULT: [[EXIT]]:
15111511
; DEFAULT-NEXT: ret void
15121512
;

‎llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -469,7 +469,7 @@ define void @old_and_new_size_equalko(ptr noalias %src, ptr noalias %dst) {
469469
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
470470
; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
471471
; CHECK: middle.block:
472-
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
472+
; CHECK-NEXT: br label [[EXIT:%.*]]
473473
; CHECK: scalar.ph:
474474
; CHECK-NEXT: br label [[LOOP:%.*]]
475475
; CHECK: loop:

‎llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,9 @@ define void @check_widen_intrinsic_with_nnan(ptr noalias %dst.0, ptr noalias %ds
7272
; CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
7373
; CHECK-NEXT: br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
7474
; CHECK: [[MIDDLE_BLOCK]]:
75-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
75+
; CHECK-NEXT: br label %[[EXIT:.*]]
7676
; CHECK: [[SCALAR_PH]]:
77-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
77+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
7878
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
7979
; CHECK: [[LOOP_HEADER]]:
8080
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/fminimumnum.ll

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
4040
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
4141
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4242
; CHECK: [[MIDDLE_BLOCK]]:
43-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
43+
; CHECK-NEXT: br label %[[EXIT:.*]]
4444
; CHECK: [[SCALAR_PH]]:
45-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
45+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
4646
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
4747
; CHECK: [[FOR_BODY]]:
4848
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -121,9 +121,9 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
121121
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
122122
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
123123
; CHECK: [[MIDDLE_BLOCK]]:
124-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
124+
; CHECK-NEXT: br label %[[EXIT:.*]]
125125
; CHECK: [[SCALAR_PH]]:
126-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
126+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
127127
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
128128
; CHECK: [[FOR_BODY]]:
129129
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -202,9 +202,9 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
202202
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
203203
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
204204
; CHECK: [[MIDDLE_BLOCK]]:
205-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
205+
; CHECK-NEXT: br label %[[EXIT:.*]]
206206
; CHECK: [[SCALAR_PH]]:
207-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
207+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
208208
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
209209
; CHECK: [[FOR_BODY]]:
210210
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -283,9 +283,9 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
283283
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
284284
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
285285
; CHECK: [[MIDDLE_BLOCK]]:
286-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
286+
; CHECK-NEXT: br label %[[EXIT:.*]]
287287
; CHECK: [[SCALAR_PH]]:
288-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
288+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
289289
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
290290
; CHECK: [[FOR_BODY]]:
291291
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -364,9 +364,9 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
364364
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
365365
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
366366
; CHECK: [[MIDDLE_BLOCK]]:
367-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
367+
; CHECK-NEXT: br label %[[EXIT:.*]]
368368
; CHECK: [[SCALAR_PH]]:
369-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
369+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
370370
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
371371
; CHECK: [[FOR_BODY]]:
372372
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -445,9 +445,9 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
445445
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
446446
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
447447
; CHECK: [[MIDDLE_BLOCK]]:
448-
; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
448+
; CHECK-NEXT: br label %[[EXIT:.*]]
449449
; CHECK: [[SCALAR_PH]]:
450-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
450+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
451451
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
452452
; CHECK: [[FOR_BODY]]:
453453
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]

0 commit comments

Comments
 (0)