Skip to content

Commit cd86af4

Browse files
ElvisWang123ElvisWang123
committed
!fixup, address comments.
1 parent 484f9cc commit cd86af4

File tree

5 files changed

+82
-194
lines changed

5 files changed

+82
-194
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 26 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -599,11 +599,6 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
599599
DisjointFlagsTy(bool IsDisjoint) : IsDisjoint(IsDisjoint) {}
600600
};
601601

602-
struct GEPFlagsTy {
603-
char IsInBounds : 1;
604-
GEPFlagsTy(bool IsInBounds) : IsInBounds(IsInBounds) {}
605-
};
606-
607602
struct NonNegFlagsTy {
608603
char NonNeg : 1;
609604
NonNegFlagsTy(bool IsNonNeg) : NonNeg(IsNonNeg) {}
@@ -2303,9 +2298,9 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
23032298
/// Note that IsNonNeg flag and the debug location are for extend instruction.
23042299
VPReductionRecipe(const unsigned char SC, const RecurrenceDescriptor &R,
23052300
ArrayRef<VPValue *> Operands, VPValue *CondOp,
2306-
bool IsOrdered, bool IsNonNeg, DebugLoc DL)
2307-
: VPRecipeWithIRFlags(SC, Operands, NonNegFlagsTy(IsNonNeg), DL),
2308-
RdxDesc(R), IsOrdered(IsOrdered), IsConditional(CondOp) {
2301+
bool IsOrdered, NonNegFlagsTy NonNeg, DebugLoc DL)
2302+
: VPRecipeWithIRFlags(SC, Operands, NonNeg, DL), RdxDesc(R),
2303+
IsOrdered(IsOrdered), IsConditional(CondOp) {
23092304
if (CondOp)
23102305
addOperand(CondOp);
23112306
}
@@ -2314,9 +2309,9 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
23142309
/// Note that the NUW/NSW and DL are for mul instruction.
23152310
VPReductionRecipe(const unsigned char SC, const RecurrenceDescriptor &R,
23162311
ArrayRef<VPValue *> Operands, VPValue *CondOp,
2317-
bool IsOrdered, bool NUW, bool NSW, DebugLoc DL)
2318-
: VPRecipeWithIRFlags(SC, Operands, WrapFlagsTy(NUW, NSW), DL),
2319-
RdxDesc(R), IsOrdered(IsOrdered), IsConditional(CondOp) {
2312+
bool IsOrdered, WrapFlagsTy WrapFlags, DebugLoc DL)
2313+
: VPRecipeWithIRFlags(SC, Operands, WrapFlags, DL), RdxDesc(R),
2314+
IsOrdered(IsOrdered), IsConditional(CondOp) {
23202315
if (CondOp)
23212316
addOperand(CondOp);
23222317
}
@@ -2441,22 +2436,24 @@ class VPExtendedReductionRecipe : public VPReductionRecipe {
24412436
VPExtendedReductionRecipe(VPExtendedReductionRecipe *ExtRed)
24422437
: VPReductionRecipe(
24432438
VPDef::VPExtendedReductionSC, ExtRed->getRecurrenceDescriptor(),
2444-
{ExtRed->getChainOp(), ExtRed->getVecOp()}, ExtRed->getCondOp(),
2445-
ExtRed->isOrdered(), ExtRed->isNonNeg(), ExtRed->getDebugLoc()),
2439+
nullptr, {ExtRed->getChainOp(), ExtRed->getVecOp()},
2440+
ExtRed->getCondOp(), ExtRed->isOrdered(), ExtRed->getDebugLoc()),
24462441
ExtOp(ExtRed->getExtOpcode()) {}
24472442

24482443
public:
24492444
VPExtendedReductionRecipe(VPReductionRecipe *R, VPWidenCastRecipe *Ext)
2450-
: VPReductionRecipe(VPDef::VPExtendedReductionSC,
2451-
R->getRecurrenceDescriptor(),
2452-
{R->getChainOp(), Ext->getOperand(0)}, R->getCondOp(),
2453-
R->isOrdered(), Ext->isNonNeg(), Ext->getDebugLoc()),
2445+
: VPReductionRecipe(
2446+
VPDef::VPExtendedReductionSC, R->getRecurrenceDescriptor(),
2447+
{R->getChainOp(), Ext->getOperand(0)}, R->getCondOp(),
2448+
R->isOrdered(), NonNegFlagsTy(Ext->isNonNeg()), Ext->getDebugLoc()),
24542449
ExtOp(Ext->getOpcode()) {}
24552450

24562451
~VPExtendedReductionRecipe() override = default;
24572452

24582453
VPExtendedReductionRecipe *clone() override {
2459-
return new VPExtendedReductionRecipe(this);
2454+
auto *Copy = new VPExtendedReductionRecipe(this);
2455+
Copy->transferFlags(*this);
2456+
return Copy;
24602457
}
24612458

24622459
VP_CLASSOF_IMPL(VPDef::VPExtendedReductionSC);
@@ -2504,11 +2501,9 @@ class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
25042501
VPMulAccumulateReductionRecipe(VPMulAccumulateReductionRecipe *MulAcc)
25052502
: VPReductionRecipe(
25062503
VPDef::VPMulAccumulateReductionSC,
2507-
MulAcc->getRecurrenceDescriptor(),
2504+
MulAcc->getRecurrenceDescriptor(), nullptr,
25082505
{MulAcc->getChainOp(), MulAcc->getVecOp0(), MulAcc->getVecOp1()},
2509-
MulAcc->getCondOp(), MulAcc->isOrdered(),
2510-
MulAcc->hasNoUnsignedWrap(), MulAcc->hasNoSignedWrap(),
2511-
MulAcc->getDebugLoc()),
2506+
MulAcc->getCondOp(), MulAcc->isOrdered(), MulAcc->getDebugLoc()),
25122507
ExtOp(MulAcc->getExtOpcode()), IsNonNeg(MulAcc->isNonNeg()) {}
25132508

25142509
public:
@@ -2518,8 +2513,9 @@ class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
25182513
: VPReductionRecipe(
25192514
VPDef::VPMulAccumulateReductionSC, R->getRecurrenceDescriptor(),
25202515
{R->getChainOp(), Ext0->getOperand(0), Ext1->getOperand(0)},
2521-
R->getCondOp(), R->isOrdered(), Mul->hasNoUnsignedWrap(),
2522-
Mul->hasNoSignedWrap(), R->getDebugLoc()),
2516+
R->getCondOp(), R->isOrdered(),
2517+
WrapFlagsTy(Mul->hasNoUnsignedWrap(), Mul->hasNoSignedWrap()),
2518+
R->getDebugLoc()),
25232519
ExtOp(Ext0->getOpcode()), IsNonNeg(Ext0->isNonNeg()) {
25242520
assert(getRecurrenceDescriptor().getOpcode() == Instruction::Add &&
25252521
"The reduction instruction in MulAccumulateteReductionRecipe must "
@@ -2530,8 +2526,9 @@ class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
25302526
: VPReductionRecipe(
25312527
VPDef::VPMulAccumulateReductionSC, R->getRecurrenceDescriptor(),
25322528
{R->getChainOp(), Mul->getOperand(0), Mul->getOperand(1)},
2533-
R->getCondOp(), R->isOrdered(), Mul->hasNoUnsignedWrap(),
2534-
Mul->hasNoSignedWrap(), R->getDebugLoc()),
2529+
R->getCondOp(), R->isOrdered(),
2530+
WrapFlagsTy(Mul->hasNoUnsignedWrap(), Mul->hasNoSignedWrap()),
2531+
R->getDebugLoc()),
25352532
ExtOp(Instruction::CastOps::CastOpsEnd) {
25362533
assert(getRecurrenceDescriptor().getOpcode() == Instruction::Add &&
25372534
"The reduction instruction in MulAccumulateReductionRecipe must be "
@@ -2541,7 +2538,9 @@ class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
25412538
~VPMulAccumulateReductionRecipe() override = default;
25422539

25432540
VPMulAccumulateReductionRecipe *clone() override {
2544-
return new VPMulAccumulateReductionRecipe(this);
2541+
auto *Copy = new VPMulAccumulateReductionRecipe(this);
2542+
Copy->transferFlags(*this);
2543+
return Copy;
25452544
}
25462545

25472546
VP_CLASSOF_IMPL(VPDef::VPMulAccumulateReductionSC);

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2416,12 +2416,8 @@ VPExtendedReductionRecipe::computeCost(ElementCount VF,
24162416
auto *SrcVecTy =
24172417
cast<VectorType>(toVectorTy(Ctx.Types.inferScalarType(getVecOp()), VF));
24182418

2419-
if (RedTy->isFloatingPointTy())
2420-
return Ctx.TTI.getExtendedReductionCost(Opcode, isZExt(), RedTy, SrcVecTy,
2421-
getFastMathFlags(), Ctx.CostKind);
2422-
2423-
// Cannot get correct cost when quering TTI with FMFs not contains `reassoc`
2424-
// for non-FP reductions.
2419+
assert(RedTy->isIntegerTy() &&
2420+
"ExtendedReduction only support integer type currently.");
24252421
return Ctx.TTI.getExtendedReductionCost(Opcode, isZExt(), RedTy, SrcVecTy,
24262422
std::nullopt, Ctx.CostKind);
24272423
}

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2273,11 +2273,10 @@ void VPlanTransforms::handleUncountableEarlyExit(
22732273
LatchExitingBranch->eraseFromParent();
22742274
}
22752275

2276-
/// This function tries to match following pattern to create
2276+
/// This function tries convert extended in-loop reductions to
22772277
/// VPExtendedReductionRecipe and clamp the \p Range if it is beneficial and
2278-
/// valid. The created VPExtendedReductionRecipe will lower to concrete recipes
2279-
/// before execution.
2280-
/// reduce(ext(...)).
2278+
/// valid. The created VPExtendedReductionRecipe must be lower to concrete
2279+
/// recipes before execution.
22812280
static VPExtendedReductionRecipe *
22822281
tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
22832282
VFRange &Range) {
@@ -2295,7 +2294,7 @@ tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
22952294
auto *SrcVecTy = cast<VectorType>(toVectorTy(SrcTy, VF));
22962295
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
22972296
InstructionCost ExtRedCost = Ctx.TTI.getExtendedReductionCost(
2298-
Opcode, isZExt, RedTy, SrcVecTy, RdxDesc.getFastMathFlags(),
2297+
Opcode, isZExt, RedTy, SrcVecTy, Red->getFastMathFlags(),
22992298
CostKind);
23002299
InstructionCost ExtCost =
23012300
cast<VPWidenCastRecipe>(VecOp)->computeCost(VF, Ctx);
@@ -2318,10 +2317,10 @@ tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
23182317
return nullptr;
23192318
}
23202319

2321-
/// This function try to match following pattern to create
2320+
/// This function tries convert extended in-loop reductions to
23222321
/// VPMulAccumulateReductionRecipe and clamp the \p Range if it is beneficial
2323-
/// and valid. The created VPMulAccumulateReduction will lower to concrete
2324-
/// before executeion.
2322+
/// and valid. The created VPExtendedReductionRecipe must be lower to concrete
2323+
/// recipes before execution. Patterns of MulAccumulateReduction:
23252324
/// reduce.add(mul(...)),
23262325
/// reduce.add(mul(ext(A), ext(B))),
23272326
/// reduce.add(ext(mul(ext(A), ext(B)))).
@@ -2431,7 +2430,7 @@ static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
24312430
void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
24322431
VFRange &Range) {
24332432
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2434-
vp_depth_first_deep(Plan.getEntry()))) {
2433+
vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
24352434
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
24362435
if (auto *Red = dyn_cast<VPReductionRecipe>(&R))
24372436
tryToCreateAbstractReductionRecipe(Red, Ctx, Range);

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,8 +179,8 @@ struct VPlanTransforms {
179179
/// Lower abstract recipes to concrete ones, that can be codegen'd.
180180
static void convertToConcreteRecipes(VPlan &Plan);
181181

182-
/// This function convert initial recipes to the abstract recipes and clamp \p
183-
/// Range based on cost model for following optimizations and cost
182+
/// This function converts initial recipes to the abstract recipes and clamps
183+
/// \p Range based on cost model for following optimizations and cost
184184
/// estimations. The converted abstract recipes will lower to concrete
185185
/// recipies before codegen.
186186
static void convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,

0 commit comments

Comments
 (0)