@@ -2056,55 +2056,6 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
2056
2056
}
2057
2057
};
2058
2058
2059
- // / A recipe for forming partial reductions. In the loop, an accumulator and
2060
- // / vector operand are added together and passed to the next iteration as the
2061
- // / next accumulator. After the loop body, the accumulator is reduced to a
2062
- // / scalar value.
2063
- class VPPartialReductionRecipe : public VPSingleDefRecipe {
2064
- unsigned Opcode;
2065
-
2066
- public:
2067
- VPPartialReductionRecipe (Instruction *ReductionInst, VPValue *Op0,
2068
- VPValue *Op1)
2069
- : VPPartialReductionRecipe(ReductionInst->getOpcode (), Op0, Op1,
2070
- ReductionInst) {}
2071
- VPPartialReductionRecipe (unsigned Opcode, VPValue *Op0, VPValue *Op1,
2072
- Instruction *ReductionInst = nullptr )
2073
- : VPSingleDefRecipe(VPDef::VPPartialReductionSC,
2074
- ArrayRef<VPValue *>({Op0, Op1}), ReductionInst),
2075
- Opcode(Opcode) {
2076
- [[maybe_unused]] auto *AccumulatorRecipe =
2077
- getOperand (1 )->getDefiningRecipe ();
2078
- assert ((isa<VPReductionPHIRecipe>(AccumulatorRecipe) ||
2079
- isa<VPPartialReductionRecipe>(AccumulatorRecipe)) &&
2080
- " Unexpected operand order for partial reduction recipe" );
2081
- }
2082
- ~VPPartialReductionRecipe () override = default ;
2083
-
2084
- VPPartialReductionRecipe *clone () override {
2085
- return new VPPartialReductionRecipe (Opcode, getOperand (0 ), getOperand (1 ),
2086
- getUnderlyingInstr ());
2087
- }
2088
-
2089
- VP_CLASSOF_IMPL (VPDef::VPPartialReductionSC)
2090
-
2091
- // / Generate the reduction in the loop.
2092
- void execute(VPTransformState &State) override ;
2093
-
2094
- // / Return the cost of this VPPartialReductionRecipe.
2095
- InstructionCost computeCost (ElementCount VF,
2096
- VPCostContext &Ctx) const override ;
2097
-
2098
- // / Get the binary op's opcode.
2099
- unsigned getOpcode () const { return Opcode; }
2100
-
2101
- #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2102
- // / Print the recipe.
2103
- void print (raw_ostream &O, const Twine &Indent,
2104
- VPSlotTracker &SlotTracker) const override ;
2105
- #endif
2106
- };
2107
-
2108
2059
// / A recipe for vectorizing a phi-node as a sequence of mask-based select
2109
2060
// / instructions.
2110
2061
class VPBlendRecipe : public VPSingleDefRecipe {
@@ -2376,6 +2327,56 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
2376
2327
}
2377
2328
};
2378
2329
2330
+ // / A recipe for forming partial reductions. In the loop, an accumulator and
2331
+ // / vector operand are added together and passed to the next iteration as the
2332
+ // / next accumulator. After the loop body, the accumulator is reduced to a
2333
+ // / scalar value.
2334
+ class VPPartialReductionRecipe : public VPSingleDefRecipe {
2335
+ unsigned Opcode;
2336
+
2337
+ public:
2338
+ VPPartialReductionRecipe (Instruction *ReductionInst, VPValue *Op0,
2339
+ VPValue *Op1)
2340
+ : VPPartialReductionRecipe(ReductionInst->getOpcode (), Op0, Op1,
2341
+ ReductionInst) {}
2342
+ VPPartialReductionRecipe (unsigned Opcode, VPValue *Op0, VPValue *Op1,
2343
+ Instruction *ReductionInst = nullptr )
2344
+ : VPSingleDefRecipe(VPDef::VPPartialReductionSC,
2345
+ ArrayRef<VPValue *>({Op0, Op1}), ReductionInst),
2346
+ Opcode(Opcode) {
2347
+ [[maybe_unused]] auto *AccumulatorRecipe =
2348
+ getOperand (1 )->getDefiningRecipe ();
2349
+ assert ((isa<VPReductionPHIRecipe>(AccumulatorRecipe) ||
2350
+ isa<VPPartialReductionRecipe>(AccumulatorRecipe)) &&
2351
+ " Unexpected operand order for partial reduction recipe" );
2352
+ }
2353
+ ~VPPartialReductionRecipe () override = default ;
2354
+
2355
+ VPPartialReductionRecipe *clone () override {
2356
+ return new VPPartialReductionRecipe (Opcode, getOperand (0 ), getOperand (1 ),
2357
+ getUnderlyingInstr ());
2358
+ }
2359
+
2360
+ VP_CLASSOF_IMPL (VPDef::VPPartialReductionSC)
2361
+
2362
+ // / Generate the reduction in the loop.
2363
+ void execute(VPTransformState &State) override ;
2364
+
2365
+ // / Return the cost of this VPPartialReductionRecipe.
2366
+ InstructionCost computeCost (ElementCount VF,
2367
+ VPCostContext &Ctx) const override ;
2368
+
2369
+ // / Get the binary op's opcode.
2370
+ unsigned getOpcode () const { return Opcode; }
2371
+
2372
+ #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2373
+ // / Print the recipe.
2374
+ void print (raw_ostream &O, const Twine &Indent,
2375
+ VPSlotTracker &SlotTracker) const override ;
2376
+ #endif
2377
+ };
2378
+
2379
+
2379
2380
// / A recipe to represent inloop reduction operations with vector-predication
2380
2381
// / intrinsics, performing a reduction on a vector operand with the explicit
2381
2382
// / vector length (EVL) into a scalar value, and adding the result to a chain.
0 commit comments