@@ -968,6 +968,29 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
968
968
969
969
// FIXME: Do we need to add an alignment parameter to the intrinsic?
970
970
unsigned VectorWidth = AddrType->getNumElements ();
971
+ auto CreateHistogramUpdateValue = [&](IntrinsicInst *CI, Value *Load,
972
+ Value *Inc) -> Value * {
973
+ Value *UpdateOp;
974
+ switch (CI->getIntrinsicID ()) {
975
+ case Intrinsic::experimental_vector_histogram_add:
976
+ UpdateOp = Builder.CreateAdd (Load, Inc);
977
+ break ;
978
+ case Intrinsic::experimental_vector_histogram_uadd_sat:
979
+ UpdateOp =
980
+ Builder.CreateIntrinsic (Intrinsic::uadd_sat, {EltTy}, {Load, Inc});
981
+ break ;
982
+ case Intrinsic::experimental_vector_histogram_umin:
983
+ UpdateOp = Builder.CreateIntrinsic (Intrinsic::umin, {EltTy}, {Load, Inc});
984
+ break ;
985
+ case Intrinsic::experimental_vector_histogram_umax:
986
+ UpdateOp = Builder.CreateIntrinsic (Intrinsic::umax, {EltTy}, {Load, Inc});
987
+ break ;
988
+
989
+ default :
990
+ llvm_unreachable (" Unexpected histogram intrinsic" );
991
+ }
992
+ return UpdateOp;
993
+ };
971
994
972
995
// Shorten the way if the mask is a vector of constants.
973
996
if (isConstantIntVector (Mask)) {
@@ -976,8 +999,9 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
976
999
continue ;
977
1000
Value *Ptr = Builder.CreateExtractElement (Ptrs, Idx, " Ptr" + Twine (Idx));
978
1001
LoadInst *Load = Builder.CreateLoad (EltTy, Ptr, " Load" + Twine (Idx));
979
- Value *Add = Builder.CreateAdd (Load, Inc);
980
- Builder.CreateStore (Add, Ptr);
1002
+ Value *Update =
1003
+ CreateHistogramUpdateValue (cast<IntrinsicInst>(CI), Load, Inc);
1004
+ Builder.CreateStore (Update, Ptr);
981
1005
}
982
1006
CI->eraseFromParent ();
983
1007
return ;
@@ -997,8 +1021,9 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
997
1021
Builder.SetInsertPoint (CondBlock->getTerminator ());
998
1022
Value *Ptr = Builder.CreateExtractElement (Ptrs, Idx, " Ptr" + Twine (Idx));
999
1023
LoadInst *Load = Builder.CreateLoad (EltTy, Ptr, " Load" + Twine (Idx));
1000
- Value *Add = Builder.CreateAdd (Load, Inc);
1001
- Builder.CreateStore (Add, Ptr);
1024
+ Value *UpdateOp =
1025
+ CreateHistogramUpdateValue (cast<IntrinsicInst>(CI), Load, Inc);
1026
+ Builder.CreateStore (UpdateOp, Ptr);
1002
1027
1003
1028
// Create "else" block, fill it in the next iteration
1004
1029
BasicBlock *NewIfBlock = ThenTerm->getSuccessor (0 );
@@ -1089,6 +1114,9 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
1089
1114
default :
1090
1115
break ;
1091
1116
case Intrinsic::experimental_vector_histogram_add:
1117
+ case Intrinsic::experimental_vector_histogram_uadd_sat:
1118
+ case Intrinsic::experimental_vector_histogram_umin:
1119
+ case Intrinsic::experimental_vector_histogram_umax:
1092
1120
if (TTI.isLegalMaskedVectorHistogram (CI->getArgOperand (0 )->getType (),
1093
1121
CI->getArgOperand (1 )->getType ()))
1094
1122
return false ;
0 commit comments