Merge commit '53efbc15cb8e2b98bbd02ff39765561d2426b111' into merge/ve-mask-broadcast

Author: Simon Moll

llvm/lib/Target/VE/ShuffleSynthesis.cpp

@@ -1173,7 +1173,7 @@ struct GatherShuffleOp final : public AbstractShuffleOp {
     }
 
     SDValue MaxVLV = CDAG.getConstEVL(MaxVL);
-    SDValue BasePtrV = CDAG.getBroadcast(PtrVecVT, VecSlotPtr);
+    SDValue BasePtrV = CDAG.getBroadcast(PtrVecVT, VecSlotPtr, MaxVLV);
     SDValue OffsetV = CDAG.getNode(
         ISD::BUILD_VECTOR, PtrVecVT,
         GatherOffsets); // TODO directly call into constant vector generation

llvm/lib/Target/VE/VE.h

@@ -38,7 +38,7 @@ namespace llvm {
 
 /// Target Constants {
 const unsigned StandardVectorWidth = 256;
-const unsigned PackedWidth = 512;
+const unsigned PackedVectorWidth = 512;
 /// } Target Constants
 
 // Enums corresponding to VE condition codes, both icc's and fcc's.  These

llvm/lib/Target/VE/VECustomDAG.cpp

@@ -696,12 +696,20 @@ static SDValue supplementPackedReplication(SDValue Op, SelectionDAG &DAG) {
   return DAG.getNode(VEISD::VEC_BROADCAST, DL, VT, {ReplOp, VLOp});
 }
 
-bool isMaskType(EVT VT) {
-  if (!VT.isVector())
-    return false;
+MVT getLegalVectorType(Packing P, MVT ElemVT) {
+  return MVT::getVectorVT(ElemVT, P == Packing::Normal ? StandardVectorWidth
+                                                       : PackedVectorWidth);
+}
 
-  // an actual bit mask type
-  return VT.getVectorElementType() == MVT::i1;
+Packing getTypePacking(EVT VT) {
+  assert(VT.isVector());
+  return isPackedVectorType(VT) ? Packing::Dense : Packing::Normal;
+}
+
+bool isMaskType(EVT SomeVT) {
+  if (!SomeVT.isVector())
+    return false;
+  return SomeVT.getVectorElementType() == MVT::i1;
 }
 
 bool maySafelyIgnoreMask(unsigned VVPOpcode) {
@@ -1085,6 +1093,48 @@ SDValue VECustomDAG::getConstant(uint64_t Val, EVT VT, bool IsTarget,
 
 void VECustomDAG::dumpValue(SDValue V) const { V->print(dbgs(), &DAG); }
 
+SDValue VECustomDAG::getConstantMask(Packing Packing, bool AllTrue) const {
+  auto MaskVT = getLegalVectorType(Packing, MVT::i1);
+
+  // VEISelDAGtoDAG will replace this pattern with the constant-true VM.
+  auto TrueVal = DAG.getConstant(-1, DL, MVT::i32);
+  auto AVL = getConstant(MaskVT.getVectorNumElements(), MVT::i32);
+  auto Res = getNode(VEISD::VEC_BROADCAST, MaskVT, {TrueVal, AVL});
+  if (AllTrue)
+    return Res;
+
+  return DAG.getNOT(DL, Res, Res.getValueType());
+}
+
+SDValue VECustomDAG::getMaskBroadcast(EVT ResultVT, SDValue Scalar,
+                                      SDValue AVL) const {
+  // Constant mask splat.
+  if (auto BcConst = dyn_cast<ConstantSDNode>(Scalar))
+    return getConstantMask(getTypePacking(ResultVT),
+                           BcConst->getSExtValue() != 0);
+
+  // Expand the broadcast to a vector comparison.
+  auto ScalarBoolVT = Scalar.getSimpleValueType();
+  assert(ScalarBoolVT == MVT::i32);
+
+  // Cast to i32 ty.
+  SDValue CmpElem = DAG.getSExtOrTrunc(Scalar, DL, MVT::i32);
+  unsigned ElemCount = ResultVT.getVectorNumElements();
+  MVT CmpVecTy = MVT::getVectorVT(ScalarBoolVT, ElemCount);
+
+  // Broadcast to vector.
+  SDValue BCVec =
+      DAG.getNode(VEISD::VEC_BROADCAST, DL, CmpVecTy, {CmpElem, AVL});
+  SDValue ZeroVec =
+      getBroadcast(CmpVecTy, {DAG.getConstant(0, DL, ScalarBoolVT)}, AVL);
+
+  MVT BoolVecTy = MVT::getVectorVT(MVT::i1, ElemCount);
+
+  // Broadcast(Data) != Broadcast(0)
+  // TODO: Use a VVP operation for this.
+  return DAG.getSetCC(DL, BoolVecTy, BCVec, ZeroVec, ISD::CondCode::SETNE);
+}
+
 SDValue VECustomDAG::getVectorExtract(SDValue VecV, SDValue IdxV) const {
   assert(VecV.getValueType().isVector());
   auto ElemVT = VecV.getValueType().getVectorElementType();
@@ -1167,7 +1217,7 @@ SDValue VECustomDAG::createConstantTargetMask(VVPWideningInfo WidenInfo) const {
   // we do not want to go through ::ReplaceNodeResults again only to have them
   // widened
   unsigned NativeVectorWidth =
-      WidenInfo.PackedMode ? PackedWidth : StandardVectorWidth;
+      WidenInfo.PackedMode ? PackedVectorWidth : StandardVectorWidth;
 
   // Generate a remainder mask for packed operations
   Packing PackFlag = WidenInfo.PackedMode ? Packing::Dense : Packing::Normal;
@@ -1182,7 +1232,7 @@ SDValue VECustomDAG::createConstantTargetMask(VVPWideningInfo WidenInfo) const {
     MaskBits.flip();
     size_t OddRemainderBitPos = WidenInfo.ActiveVectorLength;
     MaskBits[OddRemainderBitPos] = false;
-    return createConstMask<>(PackedWidth, MaskBits);
+    return createConstMask<>(PackedVectorWidth, MaskBits);
   }
 }
 

llvm/lib/Target/VE/VECustomDAG.h

@@ -195,6 +195,8 @@ SDValue getUnpackAVL(SDValue N);
 
 /// } Packing
 
+bool isMaskType(EVT SomeVT);
+
 bool isVVPOrVEC(unsigned);
 
 bool maySafelyIgnoreMask(unsigned Opc);
@@ -241,6 +243,12 @@ std::pair<SDValue, bool> getAnnotatedNodeAVL(SDValue);
 
 /// } AVL Functions
 
+// Get the vector or mask register type for this packing and element type.
+MVT getLegalVectorType(Packing P, MVT ElemVT);
+
+// Whether this type belongs to a packed mask or vector register.
+Packing getTypePacking(EVT);
+
 /// Helper class for short hand custom node creation ///
 struct VECustomDAG {
   const VELoweringInfo &VLI;
@@ -292,7 +300,6 @@ struct VECustomDAG {
   SDValue foldAndUnpackMask(SDValue MaskVector, SDValue Mask, PackElem Part,
                             SDValue AVL) const;
   /// } Mask Insert/Extract
-  SDValue getBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL = SDValue()) const;
 
   // Extract an SX register from a mask
   SDValue createMaskExtract(SDValue MaskV, SDValue Idx) const;
@@ -525,6 +532,9 @@ struct VECustomDAG {
 
   raw_ostream &print(raw_ostream &, SDValue) const;
   void dump(SDValue) const;
+  SDValue getConstantMask(Packing Packing, bool AllTrue) const;
+  SDValue getMaskBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const;
+  SDValue getBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const;
 
   // Wrap AVL in a LEGALAVL node (unless it is one already).
   SDValue annotateLegalAVL(SDValue AVL) const;

llvm/lib/Target/VE/VEISelDAGToDAG.cpp

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "VE.h"
 #include "VETargetMachine.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"

llvm/lib/Target/VE/VVPISelLowering.cpp

@@ -141,10 +141,10 @@ static bool OpNeedsSplitting(SDNode &Op) {
   return !supportsPackedMode(Op.getOpcode(), OpVecTy);
 #if 0
   unsigned OpVectorLength = OpVecTy.getVectorNumElements();
-  assert((OpVectorLength <= PackedWidth) &&
+  assert((OpVectorLength <= PackedVectorWidth) &&
          "Operation should have been split during legalization");
   return (OpVectorLength != StandardVectorWidth) &&
-         (OpVectorLength != PackedWidth);
+         (OpVectorLength != PackedVectorWidth);
 #endif
 }
 
@@ -441,7 +441,7 @@ void VETargetLowering::initVPUActions() {
   // Vector length legalization
   auto LegalizeVectorLength = [&](unsigned VL) -> unsigned {
     if (this->Subtarget->hasPackedMode()) {
-      return VL > StandardVectorWidth ? PackedWidth : StandardVectorWidth;
+      return VL > StandardVectorWidth ? PackedVectorWidth : StandardVectorWidth;
     } else {
       return StandardVectorWidth;
     }
@@ -590,7 +590,7 @@ void VETargetLowering::initVPUActions() {
     unsigned W = VT.getVectorMinNumElements();
 
     // Use default splitting for vlens > 512
-    if (W > PackedWidth)
+    if (W > PackedVectorWidth)
       continue;
 
     // Promotion rule, accept native element bit sizes
@@ -761,7 +761,7 @@ EVT VETargetLowering::LegalizeVectorType(EVT ResTy, SDValue Op,
   assert(ResTy.isVector());
   unsigned TargetWidth = (Subtarget->hasPackedMode() &&
                           ResTy.getVectorNumElements() > StandardVectorWidth)
-                             ? PackedWidth
+                             ? PackedVectorWidth
                              : StandardVectorWidth;
 
   // Use vXi1 as result type in native widening mode
@@ -1439,12 +1439,12 @@ VVPWideningInfo VETargetLowering::pickResultType(VECustomDAG &CDAG, SDValue Op,
 
     if ((ElemTy != MVT::i1 && ElemTy != MVT::i32 && ElemTy != MVT::f32 &&
          ElemTy != MVT::f64 && ElemTy != MVT::i64) ||
-        (OpVectorLength > PackedWidth)) {
+        (OpVectorLength > PackedVectorWidth)) {
       LLVM_DEBUG(dbgs() << "\tToNative: Over-sized data type\n";);
       return VVPWideningInfo();
     }
 
-    VectorWidth = PackedWidth;
+    VectorWidth = PackedVectorWidth;
   } else {
     VectorWidth = StandardVectorWidth;
   }

llvm/test/CodeGen/VE/Vector/mask_broadcast.ll

@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=ve-unknown-unknown -mattr=+vpu | FileCheck %s
+
+define fastcc <256 x i1> @brd_v256i1_s(i1 %s) {
+; CHECK-LABEL: brd_v256i1_s:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vbrd %v0, %s0
+; CHECK-NEXT:    vfmk.w.ne %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %val = insertelement <256 x i1> undef, i1 %s, i32 0
+  %ret = shufflevector <256 x i1> %val, <256 x i1> undef, <256 x i32> zeroinitializer
+  ret <256 x i1> %ret
+}
+
+define fastcc <256 x i1> @brd_v256i1_zero() {
+; CHECK-LABEL: brd_v256i1_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorm %vm1, %vm0, %vm0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %val = insertelement <256 x i1> undef, i1 0, i32 0
+  %ret = shufflevector <256 x i1> %val, <256 x i1> undef, <256 x i32> zeroinitializer
+  ret <256 x i1> %ret
+}
+
+define fastcc <256 x i1> @brd_v256i1_one() {
+; CHECK-LABEL: brd_v256i1_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andm %vm1, %vm0, %vm0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %val = insertelement <256 x i1> undef, i1 1, i32 0
+  %ret = shufflevector <256 x i1> %val, <256 x i1> undef, <256 x i32> zeroinitializer
+  ret <256 x i1> %ret
+}