[SystemZ] Recognize carry/borrow computation

Generate code using the VECTOR ADD COMPUTE CARRY and
VECTOR SUBTRACT COMPUTE BORROW INDICATION instructions
to implement open-coded IR with those semantics.

Handles integer vector types as well as i128.

Fixes: llvm#129608

Author: uweigand

llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

@@ -7343,6 +7343,37 @@ SDValue SystemZTargetLowering::combineZERO_EXTEND(
       }
     }
   }
+  // Recognize patterns for VECTOR SUBTRACT COMPUTE BORROW INDICATION
+  // and VECTOR ADD COMPUTE CARRY for i128:
+  //   (zext (setcc_uge X Y)) --> (VSCBI X Y)
+  //   (zext (setcc_ule Y X)) --> (VSCBI X Y)
+  //   (zext (setcc_ult (add X Y) X/Y) -> (VACC X Y)
+  //   (zext (setcc_ugt X/Y (add X Y)) -> (VACC X Y)
+  // For vector types, these patterns are recognized in the .td file.
+  if (N0.getOpcode() == ISD::SETCC && isTypeLegal(VT) && VT == MVT::i128 &&
+      N0.getOperand(0).getValueType() == VT) {
+    SDValue Op0 = N0.getOperand(0);
+    SDValue Op1 = N0.getOperand(1);
+    const ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
+    switch (CC) {
+    case ISD::SETULE:
+      std::swap(Op0, Op1);
+      [[fallthrough]];
+    case ISD::SETUGE:
+      return DAG.getNode(SystemZISD::VSCBI, SDLoc(N0), VT, Op0, Op1);
+    case ISD::SETUGT:
+      std::swap(Op0, Op1);
+      [[fallthrough]];
+    case ISD::SETULT:
+      if (Op0->hasOneUse() && Op0->getOpcode() == ISD::ADD &&
+          (Op0->getOperand(0) == Op1 || Op0->getOperand(1) == Op1))
+        return DAG.getNode(SystemZISD::VACC, SDLoc(N0), VT, Op0->getOperand(0),
+                           Op0->getOperand(1));
+      break;
+    default:
+      break;
+    }
+  }
 
   return SDValue();
 }

llvm/lib/Target/SystemZ/SystemZInstrVector.td

@@ -1287,6 +1287,23 @@ let Predicates = [FeatureVectorEnhancements3] in {
             (VMNLQ VR128:$x, VR128:$y)>;
 }
 
+// Instantiate comparison patterns to recognize VACC/VSCBI for TYPE.
+multiclass IntegerComputeCarryOrBorrow<ValueType type,
+                                       Instruction vacc, Instruction vscbi> {
+  let Predicates = [FeatureVector] in {
+    def : Pat<(z_vzext1 (type (z_vicmphl VR128:$x, (add VR128:$x, VR128:$y)))),
+              (vacc VR128:$x, VR128:$y)>;
+    def : Pat<(z_vzext1 (type (z_vicmphl VR128:$y, (add VR128:$x, VR128:$y)))),
+              (vacc VR128:$x, VR128:$y)>;
+    def : Pat<(z_vzext1 (z_vnot (type (z_vicmphl VR128:$y, VR128:$x)))),
+              (vscbi VR128:$x, VR128:$y)>;
+  }
+}
+defm : IntegerComputeCarryOrBorrow<v16i8, VACCB, VSCBIB>;
+defm : IntegerComputeCarryOrBorrow<v8i16, VACCH, VSCBIH>;
+defm : IntegerComputeCarryOrBorrow<v4i32, VACCF, VSCBIF>;
+defm : IntegerComputeCarryOrBorrow<v2i64, VACCG, VSCBIG>;
+
 // Instantiate full-vector shifts.
 multiclass FullVectorShiftOps<SDPatternOperator shift,
                               Instruction sbit, Instruction sbyte> {

llvm/lib/Target/SystemZ/SystemZOperators.td

@@ -1058,6 +1058,13 @@ def z_vneg : PatFrag<(ops node:$x), (sub immAllZerosV, node:$x)>;
 // Bitwise negation on vectors.
 def z_vnot : PatFrag<(ops node:$x), (xor node:$x, immAllOnesV)>;
 
+// In-register element-wise zero extension from i1 on vectors.
+def vsplat_imm_eq_1 : PatFrag<(ops), (build_vector), [{
+  APInt Imm;
+  return ISD::isConstantSplatVector(N, Imm) && Imm == 1;
+}]>;
+def z_vzext1 : PatFrag<(ops node:$x), (and node:$x, vsplat_imm_eq_1)>;
+
 // Signed "integer greater than zero" on vectors.
 def z_vicmph_zero : PatFrag<(ops node:$x), (z_vicmph node:$x, immAllZerosV)>;
 

llvm/test/CodeGen/SystemZ/int-cmp-65.ll

@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test usage of VACC/VSCBI.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+define i128 @i128_subc_1(i128 %a, i128 %b) unnamed_addr {
+; CHECK-LABEL: i128_subc_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl %v0, 0(%r4), 3
+; CHECK-NEXT:    vl %v1, 0(%r3), 3
+; CHECK-NEXT:    vscbiq %v0, %v1, %v0
+; CHECK-NEXT:    vst %v0, 0(%r2), 3
+; CHECK-NEXT:    br %r14
+  %cmp = icmp uge i128 %a, %b
+  %ext = zext i1 %cmp to i128
+  ret i128 %ext
+}
+
+define i128 @i128_subc_2(i128 %a, i128 %b) unnamed_addr {
+; CHECK-LABEL: i128_subc_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl %v0, 0(%r3), 3
+; CHECK-NEXT:    vl %v1, 0(%r4), 3
+; CHECK-NEXT:    vscbiq %v0, %v1, %v0
+; CHECK-NEXT:    vst %v0, 0(%r2), 3
+; CHECK-NEXT:    br %r14
+  %cmp = icmp ule i128 %a, %b
+  %ext = zext i1 %cmp to i128
+  ret i128 %ext
+}
+
+define i128 @i128_addc_1(i128 %a, i128 %b) {
+; CHECK-LABEL: i128_addc_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl %v0, 0(%r4), 3
+; CHECK-NEXT:    vl %v1, 0(%r3), 3
+; CHECK-NEXT:    vaccq %v0, %v1, %v0
+; CHECK-NEXT:    vst %v0, 0(%r2), 3
+; CHECK-NEXT:    br %r14
+  %sum = add i128 %a, %b
+  %cmp = icmp ult i128 %sum, %a
+  %ext = zext i1 %cmp to i128
+  ret i128 %ext
+}
+
+define i128 @i128_addc_2(i128 %a, i128 %b) {
+; CHECK-LABEL: i128_addc_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl %v0, 0(%r4), 3
+; CHECK-NEXT:    vl %v1, 0(%r3), 3
+; CHECK-NEXT:    vaccq %v0, %v1, %v0
+; CHECK-NEXT:    vst %v0, 0(%r2), 3
+; CHECK-NEXT:    br %r14
+  %sum = add i128 %a, %b
+  %cmp = icmp ult i128 %sum, %b
+  %ext = zext i1 %cmp to i128
+  ret i128 %ext
+}
+
+define i128 @i128_addc_3(i128 %a, i128 %b) {
+; CHECK-LABEL: i128_addc_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl %v0, 0(%r4), 3
+; CHECK-NEXT:    vl %v1, 0(%r3), 3
+; CHECK-NEXT:    vaccq %v0, %v1, %v0
+; CHECK-NEXT:    vst %v0, 0(%r2), 3
+; CHECK-NEXT:    br %r14
+  %sum = add i128 %a, %b
+  %cmp = icmp ugt i128 %a, %sum
+  %ext = zext i1 %cmp to i128
+  ret i128 %ext
+}
+
+define i128 @i128_addc_4(i128 %a, i128 %b) {
+; CHECK-LABEL: i128_addc_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl %v0, 0(%r4), 3
+; CHECK-NEXT:    vl %v1, 0(%r3), 3
+; CHECK-NEXT:    vaccq %v0, %v1, %v0
+; CHECK-NEXT:    vst %v0, 0(%r2), 3
+; CHECK-NEXT:    br %r14
+  %sum = add i128 %a, %b
+  %cmp = icmp ugt i128 %b, %sum
+  %ext = zext i1 %cmp to i128
+  ret i128 %ext
+}
+