Match the inverse of m_AddOverflow

llvm/include/llvm/IR/PatternMatch.h

@@ -2595,6 +2595,70 @@ struct UAddWithOverflow_match {
   }
 };
 
+//===----------------------------------------------------------------------===//
+// Matchers for overflow check patterns: e.g. (a + b) u< a, (a ^ -1) <u b
+// Note that S might be matched to other instructions than AddInst.
+//
+
+template <typename LHS_t, typename RHS_t, typename Sum_t>
+struct UAddWithOverflowInv_match {
+  LHS_t L;
+  RHS_t R;
+  Sum_t S;
+
+  UAddWithOverflowInv_match(const LHS_t &L, const RHS_t &R, const Sum_t &S)
+      : L(L), R(R), S(S) {}
+
+  template <typename OpTy> bool match(OpTy *V) const {
+    Value *ICmpLHS, *ICmpRHS;
+    CmpPredicate Pred;
+    if (!m_ICmp(Pred, m_Value(ICmpLHS), m_Value(ICmpRHS)).match(V))
+      return false;
+
+    Value *AddLHS, *AddRHS;
+    auto AddExpr = m_Add(m_Value(AddLHS), m_Value(AddRHS));
+
+    // (a + b) u>= a, (a + b) u>= b
+    if (Pred == ICmpInst::ICMP_UGE)
+      if (AddExpr.match(ICmpLHS) && (ICmpRHS == AddLHS || ICmpRHS == AddRHS))
+        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS);
+
+    // a <=u (a + b), b <=u (a + b)
+    if (Pred == ICmpInst::ICMP_ULE)
+      if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS))
+        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
+
+    Value *Op1;
+    auto XorExpr = m_OneUse(m_Not(m_Value(Op1)));
+    // (~a) >= u b
+    if (Pred == ICmpInst::ICMP_UGE) {
+      if (XorExpr.match(ICmpLHS))
+        return L.match(Op1) && R.match(ICmpRHS) && S.match(ICmpLHS);
+    }
+    //  b <= u (~a)
+    if (Pred == ICmpInst::ICMP_ULE) {
+      if (XorExpr.match(ICmpRHS))
+        return L.match(Op1) && R.match(ICmpLHS) && S.match(ICmpRHS);
+    }
+
+    // Match special-case for increment-by-1.
+    if (Pred == ICmpInst::ICMP_NE) {
+      // (a + 1) != 0
+      // (1 + a) != 0
+      if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) &&
+          (m_One().match(AddLHS) || m_One().match(AddRHS)))
+        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS);
+      // 0 != (a + 1)
+      // 0 != (1 + a)
+      if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) &&
+          (m_One().match(AddLHS) || m_One().match(AddRHS)))
+        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
+    }
+
+    return false;
+  }
+};
+
 /// Match an icmp instruction checking for unsigned overflow on addition.
 ///
 /// S is matched to the addition whose result is being checked for overflow, and
@@ -2605,6 +2669,17 @@ m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S) {
   return UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>(L, R, S);
 }
 
+/// Match an icmp instruction checking for unsigned overflow on addition, but
+/// with the opposite check.
+///
+/// S is matched to the addition whose result is being checked for overflow, and
+/// L and R are matched to the LHS and RHS of S.
+template <typename LHS_t, typename RHS_t, typename Sum_t>
+UAddWithOverflowInv_match<LHS_t, RHS_t, Sum_t>
+m_UAddWithOverflowInv(const LHS_t &L, const RHS_t &R, const Sum_t &S) {
+  return UAddWithOverflowInv_match<LHS_t, RHS_t, Sum_t>(L, R, S);
+}
+
 template <typename Opnd_t> struct Argument_match {
   unsigned OpI;
   Opnd_t Val;

llvm/lib/CodeGen/CodeGenPrepare.cpp

@@ -470,7 +470,8 @@ class CodeGenPrepare {
 
   bool tryToSinkFreeOperands(Instruction *I);
   bool replaceMathCmpWithIntrinsic(BinaryOperator *BO, Value *Arg0, Value *Arg1,
-                                   CmpInst *Cmp, Intrinsic::ID IID);
+                                   CmpInst *Cmp, Intrinsic::ID IID,
+                                   bool NegateOverflow = false);
   bool optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT);
   bool optimizeURem(Instruction *Rem);
   bool combineToUSubWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
@@ -1552,7 +1553,8 @@ static bool isIVIncrement(const Value *V, const LoopInfo *LI) {
 bool CodeGenPrepare::replaceMathCmpWithIntrinsic(BinaryOperator *BO,
                                                  Value *Arg0, Value *Arg1,
                                                  CmpInst *Cmp,
-                                                 Intrinsic::ID IID) {
+                                                 Intrinsic::ID IID,
+                                                 bool NegateOverflow) {
   auto IsReplacableIVIncrement = [this, &Cmp](BinaryOperator *BO) {
     if (!isIVIncrement(BO, LI))
       return false;
@@ -1624,6 +1626,8 @@ bool CodeGenPrepare::replaceMathCmpWithIntrinsic(BinaryOperator *BO,
     assert(BO->hasOneUse() &&
            "Patterns with XOr should use the BO only in the compare");
   Value *OV = Builder.CreateExtractValue(MathOV, 1, "ov");
+  if (NegateOverflow)
+    OV = Builder.CreateNot(OV, "not");
   replaceAllUsesWith(Cmp, OV, FreshBBs, IsHugeFunc);
   Cmp->eraseFromParent();
   BO->eraseFromParent();
@@ -1660,20 +1664,63 @@ static bool matchUAddWithOverflowConstantEdgeCases(CmpInst *Cmp,
   return false;
 }
 
+/// Match special-case patterns that check for unsigned add overflow but inverts
+/// the add check
+static bool
+matchUAddWithOverflowConstantEdgeCasesInverted(CmpInst *Cmp,
+                                               BinaryOperator *&Add) {
+  // Add = add A, 1; Cmp = icmp ne A,-1 (overflow if A is max val)
+  // Add = add A,-1; Cmp = icmp eq A, 0 (overflow if A is non-zero)
+  Value *A = Cmp->getOperand(0), *B = Cmp->getOperand(1);
+
+  // We are not expecting non-canonical/degenerate code. Just bail out.
+  if (isa<Constant>(A))
+    return false;
+
+  ICmpInst::Predicate Pred = Cmp->getPredicate();
+  if (Pred == ICmpInst::ICMP_NE && match(B, m_AllOnes()))
+    B = ConstantInt::get(B->getType(), 1);
+  else if (Pred == ICmpInst::ICMP_EQ && match(B, m_ZeroInt()))
+    B = Constant::getAllOnesValue(B->getType());
+  else
+    return false;
+
+  // Check the users of the variable operand of the compare looking for an add
+  // with the adjusted constant.
+  for (User *U : A->users()) {
+    if (match(U, m_Add(m_Specific(A), m_Specific(B)))) {
+      Add = cast<BinaryOperator>(U);
+      return true;
+    }
+  }
+  return false;
+}
+
 /// Try to combine the compare into a call to the llvm.uadd.with.overflow
 /// intrinsic. Return true if any changes were made.
 bool CodeGenPrepare::combineToUAddWithOverflow(CmpInst *Cmp,
                                                ModifyDT &ModifiedDT) {
   bool EdgeCase = false;
   Value *A, *B;
   BinaryOperator *Add;
+  bool Negate = false;
   if (!match(Cmp, m_UAddWithOverflow(m_Value(A), m_Value(B), m_BinOp(Add)))) {
-    if (!matchUAddWithOverflowConstantEdgeCases(Cmp, Add))
-      return false;
-    // Set A and B in case we match matchUAddWithOverflowConstantEdgeCases.
-    A = Add->getOperand(0);
-    B = Add->getOperand(1);
-    EdgeCase = true;
+    if (matchUAddWithOverflowConstantEdgeCases(Cmp, Add)) {
+      // Set A and B in case we match matchUAddWithOverflowConstantEdgeCases.
+      A = Add->getOperand(0);
+      B = Add->getOperand(1);
+      EdgeCase = true;
+    } else {
+      Negate = true;
+      if (!match(Cmp,
+                 m_UAddWithOverflowInv(m_Value(A), m_Value(B), m_BinOp(Add)))) {
+        if (!matchUAddWithOverflowConstantEdgeCasesInverted(Cmp, Add))
+          return false;
+        A = Add->getOperand(0);
+        B = Add->getOperand(1);
+        EdgeCase = true;
+      }
+    }
   }
 
   if (!TLI->shouldFormOverflowOp(ISD::UADDO,
@@ -1688,7 +1735,7 @@ bool CodeGenPrepare::combineToUAddWithOverflow(CmpInst *Cmp,
     return false;
 
   if (!replaceMathCmpWithIntrinsic(Add, A, B, Cmp,
-                                   Intrinsic::uadd_with_overflow))
+                                   Intrinsic::uadd_with_overflow, Negate))
     return false;
 
   // Reset callers - do not crash by iterating over a dead instruction.
@@ -2218,10 +2265,10 @@ bool CodeGenPrepare::optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT) {
   if (sinkCmpExpression(Cmp, *TLI))
     return true;
 
-  if (combineToUAddWithOverflow(Cmp, ModifiedDT))
+  if (combineToUSubWithOverflow(Cmp, ModifiedDT))
     return true;
 
-  if (combineToUSubWithOverflow(Cmp, ModifiedDT))
+  if (combineToUAddWithOverflow(Cmp, ModifiedDT))
     return true;
 
   if (unfoldPowerOf2Test(Cmp))

llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp

@@ -7821,7 +7821,7 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
         isa<IntegerType>(X->getType())) {
       Value *Result;
       Constant *Overflow;
-      // m_UAddWithOverflow can match patterns that do not include  an explicit
+      // m_UAddWithOverflow can match patterns that do not include an explicit
       // "add" instruction, so check the opcode of the matched op.
       if (AddI->getOpcode() == Instruction::Add &&
           OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, X, Y, *AddI,
@@ -7832,6 +7832,25 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
       }
     }
 
+    if (match(&I, m_UAddWithOverflowInv(m_Value(X), m_Value(Y),
+                                        m_Instruction(AddI))) &&
+        isa<IntegerType>(X->getType())) {
+      Value *Result;
+      Constant *Overflow;
+      // m_UAddWithOverflowInv can match patterns that do not include an
+      // explicit "add" instruction, so check the opcode of the matched op.
+      if (AddI->getOpcode() == Instruction::Add &&
+          OptimizeOverflowCheck(Instruction::Add, /*Signed*/ false, X, Y, *AddI,
+                                Result, Overflow)) {
+        Overflow = Overflow->isNullValue()
+                       ? ConstantInt::getTrue(Overflow->getType())
+                       : ConstantInt::getFalse(Overflow->getType());
+        replaceInstUsesWith(*AddI, Result);
+        eraseInstFromFunction(*AddI);
+        return replaceInstUsesWith(I, Overflow);
+      }
+    }
+
     // (zext X) * (zext Y)  --> llvm.umul.with.overflow.
     if (match(Op0, m_NUWMul(m_ZExt(m_Value(X)), m_ZExt(m_Value(Y)))) &&
         match(Op1, m_APInt(C))) {

llvm/test/CodeGen/AArch64/cgp-usubo.ll

@@ -108,11 +108,9 @@ define i1 @usubo_ugt_constant_op1_i8(i8 %x, ptr %p) nounwind {
 define i1 @usubo_eq_constant1_op1_i32(i32 %x, ptr %p) nounwind {
 ; CHECK-LABEL: usubo_eq_constant1_op1_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    sub w9, w0, #1
-; CHECK-NEXT:    cset w8, eq
-; CHECK-NEXT:    str w9, [x1]
-; CHECK-NEXT:    mov w0, w8
+; CHECK-NEXT:    subs w8, w0, #1
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    str w8, [x1]
 ; CHECK-NEXT:    ret
   %s = add i32 %x, -1
   %ov = icmp eq i32 %x, 0

llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll

@@ -196,12 +196,12 @@ define i1 @test_conditional2(i32 %a, i32 %b, ptr %c) {
 ; CHECK-NEXT:    mov w22, #2 ; =0x2
 ; CHECK-NEXT:  LBB3_6: ; %for.cond
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    cbz w22, LBB3_9
+; CHECK-NEXT:    subs w22, w22, #1
+; CHECK-NEXT:    b.lo LBB3_9
 ; CHECK-NEXT:  ; %bb.7: ; %for.body
 ; CHECK-NEXT:    ; in Loop: Header=BB3_6 Depth=1
-; CHECK-NEXT:    sub w22, w22, #1
-; CHECK-NEXT:    orr w9, w21, w20
 ; CHECK-NEXT:    ldr w10, [x19, w22, sxtw #2]
+; CHECK-NEXT:    orr w9, w21, w20
 ; CHECK-NEXT:    cmp w9, w10
 ; CHECK-NEXT:    b.eq LBB3_6
 ; CHECK-NEXT:  ; %bb.8: ; %if.then
@@ -238,12 +238,12 @@ define i1 @test_conditional2(i32 %a, i32 %b, ptr %c) {
 ; OUTLINE-ATOMICS-NEXT:    cset w8, eq
 ; OUTLINE-ATOMICS-NEXT:  LBB3_1: ; %for.cond
 ; OUTLINE-ATOMICS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; OUTLINE-ATOMICS-NEXT:    cbz w22, LBB3_4
+; OUTLINE-ATOMICS-NEXT:    subs w22, w22, #1
+; OUTLINE-ATOMICS-NEXT:    b.lo LBB3_4
 ; OUTLINE-ATOMICS-NEXT:  ; %bb.2: ; %for.body
 ; OUTLINE-ATOMICS-NEXT:    ; in Loop: Header=BB3_1 Depth=1
-; OUTLINE-ATOMICS-NEXT:    sub w22, w22, #1
-; OUTLINE-ATOMICS-NEXT:    orr w9, w21, w20
 ; OUTLINE-ATOMICS-NEXT:    ldr w10, [x19, w22, sxtw #2]
+; OUTLINE-ATOMICS-NEXT:    orr w9, w21, w20
 ; OUTLINE-ATOMICS-NEXT:    cmp w9, w10
 ; OUTLINE-ATOMICS-NEXT:    b.eq LBB3_1
 ; OUTLINE-ATOMICS-NEXT:  ; %bb.3: ; %if.then

llvm/test/CodeGen/X86/lack-of-signed-truncation-check.ll

@@ -506,17 +506,15 @@ define i1 @add_ugecmp_bad_i16_i8_cmp(i16 %x, i16 %y) nounwind {
 define i1 @add_ugecmp_bad_i8_i16(i16 %x) nounwind {
 ; X86-LABEL: add_ugecmp_bad_i8_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    subl $-128, %eax
-; X86-NEXT:    cmpw $127, %ax
-; X86-NEXT:    seta %al
+; X86-NEXT:    movw $128, %ax
+; X86-NEXT:    addw {{[0-9]+}}(%esp), %ax
+; X86-NEXT:    setae %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: add_ugecmp_bad_i8_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    subl $-128, %edi
-; X64-NEXT:    cmpw $127, %di
-; X64-NEXT:    seta %al
+; X64-NEXT:    addw $128, %di
+; X64-NEXT:    setae %al
 ; X64-NEXT:    retq
   %tmp0 = add i16 %x, 128 ; 1U << (8-1)
   %tmp1 = icmp uge i16 %tmp0, 128 ; 1U << (8-1)

llvm/test/CodeGen/X86/setcc-combine.ll

@@ -664,8 +664,9 @@ define <4 x i32> @cmp_sge_not_with_vec(<4 x i32> %a, <4 x i32> %b) {
 define i64 @cmp_uge_not(i64 %a, i64 %b) {
 ; CHECK-LABEL: cmp_uge_not:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    notq %rsi
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    cmpq %rdi, %rsi
+; CHECK-NEXT:    addq %rdi, %rsi
 ; CHECK-NEXT:    adcq $-1, %rax
 ; CHECK-NEXT:    retq
   %na = xor i64 %a, -1
@@ -679,8 +680,8 @@ define i64 @cmp_uge_not_with_constant(i64 %a) {
 ; CHECK-LABEL: cmp_uge_not_with_constant:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    cmpq $-42, %rdi
-; CHECK-NEXT:    sbbq %rax, %rax
+; CHECK-NEXT:    addq $42, %rdi
+; CHECK-NEXT:    adcq $-1, %rax
 ; CHECK-NEXT:    retq
   %na = xor i64 %a, -1
   %c = icmp uge i64 %na, 42
@@ -850,8 +851,9 @@ define <4 x i32> @cmp_ult_not_with_vec(<4 x i32> %a, <4 x i32> %b) {
 define i64 @cmp_ule_not(i64 %a, i64 %b) {
 ; CHECK-LABEL: cmp_ule_not:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    notq %rdi
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    cmpq %rsi, %rdi
+; CHECK-NEXT:    addq %rsi, %rdi
 ; CHECK-NEXT:    adcq $-1, %rax
 ; CHECK-NEXT:    retq
   %na = xor i64 %a, -1
@@ -983,8 +985,9 @@ define <4 x i32> @cmp_ne_not_with_vec(<4 x i32> %a, <4 x i32> %b) {
 define i64 @cmp_uge_not_commute(i64 %b, i64 %a) {
 ; CHECK-LABEL: cmp_uge_not_commute:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    notq %rdi
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    cmpq %rsi, %rdi
+; CHECK-NEXT:    addq %rsi, %rdi
 ; CHECK-NEXT:    adcq $-1, %rax
 ; CHECK-NEXT:    retq
   %na = xor i64 %a, -1