diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 73ba0f78e8053..72c03d12c4bc8 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -878,7 +878,11 @@ static Instruction *foldSetClearBits(SelectInst &Sel, // is a vector consisting of 0 and undefs. If a constant compared with x // is a scalar undefined value or undefined vector then an expression // should be already folded into a constant. -static Instruction *foldSelectZeroOrMul(SelectInst &SI, InstCombinerImpl &IC) { +// +// This also holds all operations such that Op(0) == 0 +// e.g. Shl, Umin, etc +static Instruction *foldSelectZeroOrFixedOp(SelectInst &SI, + InstCombinerImpl &IC) { auto *CondVal = SI.getCondition(); auto *TrueVal = SI.getTrueValue(); auto *FalseVal = SI.getFalseValue(); @@ -900,11 +904,18 @@ static Instruction *foldSelectZeroOrMul(SelectInst &SI, InstCombinerImpl &IC) { // non-zero elements that are masked by undef elements in the compare // constant. auto *TrueValC = dyn_cast(TrueVal); - if (TrueValC == nullptr || - !match(FalseVal, m_c_Mul(m_Specific(X), m_Value(Y))) || - !isa(FalseVal)) + if (TrueValC == nullptr || !isa(FalseVal)) return nullptr; + if (!(match(FalseVal, m_c_Mul(m_Specific(X), m_Value(Y))) || + match(FalseVal, m_c_And(m_Specific(X), m_Value(Y))) || + match(FalseVal, m_FShl(m_Specific(X), m_Specific(X), m_Value(Y))) || + match(FalseVal, m_FShr(m_Specific(X), m_Specific(X), m_Value(Y))) || + match(FalseVal, + m_c_Intrinsic(m_Specific(X), m_Value(Y))))) { + return nullptr; + } + auto *ZeroC = cast(cast(CondVal)->getOperand(1)); auto *MergedC = Constant::mergeUndefsWith(TrueValC, ZeroC); // If X is compared with 0 then TrueVal could be either zero or undef. @@ -916,7 +927,11 @@ static Instruction *foldSelectZeroOrMul(SelectInst &SI, InstCombinerImpl &IC) { auto *FalseValI = cast(FalseVal); auto *FrY = IC.InsertNewInstBefore(new FreezeInst(Y, Y->getName() + ".fr"), FalseValI->getIterator()); - IC.replaceOperand(*FalseValI, FalseValI->getOperand(0) == Y ? 0 : 1, FrY); + IC.replaceOperand(*FalseValI, + FalseValI->getOperand(0) == Y + ? 0 + : (FalseValI->getOperand(1) == Y ? 1 : 2), + FrY); return IC.replaceInstUsesWith(SI, FalseValI); } @@ -4104,7 +4119,7 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { return Add; if (Instruction *Or = foldSetClearBits(SI, Builder)) return Or; - if (Instruction *Mul = foldSelectZeroOrMul(SI, *this)) + if (Instruction *Mul = foldSelectZeroOrFixedOp(SI, *this)) return Mul; // Turn (select C, (op X, Y), (op X, Z)) -> (op X, (select C, Y, Z)) diff --git a/llvm/test/Transforms/InstCombine/icmp-select.ll b/llvm/test/Transforms/InstCombine/icmp-select.ll index a038731abbc48..c6c0ba385a6fd 100644 --- a/llvm/test/Transforms/InstCombine/icmp-select.ll +++ b/llvm/test/Transforms/InstCombine/icmp-select.ll @@ -248,10 +248,9 @@ define i1 @icmp_select_implied_cond_relational_off_by_one(i8 %x, i8 %y) { define i1 @umin_seq_comparison(i8 %x, i8 %y) { ; CHECK-LABEL: @umin_seq_comparison( -; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[X:%.*]], 0 -; CHECK-NEXT: [[CMP21:%.*]] = icmp ule i8 [[X]], [[Y:%.*]] -; CHECK-NEXT: [[CMP2:%.*]] = select i1 [[CMP1]], i1 true, i1 [[CMP21]] -; CHECK-NEXT: ret i1 [[CMP2]] +; CHECK-NEXT: [[Y:%.*]] = freeze i8 [[Y1:%.*]] +; CHECK-NEXT: [[CMP21:%.*]] = icmp ule i8 [[X:%.*]], [[Y]] +; CHECK-NEXT: ret i1 [[CMP21]] ; %min = call i8 @llvm.umin.i8(i8 %x, i8 %y) %cmp1 = icmp eq i8 %x, 0 diff --git a/llvm/test/Transforms/InstCombine/select-fixed-zero.ll b/llvm/test/Transforms/InstCombine/select-fixed-zero.ll new file mode 100644 index 0000000000000..fd1e4b460b0da --- /dev/null +++ b/llvm/test/Transforms/InstCombine/select-fixed-zero.ll @@ -0,0 +1,239 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -passes=instcombine < %s | FileCheck %s + +; (select (icmp x, 0, eq), 0, (umin x, y)) -> (umin x, y) +define i64 @umin_select(i64 %a, i64 %b) { +; FIXED-ZERO-LABEL: @umin_select( +; FIXED-ZERO-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; FIXED-ZERO-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[A:%.*]], i64 [[B_FR]]) +; FIXED-ZERO-NEXT: ret i64 [[UMIN]] +; +; CHECK-LABEL: @umin_select( +; CHECK-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[A:%.*]], i64 [[B_FR]]) +; CHECK-NEXT: ret i64 [[UMIN]] +; + %cond = icmp eq i64 %a, 0 + %umin = call i64 @llvm.umin.i64(i64 %a, i64 %b) + %select = select i1 %cond, i64 0, i64 %umin + ret i64 %select +} + +; (select (icmp x, 0, eq), 0, (mul x, y)) -> (mul x, y) +define i64 @mul_select(i64 %a, i64 %b) { +; FIXED-ZERO-LABEL: @mul_select( +; FIXED-ZERO-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; FIXED-ZERO-NEXT: [[MUL:%.*]] = mul i64 [[A:%.*]], [[B_FR]] +; FIXED-ZERO-NEXT: ret i64 [[MUL]] +; +; CHECK-LABEL: @mul_select( +; CHECK-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; CHECK-NEXT: [[MUL:%.*]] = mul i64 [[A:%.*]], [[B_FR]] +; CHECK-NEXT: ret i64 [[MUL]] +; + %cond = icmp eq i64 %a, 0 + %mul = mul i64 %a, %b + %select = select i1 %cond, i64 0, i64 %mul + ret i64 %select +} + +; (select (icmp x, 0, eq), 0, (shl x, y)) -> (shl x, y) +define i64 @shl_select(i64 %a, i64 %b) { +; FIXED-ZERO-LABEL: @shl_select( +; FIXED-ZERO-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; FIXED-ZERO-NEXT: [[SHL:%.*]] = shl i64 [[A:%.*]], [[B_FR]] +; FIXED-ZERO-NEXT: ret i64 [[SHL]] +; +; CHECK-LABEL: @shl_select( +; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[A:%.*]], 0 +; CHECK-NEXT: [[SHL:%.*]] = shl i64 [[A]], [[B_FR:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[COND]], i64 0, i64 [[SHL]] +; CHECK-NEXT: ret i64 [[SELECT]] +; + %cond = icmp eq i64 %a, 0 + %shl = shl i64 %a, %b + %select = select i1 %cond, i64 0, i64 %shl + ret i64 %select +} + +; (select (icmp x, 0, eq), 0, (and x, y)) -> (and x, y) +define i64 @and_select(i64 %a, i64 %b) { +; FIXED-ZERO-LABEL: @and_select( +; FIXED-ZERO-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; FIXED-ZERO-NEXT: [[AND:%.*]] = and i64 [[A:%.*]], [[B_FR]] +; FIXED-ZERO-NEXT: ret i64 [[AND]] +; +; CHECK-LABEL: @and_select( +; CHECK-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; CHECK-NEXT: [[AND:%.*]] = and i64 [[A:%.*]], [[B_FR]] +; CHECK-NEXT: ret i64 [[AND]] +; + %cond = icmp eq i64 %a, 0 + %and = and i64 %a, %b + %select = select i1 %cond, i64 0, i64 %and + ret i64 %select +} + +; (select (icmp x, 0, ne), (ashr x, y), 0) -> (ashr x, y) +define i64 @ashr_select(i64 %a, i64 %b) { +; FIXED-ZERO-LABEL: @ashr_select( +; FIXED-ZERO-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; FIXED-ZERO-NEXT: [[ASHR:%.*]] = ashr i64 [[A:%.*]], [[B_FR]] +; FIXED-ZERO-NEXT: ret i64 [[ASHR]] +; +; CHECK-LABEL: @ashr_select( +; CHECK-NEXT: [[COND_NOT:%.*]] = icmp eq i64 [[A:%.*]], 0 +; CHECK-NEXT: [[ASHR:%.*]] = ashr i64 [[A]], [[B_FR:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[COND_NOT]], i64 0, i64 [[ASHR]] +; CHECK-NEXT: ret i64 [[SELECT]] +; + %cond = icmp ne i64 0, %a + %ashr = ashr i64 %a, %b + %select = select i1 %cond, i64 %ashr, i64 0 + ret i64 %select +} + +; (select (icmp x, 0, ne), (lshr x, y), 0) -> (lshr x, y) +define i64 @lshr_select(i64 %a, i64 %b) { +; FIXED-ZERO-LABEL: @lshr_select( +; FIXED-ZERO-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; FIXED-ZERO-NEXT: [[LSHR:%.*]] = lshr i64 [[A:%.*]], [[B_FR]] +; FIXED-ZERO-NEXT: ret i64 [[LSHR]] +; +; CHECK-LABEL: @lshr_select( +; CHECK-NEXT: [[COND_NOT:%.*]] = icmp eq i64 [[A:%.*]], 0 +; CHECK-NEXT: [[LSHR:%.*]] = lshr i64 [[A]], [[B_FR:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[COND_NOT]], i64 0, i64 [[LSHR]] +; CHECK-NEXT: ret i64 [[SELECT]] +; + %cond = icmp ne i64 0, %a + %lshr = lshr i64 %a, %b + %select = select i1 %cond, i64 %lshr, i64 0 + ret i64 %select +} + +; (select (icmp x, 0, eq), 0, fshr(x, x, y)) -> fshr(x, x, y) +define i64 @fshr_select(i64 %a, i64 %b) { +; FIXED-ZERO-LABEL: @fshr_select( +; FIXED-ZERO-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; FIXED-ZERO-NEXT: [[FSHR:%.*]] = call i64 @llvm.fshr.i64(i64 [[A:%.*]], i64 [[A]], i64 [[B_FR]]) +; FIXED-ZERO-NEXT: ret i64 [[FSHR]] +; +; CHECK-LABEL: @fshr_select( +; CHECK-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; CHECK-NEXT: [[FSHR:%.*]] = call i64 @llvm.fshr.i64(i64 [[A:%.*]], i64 [[A]], i64 [[B_FR]]) +; CHECK-NEXT: ret i64 [[FSHR]] +; + %cond = icmp eq i64 %a, 0 + %fshr = call i64 @llvm.fshr.i64(i64 %a, i64 %a, i64 %b) + %select = select i1 %cond, i64 0, i64 %fshr + ret i64 %select +} + +; (select (icmp x, 0, eq), 0, (fshl x, x, y)) -> (fshl x, x, y) +define i64 @fshl_select(i64 %a, i64 %b) { +; FIXED-ZERO-LABEL: @fshl_select( +; FIXED-ZERO-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; FIXED-ZERO-NEXT: [[FSHL:%.*]] = call i64 @llvm.fshl.i64(i64 [[A:%.*]], i64 [[A]], i64 [[B_FR]]) +; FIXED-ZERO-NEXT: ret i64 [[FSHL]] +; +; CHECK-LABEL: @fshl_select( +; CHECK-NEXT: [[B_FR:%.*]] = freeze i64 [[B:%.*]] +; CHECK-NEXT: [[FSHL:%.*]] = call i64 @llvm.fshl.i64(i64 [[A:%.*]], i64 [[A]], i64 [[B_FR]]) +; CHECK-NEXT: ret i64 [[FSHL]] +; + %cond = icmp eq i64 %a, 0 + %fshl = call i64 @llvm.fshl.i64(i64 %a, i64 %a, i64 %b) + %select = select i1 %cond, i64 0, i64 %fshl + ret i64 %select +} + +; (select (icmp x, 0, eq), 0, (fshr x, z, y)) -> leave as is +define i64 @fshr_select_no_combine(i64 %a, i64 %b, i64 %c) { +; FIXED-ZERO-LABEL: @fshr_select_no_combine( +; FIXED-ZERO-NEXT: [[COND:%.*]] = icmp eq i64 [[A:%.*]], 0 +; FIXED-ZERO-NEXT: [[FSHR:%.*]] = call i64 @llvm.fshr.i64(i64 [[A]], i64 [[B:%.*]], i64 [[C:%.*]]) +; FIXED-ZERO-NEXT: [[SELECT:%.*]] = select i1 [[COND]], i64 0, i64 [[FSHR]] +; FIXED-ZERO-NEXT: ret i64 [[SELECT]] +; +; CHECK-LABEL: @fshr_select_no_combine( +; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[A:%.*]], 0 +; CHECK-NEXT: [[FSHR:%.*]] = call i64 @llvm.fshr.i64(i64 [[A]], i64 [[B:%.*]], i64 [[C:%.*]]) +; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[COND]], i64 0, i64 [[FSHR]] +; CHECK-NEXT: ret i64 [[SELECT]] +; + %cond = icmp eq i64 %a, 0 + %fshr = call i64 @llvm.fshr.i64(i64 %a, i64 %b, i64 %c) + %select = select i1 %cond, i64 0, i64 %fshr + ret i64 %select +} + +; (select (icmp x, 0, eq), 0, (sdiv x, y)) -> (sdiv x, y) +define i64 @sdiv_select(i64 %a, i64 %b) { +; FIXED-ZERO-LABEL: @sdiv_select( +; FIXED-ZERO-NEXT: [[B:%.*]] = freeze i64 [[B1:%.*]] +; FIXED-ZERO-NEXT: [[DIV:%.*]] = sdiv i64 [[A:%.*]], [[B]] +; FIXED-ZERO-NEXT: ret i64 [[DIV]] +; +; CHECK-LABEL: @sdiv_select( +; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[A:%.*]], 0 +; CHECK-NEXT: [[DIV:%.*]] = sdiv i64 [[A]], [[B_FR:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[COND]], i64 0, i64 [[DIV]] +; CHECK-NEXT: ret i64 [[SELECT]] +; + %cond = icmp eq i64 %a, 0 + %div = sdiv i64 %a, %b + %select = select i1 %cond, i64 0, i64 %div + ret i64 %select +} + +; (select (icmp x, 0, eq), 0, (udiv x, y)) -> (udiv x, y) +define i64 @udiv_select(i64 %a, i64 %b) { +; FIXED-ZERO-LABEL: @udiv_select( +; FIXED-ZERO-NEXT: [[B:%.*]] = freeze i64 [[B1:%.*]] +; FIXED-ZERO-NEXT: [[DIV:%.*]] = udiv i64 [[A:%.*]], [[B]] +; FIXED-ZERO-NEXT: ret i64 [[DIV]] +; +; CHECK-LABEL: @udiv_select( +; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[A:%.*]], 0 +; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[A]], [[B_FR:%.*]] +; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[COND]], i64 0, i64 [[DIV]] +; CHECK-NEXT: ret i64 [[SELECT]] +; + %cond = icmp eq i64 %a, 0 + %div = udiv i64 %a, %b + %select = select i1 %cond, i64 0, i64 %div + ret i64 %select +} + +; (select (icmp x, 0, eq), 0, (icmp x, 0, slt)) -> (icmp x, 0, slt) +define i1 @icmp_slt_select(i64 %a) { +; FIXED-ZERO-LABEL: @icmp_slt_select( +; FIXED-ZERO-NEXT: [[ICMP:%.*]] = icmp slt i64 [[A:%.*]], 0 +; FIXED-ZERO-NEXT: ret i1 [[ICMP]] +; +; CHECK-LABEL: @icmp_slt_select( +; CHECK-NEXT: [[ICMP:%.*]] = icmp slt i64 [[A:%.*]], 0 +; CHECK-NEXT: ret i1 [[ICMP]] +; + %cond = icmp eq i64 %a, 0 + %icmp = icmp slt i64 %a, 0 + %select = select i1 %cond, i1 0, i1 %icmp + ret i1 %select +} + +; (select (icmp x, 0, eq), 0, (sub 0, x)) -> (sub 0, x) +define i64 @sub_select(i64 %a) { +; FIXED-ZERO-LABEL: @sub_select( +; FIXED-ZERO-NEXT: [[SUB:%.*]] = sub i64 0, [[A:%.*]] +; FIXED-ZERO-NEXT: ret i64 [[SUB]] +; +; CHECK-LABEL: @sub_select( +; CHECK-NEXT: [[SUB:%.*]] = sub i64 0, [[A:%.*]] +; CHECK-NEXT: ret i64 [[SUB]] +; + %cond = icmp eq i64 %a, 0 + %sub = sub i64 0, %a + %select = select i1 %cond, i64 0, i64 %sub + ret i64 %select +} diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll index ef5874ffd46ad..1f9ee83536016 100644 --- a/llvm/test/Transforms/InstCombine/select.ll +++ b/llvm/test/Transforms/InstCombine/select.ll @@ -893,10 +893,9 @@ define i32 @test56(i16 %x) { define i32 @test57(i32 %x, i32 %y) { ; CHECK-LABEL: @test57( -; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]] -; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[X]], 0 -; CHECK-NEXT: [[DOTAND:%.*]] = select i1 [[TOBOOL]], i32 0, i32 [[AND]] -; CHECK-NEXT: ret i32 [[DOTAND]] +; CHECK-NEXT: [[Y:%.*]] = freeze i32 [[Y1:%.*]] +; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], [[Y]] +; CHECK-NEXT: ret i32 [[AND]] ; %and = and i32 %x, %y %tobool = icmp eq i32 %x, 0