|
| 1 | +; RUN: opt %s -loop-vectorize -force-vector-interleave=3 -force-vector-width=4 -S | FileCheck --check-prefix=UF3 %s |
| 2 | +; RUN: opt %s -loop-vectorize -force-vector-interleave=5 -force-vector-width=4 -S | FileCheck --check-prefix=UF5 %s |
| 3 | + |
| 4 | +define i32 @reduction_sum(i64 %n, i32* noalias nocapture %A) { |
| 5 | +; UF3-LABEL: vector.body: |
| 6 | +; UF3-NEXT: [[IV:%.+]] = phi i64 [ 0, %vector.ph ], [ [[IV_NEXT:%.+]], %vector.body ] |
| 7 | +; UF3-NEXT: [[SUM0:%.+]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[SUM0_NEXT:%.+]], %vector.body ] |
| 8 | +; UF3-NEXT: [[SUM1:%.+]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[SUM1_NEXT:%.+]], %vector.body ] |
| 9 | +; UF3-NEXT: [[SUM2:%.+]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[SUM2_NEXT:%.+]], %vector.body ] |
| 10 | +; UF3-NEXT: [[IV0:%.+]] = add i64 [[IV]], 0 |
| 11 | +; UF3-NEXT: [[IV1:%.+]] = add i64 [[IV]], 4 |
| 12 | +; UF3-NEXT: [[IV2:%.+]] = add i64 [[IV]], 8 |
| 13 | +; UF3-NEXT: [[GEP0:%.+]] = getelementptr inbounds i32, i32* %A, i64 [[IV0]] |
| 14 | +; UF3-NEXT: [[GEP1:%.+]] = getelementptr inbounds i32, i32* %A, i64 [[IV1]] |
| 15 | +; UF3-NEXT: [[GEP2:%.+]] = getelementptr inbounds i32, i32* %A, i64 [[IV2]] |
| 16 | +; UF3-NEXT: [[L_GEP0:%.+]] = getelementptr inbounds i32, i32* [[GEP0]], i32 0 |
| 17 | +; UF3-NEXT: [[BC0:%.+]] = bitcast i32* [[L_GEP0]] to <4 x i32>* |
| 18 | +; UF3-NEXT: [[L0:%.+]] = load <4 x i32>, <4 x i32>* [[BC0]], align 4 |
| 19 | +; UF3-NEXT: [[L_GEP1:%.+]] = getelementptr inbounds i32, i32* [[GEP0]], i32 4 |
| 20 | +; UF3-NEXT: [[BC1:%.+]] = bitcast i32* [[L_GEP1]] to <4 x i32>* |
| 21 | +; UF3-NEXT: [[L1:%.+]] = load <4 x i32>, <4 x i32>* [[BC1]], align 4 |
| 22 | +; UF3-NEXT: [[L_GEP2:%.+]] = getelementptr inbounds i32, i32* [[GEP0]], i32 8 |
| 23 | +; UF3-NEXT: [[BC2:%.+]] = bitcast i32* [[L_GEP2]] to <4 x i32>* |
| 24 | +; UF3-NEXT: [[L2:%.+]] = load <4 x i32>, <4 x i32>* [[BC2]], align 4 |
| 25 | +; UF3-NEXT: [[SUM0_NEXT]] = add <4 x i32> [[SUM0]], [[L0]] |
| 26 | +; UF3-NEXT: [[SUM1_NEXT]] = add <4 x i32> [[SUM1]], [[L1]] |
| 27 | +; UF3-NEXT: [[SUM2_NEXT]] = add <4 x i32> [[SUM2]], [[L2]] |
| 28 | +; UF3-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 12 |
| 29 | +; UF3-NEXT: [[EC:%.+]] = icmp eq i64 [[IV_NEXT]], %n.vec |
| 30 | +; UF3-NEXT: br i1 [[EC]], label %middle.block, label %vector.body |
| 31 | +; |
| 32 | +; UF3-LABEL: middle.block: |
| 33 | +; UF3-NEXT: [[RDX0:%.+]] = add <4 x i32> [[SUM1_NEXT]], [[SUM0_NEXT]] |
| 34 | +; UF3-NEXT: [[RDX1:%.+]] = add <4 x i32> [[SUM2_NEXT]], [[RDX0]] |
| 35 | +; UF3-NEXT: call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[RDX1]]) |
| 36 | +; |
| 37 | + |
| 38 | +; UF5-LABEL: vector.body: |
| 39 | +; UF5-NEXT: [[IV:%.+]] = phi i64 [ 0, %vector.ph ], [ [[IV_NEXT:%.+]], %vector.body ] |
| 40 | +; UF5-NEXT: [[SUM0:%.+]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[SUM0_NEXT:%.+]], %vector.body ] |
| 41 | +; UF5-NEXT: [[SUM1:%.+]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[SUM1_NEXT:%.+]], %vector.body ] |
| 42 | +; UF5-NEXT: [[SUM2:%.+]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[SUM2_NEXT:%.+]], %vector.body ] |
| 43 | +; UF5-NEXT: [[SUM3:%.+]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[SUM3_NEXT:%.+]], %vector.body ] |
| 44 | +; UF5-NEXT: [[SUM4:%.+]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[SUM4_NEXT:%.+]], %vector.body ] |
| 45 | +; UF5-NEXT: [[IV0:%.+]] = add i64 [[IV]], 0 |
| 46 | +; UF5-NEXT: [[IV1:%.+]] = add i64 [[IV]], 4 |
| 47 | +; UF5-NEXT: [[IV2:%.+]] = add i64 [[IV]], 8 |
| 48 | +; UF5-NEXT: [[IV3:%.+]] = add i64 [[IV]], 12 |
| 49 | +; UF5-NEXT: [[IV4:%.+]] = add i64 [[IV]], 16 |
| 50 | +; UF5-NEXT: [[GEP0:%.+]] = getelementptr inbounds i32, i32* %A, i64 [[IV0]] |
| 51 | +; UF5-NEXT: [[GEP1:%.+]] = getelementptr inbounds i32, i32* %A, i64 [[IV1]] |
| 52 | +; UF5-NEXT: [[GEP2:%.+]] = getelementptr inbounds i32, i32* %A, i64 [[IV2]] |
| 53 | +; UF5-NEXT: [[GEP3:%.+]] = getelementptr inbounds i32, i32* %A, i64 [[IV3]] |
| 54 | +; UF5-NEXT: [[GEP4:%.+]] = getelementptr inbounds i32, i32* %A, i64 [[IV4]] |
| 55 | +; UF5-NEXT: [[L_GEP0:%.+]] = getelementptr inbounds i32, i32* [[GEP0]], i32 0 |
| 56 | +; UF5-NEXT: [[BC0:%.+]] = bitcast i32* [[L_GEP0]] to <4 x i32>* |
| 57 | +; UF5-NEXT: [[L0:%.+]] = load <4 x i32>, <4 x i32>* [[BC0]], align 4 |
| 58 | +; UF5-NEXT: [[L_GEP1:%.+]] = getelementptr inbounds i32, i32* [[GEP0]], i32 4 |
| 59 | +; UF5-NEXT: [[BC1:%.+]] = bitcast i32* [[L_GEP1]] to <4 x i32>* |
| 60 | +; UF5-NEXT: [[L1:%.+]] = load <4 x i32>, <4 x i32>* [[BC1]], align 4 |
| 61 | +; UF5-NEXT: [[L_GEP2:%.+]] = getelementptr inbounds i32, i32* [[GEP0]], i32 8 |
| 62 | +; UF5-NEXT: [[BC2:%.+]] = bitcast i32* [[L_GEP2]] to <4 x i32>* |
| 63 | +; UF5-NEXT: [[L2:%.+]] = load <4 x i32>, <4 x i32>* [[BC2]], align 4 |
| 64 | +; UF5-NEXT: [[L_GEP3:%.+]] = getelementptr inbounds i32, i32* [[GEP0]], i32 12 |
| 65 | +; UF5-NEXT: [[BC3:%.+]] = bitcast i32* [[L_GEP3]] to <4 x i32>* |
| 66 | +; UF5-NEXT: [[L3:%.+]] = load <4 x i32>, <4 x i32>* [[BC3]], align 4 |
| 67 | +; UF5-NEXT: [[L_GEP4:%.+]] = getelementptr inbounds i32, i32* [[GEP0]], i32 16 |
| 68 | +; UF5-NEXT: [[BC4:%.+]] = bitcast i32* [[L_GEP4]] to <4 x i32>* |
| 69 | +; UF5-NEXT: [[L4:%.+]] = load <4 x i32>, <4 x i32>* [[BC4]], align 4 |
| 70 | +; UF5-NEXT: [[SUM0_NEXT]] = add <4 x i32> [[SUM0]], [[L0]] |
| 71 | +; UF5-NEXT: [[SUM1_NEXT]] = add <4 x i32> [[SUM1]], [[L1]] |
| 72 | +; UF5-NEXT: [[SUM2_NEXT]] = add <4 x i32> [[SUM2]], [[L2]] |
| 73 | +; UF5-NEXT: [[SUM3_NEXT]] = add <4 x i32> [[SUM3]], [[L3]] |
| 74 | +; UF5-NEXT: [[SUM4_NEXT]] = add <4 x i32> [[SUM4]], [[L4]] |
| 75 | +; UF5-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 20 |
| 76 | +; UF5-NEXT: [[EC:%.+]] = icmp eq i64 [[IV_NEXT]], %n.vec |
| 77 | +; UF5-NEXT: br i1 [[EC]], label %middle.block, label %vector.body |
| 78 | +; |
| 79 | +; UF5-LABEL: middle.block: |
| 80 | +; UF5-NEXT: [[RDX0:%.+]] = add <4 x i32> [[SUM1_NEXT]], [[SUM0_NEXT]] |
| 81 | +; UF5-NEXT: [[RDX1:%.+]] = add <4 x i32> [[SUM2_NEXT]], [[RDX0]] |
| 82 | +; UF5-NEXT: [[RDX2:%.+]] = add <4 x i32> [[SUM3_NEXT]], [[RDX1]] |
| 83 | +; UF5-NEXT: [[RDX3:%.+]] = add <4 x i32> [[SUM4_NEXT]], [[RDX2]] |
| 84 | +; UF5-NEXT: call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[RDX3]]) |
| 85 | +; |
| 86 | + |
| 87 | +entry: |
| 88 | + br label %loop |
| 89 | + |
| 90 | +loop: |
| 91 | + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| 92 | + %sum.02 = phi i32 [ 0, %entry ], [ %sum.next, %loop ] |
| 93 | + %gep.A = getelementptr inbounds i32, i32* %A, i64 %iv |
| 94 | + %lv.A = load i32, i32* %gep.A, align 4 |
| 95 | + %sum.next = add i32 %sum.02, %lv.A |
| 96 | + %iv.next = add i64 %iv, 1 |
| 97 | + %exitcond = icmp eq i64 %iv, %n |
| 98 | + br i1 %exitcond, label %exit, label %loop |
| 99 | + |
| 100 | +exit: |
| 101 | + %sum.0.lcssa = phi i32 [ %sum.next, %loop ] |
| 102 | + ret i32 %sum.0.lcssa |
| 103 | +} |
0 commit comments