Skip to content

Commit 896d900

Browse files
LU-JOHNLU-JOHN
authored
[AMDGPU] Create hi-half of 64-bit ashr with mov of -1 (#146569)
When performing a 64-bit sra of a negative value with a shift range from [32-63], create the hi-half with a move of -1. Alive verification: https://alive2.llvm.org/ce/z/kXd7Ac Also, preserve exact flag. Alive verification: https://alive2.llvm.org/ce/z/L86tXf. --------- Signed-off-by: John Lu <John.Lu@amd.com>
1 parent 23b4f4e commit 896d900

File tree

3 files changed

+242
-3
lines changed

3 files changed

+242
-3
lines changed

‎llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4223,10 +4223,17 @@ SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
42234223
SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
42244224
Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, LHSSL, TargetType, SplitLHS, One);
42254225
}
4226-
Hi = DAG.getFreeze(Hi);
42274226

4228-
SDValue HiShift = DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftFullAmt);
4229-
SDValue NewShift = DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftAmt);
4227+
KnownBits KnownLHS = DAG.computeKnownBits(LHS);
4228+
SDValue HiShift;
4229+
if (KnownLHS.isNegative()) {
4230+
HiShift = DAG.getAllOnesConstant(SL, TargetType);
4231+
} else {
4232+
Hi = DAG.getFreeze(Hi);
4233+
HiShift = DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftFullAmt);
4234+
}
4235+
SDValue NewShift =
4236+
DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftAmt, N->getFlags());
42304237

42314238
SDValue Vec;
42324239
if (VT.isVector()) {

‎llvm/test/CodeGen/AMDGPU/ashr64_reduce_flags.ll

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
2+
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -stop-after=finalize-isel -o - %s | FileCheck %s
3+
4+
;; Test that reduction of:
5+
;;
6+
;; DST = ashr i64 X, Y
7+
;;
8+
;; where Y is in the range [63-32] to:
9+
;;
10+
;; DST = [ashr i32 HI(X), (Y & 0x1F), ashr i32 HI(X), 31]
11+
;;
12+
;; preserves flags
13+
14+
define i64 @ashr_exact(i64 %arg0, i64 %shift_amt) {
15+
; CHECK-LABEL: name: ashr_exact
16+
; CHECK: bb.0 (%ir-block.0):
17+
; CHECK-NEXT: liveins: $vgpr1, $vgpr2
18+
; CHECK-NEXT: {{ $}}
19+
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr2
20+
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
21+
; CHECK-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
22+
; CHECK-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
23+
; CHECK-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
24+
; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
25+
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
26+
; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
27+
; CHECK-NEXT: [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
28+
; CHECK-NEXT: [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
29+
; CHECK-NEXT: [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
30+
; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[DEF3]]
31+
; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
32+
; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
33+
; CHECK-NEXT: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = exact V_ASHRREV_I32_e64 killed [[COPY5]], [[COPY3]], implicit $exec
34+
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 31
35+
; CHECK-NEXT: [[V_ASHRREV_I32_e64_1:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 killed [[S_MOV_B32_]], [[COPY3]], implicit $exec
36+
; CHECK-NEXT: $vgpr0 = COPY [[V_ASHRREV_I32_e64_]]
37+
; CHECK-NEXT: $vgpr1 = COPY [[V_ASHRREV_I32_e64_1]]
38+
; CHECK-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
39+
%or = or i64 %shift_amt, 32
40+
%ashr = ashr exact i64 %arg0, %or
41+
ret i64 %ashr
42+
}
43+

‎llvm/test/CodeGen/AMDGPU/neg_ashr64_reduce.ll

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
2+
; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck %s
3+
4+
; Test that negative 64-bit values shifted by [32-63] bits have
5+
; a hi-result created by moving an all-ones constant.
6+
7+
; FIXME: Range metadata is invalidated when i64 types are legalized to v2i32 types.
8+
; We could call performSraCombine before legalization, but other optimizations only work
9+
; with 64-bit sra.
10+
define i64 @scalar_ashr_metadata(ptr %arg0.ptr, ptr %arg1.ptr) {
11+
; CHECK-LABEL: scalar_ashr_metadata:
12+
; CHECK: ; %bb.0:
13+
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
14+
; CHECK-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
15+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
16+
; CHECK-NEXT: flat_load_dword v4, v[2:3]
17+
; CHECK-NEXT: ; kill: killed $vgpr0 killed $vgpr1
18+
; CHECK-NEXT: ; kill: killed $vgpr2 killed $vgpr3
19+
; CHECK-NEXT: v_ashrrev_i32_e32 v1, 31, v5
20+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
21+
; CHECK-NEXT: v_ashrrev_i32_e32 v0, v4, v5
22+
; CHECK-NEXT: s_setpc_b64 s[30:31]
23+
%val = load i64, ptr %arg0.ptr, !range !0, !noundef !{}
24+
%shift.amt = load i64, ptr %arg1.ptr, !range !1, !noundef !{}
25+
%ashr = ashr i64 %val, %shift.amt
26+
ret i64 %ashr
27+
}
28+
29+
define <2 x i64> @v2_ashr_metadata(ptr %arg0.ptr, ptr %arg1.ptr) {
30+
; CHECK-LABEL: v2_ashr_metadata:
31+
; CHECK: ; %bb.0:
32+
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
33+
; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
34+
; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[2:3]
35+
; CHECK-NEXT: v_mov_b32_e32 v1, -1
36+
; CHECK-NEXT: v_mov_b32_e32 v3, -1
37+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
38+
; CHECK-NEXT: v_ashrrev_i32_e32 v0, v8, v5
39+
; CHECK-NEXT: v_ashrrev_i32_e32 v2, v10, v7
40+
; CHECK-NEXT: s_setpc_b64 s[30:31]
41+
%val = load <2 x i64>, ptr %arg0.ptr, !range !2, !noundef !{}
42+
%shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !3, !noundef !{}
43+
%ashr = ashr <2 x i64> %val, %shift.amt
44+
ret <2 x i64> %ashr
45+
}
46+
47+
define <3 x i64> @v3_ashr_metadata(ptr %arg0.ptr, ptr %arg1.ptr) {
48+
; CHECK-LABEL: v3_ashr_metadata:
49+
; CHECK: ; %bb.0:
50+
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
51+
; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
52+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
53+
; CHECK-NEXT: flat_load_dword v4, v[0:1] offset:20
54+
; CHECK-NEXT: flat_load_dword v6, v[2:3] offset:16
55+
; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[2:3]
56+
; CHECK-NEXT: v_mov_b32_e32 v1, -1
57+
; CHECK-NEXT: v_mov_b32_e32 v3, -1
58+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
59+
; CHECK-NEXT: v_ashrrev_i32_e32 v4, v6, v4
60+
; CHECK-NEXT: v_ashrrev_i32_e32 v0, v8, v5
61+
; CHECK-NEXT: v_ashrrev_i32_e32 v2, v10, v7
62+
; CHECK-NEXT: v_mov_b32_e32 v5, -1
63+
; CHECK-NEXT: s_setpc_b64 s[30:31]
64+
%val = load <3 x i64>, ptr %arg0.ptr, !range !4, !noundef !{}
65+
%shift.amt = load <3 x i64>, ptr %arg1.ptr, !range !5, !noundef !{}
66+
%ashr = ashr <3 x i64> %val, %shift.amt
67+
ret <3 x i64> %ashr
68+
}
69+
70+
define <4 x i64> @v4_ashr_metadata(ptr %arg0.ptr, ptr %arg1.ptr) {
71+
; CHECK-LABEL: v4_ashr_metadata:
72+
; CHECK: ; %bb.0:
73+
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
74+
; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[2:3]
75+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
76+
; CHECK-NEXT: flat_load_dwordx4 v[7:10], v[0:1]
77+
; CHECK-NEXT: flat_load_dwordx4 v[11:14], v[0:1] offset:16
78+
; CHECK-NEXT: flat_load_dwordx4 v[15:18], v[2:3] offset:16
79+
; CHECK-NEXT: v_mov_b32_e32 v1, -1
80+
; CHECK-NEXT: v_mov_b32_e32 v3, -1
81+
; CHECK-NEXT: v_mov_b32_e32 v5, -1
82+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
83+
; CHECK-NEXT: v_mov_b32_e32 v7, -1
84+
; CHECK-NEXT: v_ashrrev_i32_e32 v0, v4, v8
85+
; CHECK-NEXT: v_ashrrev_i32_e32 v2, v6, v10
86+
; CHECK-NEXT: v_ashrrev_i32_e32 v4, v15, v12
87+
; CHECK-NEXT: v_ashrrev_i32_e32 v6, v17, v14
88+
; CHECK-NEXT: s_setpc_b64 s[30:31]
89+
%val = load <4 x i64>, ptr %arg0.ptr, !range !6, !noundef !{}
90+
%shift.amt = load <4 x i64>, ptr %arg1.ptr, !range !7, !noundef !{}
91+
%ashr = ashr <4 x i64> %val, %shift.amt
92+
ret <4 x i64> %ashr
93+
}
94+
95+
; Ranges used when transformation is valid
96+
!0 = !{i64 -6000000000, i64 0}
97+
!1 = !{i64 32, i64 64}
98+
!2 = !{i64 -7000000000, i64 -1000}
99+
!3 = !{i64 38, i64 64}
100+
!4 = !{i64 -8000000000, i64 -2001}
101+
!5 = !{i64 38, i64 60}
102+
!6 = !{i64 -9000000000, i64 -3002}
103+
!7 = !{i64 38, i64 50}
104+
105+
; Test that negative 64-bit values shifted by [2?-63] bits do NOT have
106+
; a hi-result created by moving an all-ones constant.
107+
108+
define i64 @no_transform_scalar_ashr_metadata(ptr %arg0.ptr, ptr %arg1.ptr) {
109+
; CHECK-LABEL: no_transform_scalar_ashr_metadata:
110+
; CHECK: ; %bb.0:
111+
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
112+
; CHECK-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
113+
; CHECK-NEXT: flat_load_dword v6, v[2:3]
114+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
115+
; CHECK-NEXT: v_ashrrev_i64 v[0:1], v6, v[4:5]
116+
; CHECK-NEXT: s_setpc_b64 s[30:31]
117+
%val = load i64, ptr %arg0.ptr, !range !8, !noundef !{}
118+
%shift.amt = load i64, ptr %arg1.ptr, !range !9, !noundef !{}
119+
%ashr = ashr i64 %val, %shift.amt
120+
ret i64 %ashr
121+
}
122+
123+
define <2 x i64> @no_transform_v2_ashr_metadata(ptr %arg0.ptr, ptr %arg1.ptr) {
124+
; CHECK-LABEL: no_transform_v2_ashr_metadata:
125+
; CHECK: ; %bb.0:
126+
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
127+
; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
128+
; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[2:3]
129+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
130+
; CHECK-NEXT: v_ashrrev_i64 v[0:1], v8, v[4:5]
131+
; CHECK-NEXT: v_ashrrev_i64 v[2:3], v10, v[6:7]
132+
; CHECK-NEXT: s_setpc_b64 s[30:31]
133+
%val = load <2 x i64>, ptr %arg0.ptr, !range !10, !noundef !{}
134+
%shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !11, !noundef !{}
135+
%ashr = ashr <2 x i64> %val, %shift.amt
136+
ret <2 x i64> %ashr
137+
}
138+
139+
define <3 x i64> @no_transform_v3_ashr_metadata(ptr %arg0.ptr, ptr %arg1.ptr) {
140+
; CHECK-LABEL: no_transform_v3_ashr_metadata:
141+
; CHECK: ; %bb.0:
142+
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
143+
; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[2:3]
144+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
145+
; CHECK-NEXT: flat_load_dwordx4 v[7:10], v[0:1]
146+
; CHECK-NEXT: flat_load_dwordx2 v[11:12], v[0:1] offset:16
147+
; CHECK-NEXT: flat_load_dword v5, v[2:3] offset:16
148+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
149+
; CHECK-NEXT: v_ashrrev_i64 v[0:1], v4, v[7:8]
150+
; CHECK-NEXT: v_ashrrev_i64 v[2:3], v6, v[9:10]
151+
; CHECK-NEXT: v_ashrrev_i64 v[4:5], v5, v[11:12]
152+
; CHECK-NEXT: s_setpc_b64 s[30:31]
153+
%val = load <3 x i64>, ptr %arg0.ptr, !range !12, !noundef !{}
154+
%shift.amt = load <3 x i64>, ptr %arg1.ptr, !range !13, !noundef !{}
155+
%ashr = ashr <3 x i64> %val, %shift.amt
156+
ret <3 x i64> %ashr
157+
}
158+
159+
define <4 x i64> @no_transform_v4_ashr_metadata(ptr %arg0.ptr, ptr %arg1.ptr) {
160+
; CHECK-LABEL: no_transform_v4_ashr_metadata:
161+
; CHECK: ; %bb.0:
162+
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
163+
; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[2:3]
164+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
165+
; CHECK-NEXT: flat_load_dwordx4 v[7:10], v[0:1]
166+
; CHECK-NEXT: flat_load_dwordx4 v[11:14], v[0:1] offset:16
167+
; CHECK-NEXT: flat_load_dwordx4 v[15:18], v[2:3] offset:16
168+
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
169+
; CHECK-NEXT: v_ashrrev_i64 v[0:1], v4, v[7:8]
170+
; CHECK-NEXT: v_ashrrev_i64 v[2:3], v6, v[9:10]
171+
; CHECK-NEXT: v_ashrrev_i64 v[4:5], v15, v[11:12]
172+
; CHECK-NEXT: v_ashrrev_i64 v[6:7], v17, v[13:14]
173+
; CHECK-NEXT: s_setpc_b64 s[30:31]
174+
%val = load <4 x i64>, ptr %arg0.ptr, !range !14, !noundef !{}
175+
%shift.amt = load <4 x i64>, ptr %arg1.ptr, !range !15, !noundef !{}
176+
%ashr = ashr <4 x i64> %val, %shift.amt
177+
ret <4 x i64> %ashr
178+
}
179+
180+
; Ranges used when transformation is invalid
181+
!8 = !{i64 -10000000000, i64 0}
182+
!9 = !{i64 29, i64 64}
183+
!10 = !{i64 -11000000000, i64 -1000}
184+
!11 = !{i64 28, i64 64}
185+
!12 = !{i64 -12000000000, i64 -2001}
186+
!13 = !{i64 27, i64 60}
187+
!14 = !{i64 -13000000000, i64 -3002}
188+
!15 = !{i64 26, i64 50}
189+

0 commit comments

Comments
 (0)