Skip to content

Commit c73288a

Browse files
guy-davidguy-david
committed
[AArch64] Keep floating-point conversion in SIMD
Stores can be issued faster if the result is kept in the SIMD/FP registers.
1 parent a7a7e95 commit c73288a

File tree

2 files changed

+161
-0
lines changed

2 files changed

+161
-0
lines changed

‎llvm/lib/Target/AArch64/AArch64InstrInfo.td

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6598,6 +6598,18 @@ def : Pat<(f64 (AArch64frsqrts (f64 FPR64:$Rn), (f64 FPR64:$Rm))),
65986598
def : Pat<(v2f64 (AArch64frsqrts (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm))),
65996599
(FRSQRTSv2f64 FPR128:$Rn, FPR128:$Rm)>;
66006600

6601+
let HasOneUse = 1 in {
6602+
def fp_to_uint_oneuse : PatFrag<(ops node:$src0), (fp_to_uint $src0)>;
6603+
def fp_to_sint_oneuse : PatFrag<(ops node:$src0), (fp_to_sint $src0)>;
6604+
}
6605+
6606+
class StoreMaybeAssertZext<PatFrag op> : PatFrags<(ops node:$val, node:$ptr),
6607+
[(op node:$val, node:$ptr),
6608+
(op (assertzext node:$val), node:$ptr)]>;
6609+
6610+
def truncstorei8_maybe_assertzext : StoreMaybeAssertZext<truncstorei8>;
6611+
def truncstorei16_maybe_assertzext : StoreMaybeAssertZext<truncstorei16>;
6612+
66016613
// Some float -> int -> float conversion patterns for which we want to keep the
66026614
// int values in FP registers using the corresponding NEON instructions to
66036615
// avoid more costly int <-> fp register transfers.
@@ -6632,6 +6644,38 @@ def : Pat<(f64 (sint_to_fp (i64 (vector_extract (v2i64 FPR128:$Rn), (i64 0))))),
66326644
def : Pat<(f64 (uint_to_fp (i64 (vector_extract (v2i64 FPR128:$Rn), (i64 0))))),
66336645
(UCVTFv1i64 (i64 (EXTRACT_SUBREG (v2i64 FPR128:$Rn), dsub)))>;
66346646

6647+
// float -> int conversion followed by a store should use the value in the first
6648+
// lane to avoid expensive fpr -> gpr transfers.
6649+
let AddedComplexity = 19 in {
6650+
// f32 -> i32
6651+
def : Pat<(store (i32 (fp_to_uint_oneuse f32:$src)), GPR64sp:$Rn),
6652+
(STRSui (FCVTZUv1i32 f32:$src), GPR64sp:$Rn, (i64 0))>;
6653+
def : Pat<(store (i32 (fp_to_sint_oneuse f32:$src)), GPR64sp:$Rn),
6654+
(STRSui (FCVTZSv1i32 f32:$src), GPR64sp:$Rn, (i64 0))>;
6655+
6656+
// f64 -> i64
6657+
def : Pat<(store (i64 (fp_to_uint_oneuse f64:$src)), GPR64sp:$Rn),
6658+
(STRDui (FCVTZUv1i64 f64:$src), GPR64sp:$Rn, (i64 0))>;
6659+
def : Pat<(store (i64 (fp_to_sint_oneuse f64:$src)), GPR64sp:$Rn),
6660+
(STRDui (FCVTZSv1i64 f64:$src), GPR64sp:$Rn, (i64 0))>;
6661+
6662+
// f32 -> i8
6663+
def : Pat<(truncstorei8_maybe_assertzext (i32 (fp_to_uint_oneuse (f32 FPR32:$src))), GPR64sp:$Rn),
6664+
(STRBui (aarch64mfp8 (EXTRACT_SUBREG (FCVTZUv1i32 (f32 FPR32:$src)), bsub)),
6665+
GPR64sp:$Rn, (i64 0))>;
6666+
def : Pat<(truncstorei8_maybe_assertzext (i32 (fp_to_sint_oneuse (f32 FPR32:$src))), GPR64sp:$Rn),
6667+
(STRBui (aarch64mfp8 (EXTRACT_SUBREG (FCVTZSv1i32 (f32 FPR32:$src)), bsub)),
6668+
GPR64sp:$Rn, (i64 0))>;
6669+
6670+
// f32 -> i16
6671+
def : Pat<(truncstorei16_maybe_assertzext (i32 (fp_to_uint_oneuse (f32 FPR32:$src))), GPR64sp:$Rn),
6672+
(STRHui (f16 (EXTRACT_SUBREG (FCVTZUv1i32 (f32 FPR32:$src)), hsub)),
6673+
GPR64sp:$Rn, (i64 0))>;
6674+
def : Pat<(truncstorei16_maybe_assertzext (i32 (fp_to_sint_oneuse (f32 FPR32:$src))), GPR64sp:$Rn),
6675+
(STRHui (f16 (EXTRACT_SUBREG (FCVTZSv1i32 (f32 FPR32:$src)), hsub)),
6676+
GPR64sp:$Rn, (i64 0))>;
6677+
}
6678+
66356679
// fp16: integer extraction from vector must be at least 32-bits to be legal.
66366680
// Actual extraction result is then an in-reg sign-extension of lower 16-bits.
66376681
let Predicates = [HasNEONandIsSME2p2StreamingSafe, HasFullFP16] in {

‎llvm/test/CodeGen/AArch64/store-float-conversion.ll

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
2+
; RUN: llc -verify-machineinstrs -mtriple=aarch64 < %s | FileCheck %s
3+
4+
define void @f32_to_u8(float %f, ptr %dst) {
5+
; CHECK-LABEL: f32_to_u8:
6+
; CHECK: // %bb.0: // %entry
7+
; CHECK-NEXT: fcvtzu s0, s0
8+
; CHECK-NEXT: str b0, [x0]
9+
; CHECK-NEXT: ret
10+
entry:
11+
%conv = fptoui float %f to i32
12+
%trunc = trunc i32 %conv to i8
13+
store i8 %trunc, ptr %dst
14+
ret void
15+
}
16+
17+
define void @f32_to_s8(float %f, ptr %dst) {
18+
; CHECK-LABEL: f32_to_s8:
19+
; CHECK: // %bb.0: // %entry
20+
; CHECK-NEXT: fcvtzs s0, s0
21+
; CHECK-NEXT: str b0, [x0]
22+
; CHECK-NEXT: ret
23+
entry:
24+
%conv = fptosi float %f to i32
25+
%trunc = trunc i32 %conv to i8
26+
store i8 %trunc, ptr %dst
27+
ret void
28+
}
29+
30+
define void @f32_to_u16(float %f, ptr %dst) {
31+
; CHECK-LABEL: f32_to_u16:
32+
; CHECK: // %bb.0: // %entry
33+
; CHECK-NEXT: fcvtzu s0, s0
34+
; CHECK-NEXT: str h0, [x0]
35+
; CHECK-NEXT: ret
36+
entry:
37+
%conv = fptoui float %f to i32
38+
%trunc = trunc i32 %conv to i16
39+
store i16 %trunc, ptr %dst
40+
ret void
41+
}
42+
43+
define void @f32_to_s16(float %f, ptr %dst) {
44+
; CHECK-LABEL: f32_to_s16:
45+
; CHECK: // %bb.0: // %entry
46+
; CHECK-NEXT: fcvtzs s0, s0
47+
; CHECK-NEXT: str h0, [x0]
48+
; CHECK-NEXT: ret
49+
entry:
50+
%conv = fptosi float %f to i32
51+
%trunc = trunc i32 %conv to i16
52+
store i16 %trunc, ptr %dst
53+
ret void
54+
}
55+
56+
define void @f32_to_u32(float %f, ptr %dst) {
57+
; CHECK-LABEL: f32_to_u32:
58+
; CHECK: // %bb.0: // %entry
59+
; CHECK-NEXT: fcvtzu s0, s0
60+
; CHECK-NEXT: str s0, [x0]
61+
; CHECK-NEXT: ret
62+
entry:
63+
%conv = fptoui float %f to i32
64+
store i32 %conv, ptr %dst
65+
ret void
66+
}
67+
68+
define void @f32_to_s32(float %f, ptr %dst) {
69+
; CHECK-LABEL: f32_to_s32:
70+
; CHECK: // %bb.0: // %entry
71+
; CHECK-NEXT: fcvtzs s0, s0
72+
; CHECK-NEXT: str s0, [x0]
73+
; CHECK-NEXT: ret
74+
entry:
75+
%conv = fptosi float %f to i32
76+
store i32 %conv, ptr %dst
77+
ret void
78+
}
79+
80+
define void @f64_to_u64(double %d, ptr %dst) {
81+
; CHECK-LABEL: f64_to_u64:
82+
; CHECK: // %bb.0: // %entry
83+
; CHECK-NEXT: fcvtzu d0, d0
84+
; CHECK-NEXT: str d0, [x0]
85+
; CHECK-NEXT: ret
86+
entry:
87+
%conv = fptoui double %d to i64
88+
store i64 %conv, ptr %dst
89+
ret void
90+
}
91+
92+
define void @f64_to_s64(double %d, ptr %dst) {
93+
; CHECK-LABEL: f64_to_s64:
94+
; CHECK: // %bb.0: // %entry
95+
; CHECK-NEXT: fcvtzs d0, d0
96+
; CHECK-NEXT: str d0, [x0]
97+
; CHECK-NEXT: ret
98+
entry:
99+
%conv = fptosi double %d to i64
100+
store i64 %conv, ptr %dst
101+
ret void
102+
}
103+
104+
define i32 @f32_to_i32_multiple_uses(float %f, ptr %dst) {
105+
; CHECK-LABEL: f32_to_i32_multiple_uses:
106+
; CHECK: // %bb.0: // %entry
107+
; CHECK-NEXT: fcvtzs w8, s0
108+
; CHECK-NEXT: mov x9, x0
109+
; CHECK-NEXT: mov w0, w8
110+
; CHECK-NEXT: strb w8, [x9]
111+
; CHECK-NEXT: ret
112+
entry:
113+
%conv = fptosi float %f to i32
114+
%trunc = trunc i32 %conv to i8
115+
store i8 %trunc, ptr %dst
116+
ret i32 %conv
117+
}

0 commit comments

Comments
 (0)