Skip to content

Commit 7cc8fe2

Browse files
paulwalker-armpaulwalker-arm
authored
[LLVM][AArch64] Relax SVE/SME codegen predicates. (#145322)
Code generation predicates like HasSVE2_or_SME implemented a strict divide between streaming and non-streaming which meant some SME instructions were not available unless a matching SVE feature was enabled.
1 parent 38200e9 commit 7cc8fe2

File tree

48 files changed

+127
-45
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+127
-45
lines changed

‎llvm/lib/Target/AArch64/AArch64.td

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,11 @@ include "AArch64SystemOperands.td"
5858

5959
class AArch64Unsupported { list<Predicate> F; }
6060

61-
let F = [HasSVE2p1, HasSVE2p1_or_SME2, HasSVE2p1_or_SME2p1] in
61+
let F = [HasSVE2p1, HasSVE2p1_or_SME2, HasSVE2p1_or_StreamingSME2, HasSVE2p1_or_SME2p1] in
6262
def SVE2p1Unsupported : AArch64Unsupported;
6363

6464
def SVE2Unsupported : AArch64Unsupported {
65-
let F = !listconcat([HasSVE2, HasSVE2_or_SME, HasSVE2_or_SME2, HasSSVE_FP8FMA, HasSMEF8F16,
65+
let F = !listconcat([HasSVE2, HasSVE2_or_SME, HasNonStreamingSVE2_or_SME2, HasSSVE_FP8FMA, HasSMEF8F16,
6666
HasSMEF8F32, HasSVEAES, HasSVESHA3, HasSVE2SM4, HasSVEBitPerm,
6767
HasSVEB16B16],
6868
SVE2p1Unsupported.F);
@@ -85,9 +85,9 @@ def SME2p1Unsupported : AArch64Unsupported {
8585
}
8686

8787
def SME2Unsupported : AArch64Unsupported {
88-
let F = !listconcat([HasSME2, HasSVE2_or_SME2, HasSVE2p1_or_SME2, HasSSVE_FP8FMA,
88+
let F = !listconcat([HasSME2, HasNonStreamingSVE2_or_SME2, HasSVE2p1_or_SME2, HasSSVE_FP8FMA,
8989
HasSMEF8F16, HasSMEF8F32, HasSMEF16F16_or_SMEF8F16, HasSMEB16B16,
90-
HasNonStreamingSVE_or_SSVE_AES],
90+
HasNonStreamingSVE_or_SSVE_AES, HasSVE2p1_or_StreamingSME2],
9191
SME2p1Unsupported.F);
9292
}
9393

‎llvm/lib/Target/AArch64/AArch64InstrInfo.td

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ def HasNonStreamingSVE_or_SME2p1
254254
AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME2p1),
255255
"sve or sme2p1">;
256256
def HasNonStreamingSVE_or_SME2p2
257-
: Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE()) ||"
257+
: Predicate<"Subtarget->isSVEAvailable() ||"
258258
"(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())">,
259259
AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME2p2),
260260
"sve or sme2p2">;
@@ -269,32 +269,38 @@ def HasNonStreamingSVE_or_SSVE_BitPerm
269269
AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSSVE_BitPerm),
270270
"sve or ssve-bitperm">;
271271
def HasNonStreamingSVE_or_SSVE_FEXPA
272-
: Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE()) ||"
272+
: Predicate<"Subtarget->isSVEAvailable() ||"
273273
"(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_FEXPA())">,
274274
AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSSVE_FEXPA),
275275
"sve or ssve-fexpa">;
276276

277277
def HasSVE2_or_SME
278-
: Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
278+
: Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME())">,
279279
AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSME),
280280
"sve2 or sme">;
281-
def HasSVE2_or_SME2
282-
: Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
281+
def HasNonStreamingSVE2_or_SME2
282+
: Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2()) ||"
283+
"(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2())">,
283284
AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSME2),
284285
"sve2 or sme2">;
285286

286287
def HasSVE2p1_or_SME
287-
: Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
288+
: Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p1() || Subtarget->hasSME())">,
288289
AssemblerPredicateWithAll<(any_of FeatureSME, FeatureSVE2p1),
289290
"sme or sve2p1">;
290291
def HasSVE2p1_or_SME2
291-
: Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
292+
: Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p1() || Subtarget->hasSME2())">,
292293
AssemblerPredicateWithAll<(any_of FeatureSME2, FeatureSVE2p1),
293294
"sme2 or sve2p1">;
294295
def HasSVE2p1_or_SME2p1
295-
: Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2p1())">,
296+
: Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p1() || Subtarget->hasSME2p1())">,
296297
AssemblerPredicateWithAll<(any_of FeatureSME2p1, FeatureSVE2p1),
297298
"sme2p1 or sve2p1">;
299+
def HasSVE2p1_or_StreamingSME2
300+
: Predicate<"(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVE2p1()) ||"
301+
"(Subtarget->isStreaming() && Subtarget->hasSME2())">,
302+
AssemblerPredicateWithAll<(any_of FeatureSME2, FeatureSVE2p1),
303+
"sme2 or sve2p1">;
298304

299305
def HasSVE2p2_or_SME2p2
300306
: Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2())">,

‎llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -4183,18 +4183,29 @@ defm UDOT_ZZZ_HtoS : sve2p1_two_way_dot_vv<"udot", 0b1, int_aarch64_sve_udot_x2
41834183
defm SDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"sdot", 0b0, int_aarch64_sve_sdot_lane_x2>;
41844184
defm UDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"udot", 0b1, int_aarch64_sve_udot_lane_x2>;
41854185

4186-
defm CNTP_XCI : sve2p1_pcount_pn<"cntp", 0b000>;
4187-
defm PEXT_PCI : sve2p1_pred_as_ctr_to_mask<"pext", int_aarch64_sve_pext>;
4188-
defm PEXT_2PCI : sve2p1_pred_as_ctr_to_mask_pair<"pext">;
4189-
defm PTRUE_C : sve2p1_ptrue_pn<"ptrue">;
4190-
41914186
defm SQCVTN_Z2Z_StoH : sve2p1_multi_vec_extract_narrow<"sqcvtn", 0b00, int_aarch64_sve_sqcvtn_x2>;
41924187
defm UQCVTN_Z2Z_StoH : sve2p1_multi_vec_extract_narrow<"uqcvtn", 0b01, int_aarch64_sve_uqcvtn_x2>;
41934188
defm SQCVTUN_Z2Z_StoH : sve2p1_multi_vec_extract_narrow<"sqcvtun", 0b10, int_aarch64_sve_sqcvtun_x2>;
41944189
defm SQRSHRN_Z2ZI_StoH : sve2p1_multi_vec_shift_narrow<"sqrshrn", 0b101, int_aarch64_sve_sqrshrn_x2>;
41954190
defm UQRSHRN_Z2ZI_StoH : sve2p1_multi_vec_shift_narrow<"uqrshrn", 0b111, int_aarch64_sve_uqrshrn_x2>;
41964191
defm SQRSHRUN_Z2ZI_StoH : sve2p1_multi_vec_shift_narrow<"sqrshrun", 0b001, int_aarch64_sve_sqrshrun_x2>;
41974192

4193+
defm WHILEGE_2PXX : sve2p1_int_while_rr_pair<"whilege", 0b000>;
4194+
defm WHILEGT_2PXX : sve2p1_int_while_rr_pair<"whilegt", 0b001>;
4195+
defm WHILELT_2PXX : sve2p1_int_while_rr_pair<"whilelt", 0b010>;
4196+
defm WHILELE_2PXX : sve2p1_int_while_rr_pair<"whilele", 0b011>;
4197+
defm WHILEHS_2PXX : sve2p1_int_while_rr_pair<"whilehs", 0b100>;
4198+
defm WHILEHI_2PXX : sve2p1_int_while_rr_pair<"whilehi", 0b101>;
4199+
defm WHILELO_2PXX : sve2p1_int_while_rr_pair<"whilelo", 0b110>;
4200+
defm WHILELS_2PXX : sve2p1_int_while_rr_pair<"whilels", 0b111>;
4201+
} // End HasSVE2p1_or_SME2
4202+
4203+
let Predicates = [HasSVE2p1_or_StreamingSME2] in {
4204+
defm CNTP_XCI : sve2p1_pcount_pn<"cntp", 0b000>;
4205+
defm PEXT_PCI : sve2p1_pred_as_ctr_to_mask<"pext", int_aarch64_sve_pext>;
4206+
defm PEXT_2PCI : sve2p1_pred_as_ctr_to_mask_pair<"pext">;
4207+
defm PTRUE_C : sve2p1_ptrue_pn<"ptrue">;
4208+
41984209
// Load to two registers
41994210
defm LD1B_2Z : sve2p1_mem_cld_ss_2z<"ld1b", 0b00, 0b0, ZZ_b_mul_r, GPR64shifted8, ZZ_b_strided_and_contiguous>;
42004211
defm LD1H_2Z : sve2p1_mem_cld_ss_2z<"ld1h", 0b01, 0b0, ZZ_h_mul_r, GPR64shifted16, ZZ_h_strided_and_contiguous>;
@@ -4318,14 +4329,6 @@ defm : store_pn_x4<nxv8bf16, int_aarch64_sve_stnt1_pn_x4, STNT1H_4Z_IMM>;
43184329
defm : store_pn_x4<nxv4f32, int_aarch64_sve_stnt1_pn_x4, STNT1W_4Z_IMM>;
43194330
defm : store_pn_x4<nxv2f64, int_aarch64_sve_stnt1_pn_x4, STNT1D_4Z_IMM>;
43204331

4321-
defm WHILEGE_2PXX : sve2p1_int_while_rr_pair<"whilege", 0b000>;
4322-
defm WHILEGT_2PXX : sve2p1_int_while_rr_pair<"whilegt", 0b001>;
4323-
defm WHILELT_2PXX : sve2p1_int_while_rr_pair<"whilelt", 0b010>;
4324-
defm WHILELE_2PXX : sve2p1_int_while_rr_pair<"whilele", 0b011>;
4325-
defm WHILEHS_2PXX : sve2p1_int_while_rr_pair<"whilehs", 0b100>;
4326-
defm WHILEHI_2PXX : sve2p1_int_while_rr_pair<"whilehi", 0b101>;
4327-
defm WHILELO_2PXX : sve2p1_int_while_rr_pair<"whilelo", 0b110>;
4328-
defm WHILELS_2PXX : sve2p1_int_while_rr_pair<"whilels", 0b111>;
43294332
defm WHILEGE_CXX : sve2p1_int_while_rr_pn<"whilege", 0b000>;
43304333
defm WHILEGT_CXX : sve2p1_int_while_rr_pn<"whilegt", 0b001>;
43314334
defm WHILELT_CXX : sve2p1_int_while_rr_pn<"whilelt", 0b010>;
@@ -4334,7 +4337,7 @@ defm WHILEHS_CXX : sve2p1_int_while_rr_pn<"whilehs", 0b100>;
43344337
defm WHILEHI_CXX : sve2p1_int_while_rr_pn<"whilehi", 0b101>;
43354338
defm WHILELO_CXX : sve2p1_int_while_rr_pn<"whilelo", 0b110>;
43364339
defm WHILELS_CXX : sve2p1_int_while_rr_pn<"whilels", 0b111>;
4337-
} // End HasSVE2p1_or_SME2
4340+
} // End HasSVE2p1_or_StreamingSME2
43384341

43394342
let Predicates = [HasSVE_or_SME] in {
43404343

@@ -4539,7 +4542,7 @@ let Predicates = [HasNonStreamingSVE2p2_or_SME2p2] in {
45394542
//===----------------------------------------------------------------------===//
45404543
// SVE2 FP8 instructions
45414544
//===----------------------------------------------------------------------===//
4542-
let Predicates = [HasSVE2_or_SME2, HasFP8] in {
4545+
let Predicates = [HasNonStreamingSVE2_or_SME2, HasFP8] in {
45434546
// FP8 upconvert
45444547
defm F1CVT_ZZ : sve2_fp8_cvt_single<0b0, 0b00, "f1cvt", nxv8f16, int_aarch64_sve_fp8_cvt1>;
45454548
defm F2CVT_ZZ : sve2_fp8_cvt_single<0b0, 0b01, "f2cvt", nxv8f16, int_aarch64_sve_fp8_cvt2>;
@@ -4556,15 +4559,15 @@ defm FCVTNB_Z2Z_StoB : sve2_fp8_down_cvt_single<0b01, "fcvtnb", ZZ_s_mul_r, nxv4
45564559
defm BFCVTN_Z2Z_HtoB : sve2_fp8_down_cvt_single<0b10, "bfcvtn", ZZ_h_mul_r, nxv8bf16, int_aarch64_sve_fp8_cvtn>;
45574560

45584561
defm FCVTNT_Z2Z_StoB : sve2_fp8_down_cvt_single_top<0b11, "fcvtnt", ZZ_s_mul_r, nxv4f32, int_aarch64_sve_fp8_cvtnt>;
4559-
} // End HasSVE2_or_SME2, HasFP8
4562+
} // End HasNonStreamingSVE2_or_SME2, HasFP8
45604563

4561-
let Predicates = [HasSVE2_or_SME2, HasFAMINMAX] in {
4564+
let Predicates = [HasNonStreamingSVE2_or_SME2, HasFAMINMAX] in {
45624565
defm FAMIN_ZPmZ : sve_fp_2op_p_zds<0b1111, "famin", "FAMIN_ZPZZ", int_aarch64_sve_famin, DestructiveBinaryComm>;
45634566
defm FAMAX_ZPmZ : sve_fp_2op_p_zds<0b1110, "famax", "FAMAX_ZPZZ", int_aarch64_sve_famax, DestructiveBinaryComm>;
45644567

45654568
defm FAMAX_ZPZZ : sve_fp_bin_pred_hfd<AArch64famax_p>;
45664569
defm FAMIN_ZPZZ : sve_fp_bin_pred_hfd<AArch64famin_p>;
4567-
} // End HasSVE2_or_SME2, HasFAMINMAX
4570+
} // End HasNonStreamingSVE2_or_SME2, HasFAMINMAX
45684571

45694572
let Predicates = [HasSSVE_FP8FMA] in {
45704573
// FP8 Widening Multiply-Add Long - Indexed Group
@@ -4608,14 +4611,14 @@ defm FDOT_ZZZI_BtoS : sve2_fp8_dot_indexed_s<"fdot", int_aarch64_sve_fp8_fdot_la
46084611
defm FDOT_ZZZ_BtoS : sve_fp8_dot<0b1, ZPR32, "fdot", nxv4f32, int_aarch64_sve_fp8_fdot>;
46094612
}
46104613

4611-
let Predicates = [HasSVE2_or_SME2, HasLUT] in {
4614+
let Predicates = [HasNonStreamingSVE2_or_SME2, HasLUT] in {
46124615
// LUTI2
46134616
defm LUTI2_ZZZI : sve2_luti2_vector_index<"luti2">;
46144617
// LUTI4
46154618
defm LUTI4_ZZZI : sve2_luti4_vector_index<"luti4">;
46164619
// LUTI4 (two contiguous registers)
46174620
defm LUTI4_Z2ZZI : sve2_luti4_vector_vg2_index<"luti4">;
4618-
} // End HasSVE2_or_SME2, HasLUT
4621+
} // End HasNonStreamingSVE2_or_SME2, HasLUT
46194622

46204623
//===----------------------------------------------------------------------===//
46214624
// Checked Pointer Arithmetic (FEAT_CPA)

‎llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
22
; RUN: llc -mattr=+sve2,+fp8 < %s | FileCheck %s
3+
; RUN: llc -mattr=+sve,+sme2,+fp8 < %s | FileCheck %s
34
; RUN: llc -mattr=+sme2,+fp8 --force-streaming < %s | FileCheck %s
45

56
target triple = "aarch64-linux"

‎llvm/test/CodeGen/AArch64/fp8-sve-cvtn.ll

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
22
; RUN: llc -mattr=+sve2,+fp8 < %s | FileCheck %s
3+
; RUN: llc -mattr=+sve,+sme2,+fp8 < %s | FileCheck %s
34
; RUN: llc -mattr=+sme2,+fp8 --force-streaming < %s | FileCheck %s
45

56
target triple = "aarch64-linux"

‎llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
22
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
3-
; RUN: llc -mtriple=aarch64-linux-gnu -force-streaming -mattr=+ssve-fexpa < %s | FileCheck %s
3+
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+ssve-fexpa -force-streaming < %s | FileCheck %s
44

55
define <vscale x 8 x half> @fexpa_h(<vscale x 8 x i16> %a) {
66
; CHECK-LABEL: fexpa_h:
@@ -27,4 +27,4 @@ define <vscale x 2 x double> @fexpa_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
2727
; CHECK-NEXT: ret
2828
%out = call <vscale x 2 x double> @llvm.aarch64.sve.fexpa.x.nxv2f64(<vscale x 2 x i64> %a)
2929
ret <vscale x 2 x double> %out
30-
}
30+
}

‎llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
22
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
3+
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
34
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
45

56
; ADDHNB

‎llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
22
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
3+
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
34
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
45

56
;

‎llvm/test/CodeGen/AArch64/sve2-intrinsics-complex-dot.ll

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
22
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
3+
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
34
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
45

56
;

‎llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
22
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
3+
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
34
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
45

56
;

0 commit comments

Comments
 (0)