Skip to content

Commit 07ce659

Browse files
gnzlbggnzlbg
committed
expose ordered/unordered/nanless intirnsics
1 parent 01cc5b3 commit 07ce659

File tree

5 files changed

+202
-229
lines changed

5 files changed

+202
-229
lines changed

‎src/librustc_trans/builder.rs

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -958,6 +958,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
958958
pub fn vector_reduce_fadd_fast(&self, acc: ValueRef, src: ValueRef) -> ValueRef {
959959
self.count_insn("vector.reduce.fadd_fast");
960960
unsafe {
961+
// FIXME: add a non-fast math version once
962+
// https://bugs.llvm.org/show_bug.cgi?id=36732
963+
// is fixed.
961964
let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
962965
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
963966
instr
@@ -966,6 +969,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
966969
pub fn vector_reduce_fmul_fast(&self, acc: ValueRef, src: ValueRef) -> ValueRef {
967970
self.count_insn("vector.reduce.fmul_fast");
968971
unsafe {
972+
// FIXME: add a non-fast math version once
973+
// https://bugs.llvm.org/show_bug.cgi?id=36732
974+
// is fixed.
969975
let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
970976
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
971977
instr
@@ -1001,6 +1007,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
10011007
llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src)
10021008
}
10031009
}
1010+
pub fn vector_reduce_fmin(&self, src: ValueRef) -> ValueRef {
1011+
self.count_insn("vector.reduce.fmin");
1012+
unsafe {
1013+
llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, true)
1014+
}
1015+
}
1016+
pub fn vector_reduce_fmax(&self, src: ValueRef) -> ValueRef {
1017+
self.count_insn("vector.reduce.fmax");
1018+
unsafe {
1019+
llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, true)
1020+
}
1021+
}
10041022
pub fn vector_reduce_fmin_fast(&self, src: ValueRef) -> ValueRef {
10051023
self.count_insn("vector.reduce.fmin_fast");
10061024
unsafe {

‎src/librustc_trans/intrinsic.rs

Lines changed: 112 additions & 188 deletions
Original file line numberDiff line numberDiff line change
@@ -1150,210 +1150,134 @@ fn generic_simd_intrinsic<'a, 'tcx>(
11501150
return Ok(bx.extract_element(args[0].immediate(), args[1].immediate()))
11511151
}
11521152

1153-
if name == "simd_reduce_add" {
1154-
require!(ret_ty == in_elem,
1155-
"expected return type `{}` (element of input `{}`), found `{}`",
1156-
in_elem, in_ty, ret_ty);
1157-
return match in_elem.sty {
1158-
ty::TyInt(_i) => {
1159-
Ok(bx.vector_reduce_add(args[0].immediate()))
1160-
},
1161-
ty::TyUint(_u) => {
1162-
Ok(bx.vector_reduce_add(args[0].immediate()))
1163-
},
1164-
ty::TyFloat(f) => {
1165-
// undef as accumulator makes the reduction unordered:
1166-
let acc = match f.bit_width() {
1167-
32 => C_undef(Type::f32(bx.cx)),
1168-
64 => C_undef(Type::f64(bx.cx)),
1169-
v => {
1170-
return_error!(
1171-
"unsupported {} from `{}` with element `{}` of size `{}` to `{}`",
1172-
"simd_reduce_add", in_ty, in_elem, v, ret_ty)
1153+
macro_rules! arith_red {
1154+
($name:tt : $integer_reduce:ident, $float_reduce:ident, $ordered:expr) => {
1155+
if name == $name {
1156+
require!(ret_ty == in_elem,
1157+
"expected return type `{}` (element of input `{}`), found `{}`",
1158+
in_elem, in_ty, ret_ty);
1159+
return match in_elem.sty {
1160+
ty::TyInt(_) | ty::TyUint(_) => {
1161+
let r = bx.$integer_reduce(args[0].immediate());
1162+
if $ordered {
1163+
// if overflow occurs, the result is the
1164+
// mathematical result modulo 2^n:
1165+
if name.contains("mul") {
1166+
Ok(bx.mul(args[1].immediate(), r))
1167+
} else {
1168+
Ok(bx.add(args[1].immediate(), r))
1169+
}
1170+
} else {
1171+
Ok(bx.$integer_reduce(args[0].immediate()))
1172+
}
1173+
},
1174+
ty::TyFloat(f) => {
1175+
// ordered arithmetic reductions take an accumulator
1176+
let acc = if $ordered {
1177+
args[1].immediate()
1178+
} else {
1179+
// unordered arithmetic reductions do not:
1180+
match f.bit_width() {
1181+
32 => C_undef(Type::f32(bx.cx)),
1182+
64 => C_undef(Type::f64(bx.cx)),
1183+
v => {
1184+
return_error!(
1185+
"unsupported {} from `{}` with element `{}` of size `{}` to `{}`",
1186+
$name, in_ty, in_elem, v, ret_ty
1187+
)
1188+
}
1189+
}
1190+
1191+
};
1192+
Ok(bx.$float_reduce(acc, args[0].immediate()))
11731193
}
1174-
};
1175-
Ok(bx.vector_reduce_fadd_fast(acc, args[0].immediate()))
1194+
_ => {
1195+
return_error!(
1196+
"unsupported {} from `{}` with element `{}` to `{}`",
1197+
$name, in_ty, in_elem, ret_ty
1198+
)
1199+
},
1200+
}
11761201
}
1177-
_ => {
1178-
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1179-
"simd_reduce_add", in_ty, in_elem, ret_ty)
1180-
},
11811202
}
11821203
}
11831204

1184-
if name == "simd_reduce_mul" {
1185-
require!(ret_ty == in_elem,
1186-
"expected return type `{}` (element of input `{}`), found `{}`",
1187-
in_elem, in_ty, ret_ty);
1188-
return match in_elem.sty {
1189-
ty::TyInt(_i) => {
1190-
Ok(bx.vector_reduce_mul(args[0].immediate()))
1191-
},
1192-
ty::TyUint(_u) => {
1193-
Ok(bx.vector_reduce_mul(args[0].immediate()))
1194-
},
1195-
ty::TyFloat(f) => {
1196-
// undef as accumulator makes the reduction unordered:
1197-
let acc = match f.bit_width() {
1198-
32 => C_undef(Type::f32(bx.cx)),
1199-
64 => C_undef(Type::f64(bx.cx)),
1200-
v => {
1201-
return_error!(
1202-
"unsupported {} from `{}` with element `{}` of size `{}` to `{}`",
1203-
"simd_reduce_mul", in_ty, in_elem, v, ret_ty)
1205+
arith_red!("simd_reduce_add_ordered": vector_reduce_add, vector_reduce_fadd_fast, true);
1206+
arith_red!("simd_reduce_mul_ordered": vector_reduce_mul, vector_reduce_fmul_fast, true);
1207+
arith_red!("simd_reduce_add_unordered": vector_reduce_add, vector_reduce_fadd_fast, false);
1208+
arith_red!("simd_reduce_mul_unordered": vector_reduce_mul, vector_reduce_fmul_fast, false);
1209+
1210+
macro_rules! minmax_red {
1211+
($name:tt: $int_red:ident, $float_red:ident) => {
1212+
if name == $name {
1213+
require!(ret_ty == in_elem,
1214+
"expected return type `{}` (element of input `{}`), found `{}`",
1215+
in_elem, in_ty, ret_ty);
1216+
return match in_elem.sty {
1217+
ty::TyInt(_i) => {
1218+
Ok(bx.$int_red(args[0].immediate(), true))
1219+
},
1220+
ty::TyUint(_u) => {
1221+
Ok(bx.$int_red(args[0].immediate(), false))
1222+
},
1223+
ty::TyFloat(_f) => {
1224+
Ok(bx.$float_red(args[0].immediate()))
12041225
}
1205-
};
1206-
Ok(bx.vector_reduce_fmul_fast(acc, args[0].immediate()))
1226+
_ => {
1227+
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1228+
$name, in_ty, in_elem, ret_ty)
1229+
},
1230+
}
12071231
}
1208-
_ => {
1209-
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1210-
"simd_reduce_mul", in_ty, in_elem, ret_ty)
1211-
},
1212-
}
1213-
}
12141232

1215-
if name == "simd_reduce_min" {
1216-
require!(ret_ty == in_elem,
1217-
"expected return type `{}` (element of input `{}`), found `{}`",
1218-
in_elem, in_ty, ret_ty);
1219-
return match in_elem.sty {
1220-
ty::TyInt(_i) => {
1221-
Ok(bx.vector_reduce_min(args[0].immediate(), true))
1222-
},
1223-
ty::TyUint(_u) => {
1224-
Ok(bx.vector_reduce_min(args[0].immediate(), false))
1225-
},
1226-
ty::TyFloat(_f) => {
1227-
Ok(bx.vector_reduce_fmin_fast(args[0].immediate()))
1228-
}
1229-
_ => {
1230-
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1231-
"simd_reduce_min", in_ty, in_elem, ret_ty)
1232-
},
12331233
}
12341234
}
12351235

1236-
if name == "simd_reduce_max" {
1237-
require!(ret_ty == in_elem,
1238-
"expected return type `{}` (element of input `{}`), found `{}`",
1239-
in_elem, in_ty, ret_ty);
1240-
return match in_elem.sty {
1241-
ty::TyInt(_i) => {
1242-
Ok(bx.vector_reduce_max(args[0].immediate(), true))
1243-
},
1244-
ty::TyUint(_u) => {
1245-
Ok(bx.vector_reduce_max(args[0].immediate(), false))
1246-
},
1247-
ty::TyFloat(_f) => {
1248-
Ok(bx.vector_reduce_fmax_fast(args[0].immediate()))
1249-
}
1250-
_ => {
1251-
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1252-
"simd_reduce_max", in_ty, in_elem, ret_ty)
1253-
},
1254-
}
1255-
}
1236+
minmax_red!("simd_reduce_min": vector_reduce_min, vector_reduce_fmin);
1237+
minmax_red!("simd_reduce_max": vector_reduce_max, vector_reduce_fmax);
12561238

1257-
if name == "simd_reduce_and" {
1258-
require!(ret_ty == in_elem,
1259-
"expected return type `{}` (element of input `{}`), found `{}`",
1260-
in_elem, in_ty, ret_ty);
1261-
return match in_elem.sty {
1262-
ty::TyInt(_i) => {
1263-
Ok(bx.vector_reduce_and(args[0].immediate()))
1264-
},
1265-
ty::TyUint(_u) => {
1266-
Ok(bx.vector_reduce_and(args[0].immediate()))
1267-
},
1268-
_ => {
1269-
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1270-
"simd_reduce_and", in_ty, in_elem, ret_ty)
1271-
},
1272-
}
1273-
}
1239+
minmax_red!("simd_reduce_min_nanless": vector_reduce_min, vector_reduce_fmin_fast);
1240+
minmax_red!("simd_reduce_max_nanless": vector_reduce_max, vector_reduce_fmax_fast);
12741241

1275-
if name == "simd_reduce_or" {
1276-
require!(ret_ty == in_elem,
1277-
"expected return type `{}` (element of input `{}`), found `{}`",
1278-
in_elem, in_ty, ret_ty);
1279-
return match in_elem.sty {
1280-
ty::TyInt(_i) => {
1281-
Ok(bx.vector_reduce_or(args[0].immediate()))
1282-
},
1283-
ty::TyUint(_u) => {
1284-
Ok(bx.vector_reduce_or(args[0].immediate()))
1285-
},
1286-
_ => {
1287-
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1288-
"simd_reduce_or", in_ty, in_elem, ret_ty)
1289-
},
1290-
}
1291-
}
1292-
1293-
if name == "simd_reduce_xor" {
1294-
require!(ret_ty == in_elem,
1295-
"expected return type `{}` (element of input `{}`), found `{}`",
1296-
in_elem, in_ty, ret_ty);
1297-
return match in_elem.sty {
1298-
ty::TyInt(_i) => {
1299-
Ok(bx.vector_reduce_xor(args[0].immediate()))
1300-
},
1301-
ty::TyUint(_u) => {
1302-
Ok(bx.vector_reduce_xor(args[0].immediate()))
1303-
},
1304-
_ => {
1305-
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1306-
"simd_reduce_xor", in_ty, in_elem, ret_ty)
1307-
},
1242+
macro_rules! bitwise_red {
1243+
($name:tt : $red:ident, $boolean:expr) => {
1244+
if name == $name {
1245+
let input = if !$boolean {
1246+
require!(ret_ty == in_elem,
1247+
"expected return type `{}` (element of input `{}`), found `{}`",
1248+
in_elem, in_ty, ret_ty);
1249+
args[0].immediate()
1250+
} else {
1251+
// boolean reductions operate on vectors of i1s:
1252+
let i1 = Type::i1(bx.cx);
1253+
let i1xn = Type::vector(&i1, in_len as u64);
1254+
bx.trunc(args[0].immediate(), i1xn)
1255+
};
1256+
return match in_elem.sty {
1257+
ty::TyInt(_) | ty::TyUint(_) => {
1258+
let r = bx.$red(input);
1259+
Ok(
1260+
if !$boolean {
1261+
r
1262+
} else {
1263+
bx.zext(r, Type::bool(bx.cx))
1264+
}
1265+
)
1266+
},
1267+
_ => {
1268+
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1269+
$name, in_ty, in_elem, ret_ty)
1270+
},
1271+
}
1272+
}
13081273
}
13091274
}
13101275

1311-
if name == "simd_reduce_all" {
1312-
//require!(ret_ty == in_elem,
1313-
// "expected return type `{}` (element of input `{}`), found `{}`",
1314-
// in_elem, in_ty, ret_ty);
1315-
let i1 = Type::i1(bx.cx);
1316-
let i1xn = Type::vector(&i1, in_len as u64);
1317-
let v = bx.trunc(args[0].immediate(), i1xn);
1318-
1319-
let red = match in_elem.sty {
1320-
ty::TyInt(_i) => {
1321-
bx.vector_reduce_and(v)
1322-
},
1323-
ty::TyUint(_u) => {
1324-
bx.vector_reduce_and(v)
1325-
},
1326-
_ => {
1327-
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1328-
"simd_reduce_and", in_ty, in_elem, ret_ty)
1329-
},
1330-
};
1331-
return Ok(bx.zext(red, Type::bool(bx.cx)));
1332-
}
1333-
1334-
if name == "simd_reduce_any" {
1335-
//require!(ret_ty == in_elem,
1336-
// "expected return type `{}` (element of input `{}`), found `{}`",
1337-
// in_elem, in_ty, ret_ty);
1338-
let i1 = Type::i1(bx.cx);
1339-
let i1xn = Type::vector(&i1, in_len as u64);
1340-
let v = bx.trunc(args[0].immediate(), i1xn);
1341-
1342-
let red = match in_elem.sty {
1343-
ty::TyInt(_i) => {
1344-
bx.vector_reduce_or(v)
1345-
},
1346-
ty::TyUint(_u) => {
1347-
bx.vector_reduce_or(v)
1348-
},
1349-
_ => {
1350-
return_error!("unsupported {} from `{}` with element `{}` to `{}`",
1351-
"simd_reduce_and", in_ty, in_elem, ret_ty)
1352-
},
1353-
};
1354-
return Ok(bx.zext(red, Type::bool(bx.cx)));
1355-
}
1356-
1276+
bitwise_red!("simd_reduce_and": vector_reduce_and, false);
1277+
bitwise_red!("simd_reduce_or": vector_reduce_or, false);
1278+
bitwise_red!("simd_reduce_xor": vector_reduce_xor, false);
1279+
bitwise_red!("simd_reduce_all": vector_reduce_and, true);
1280+
bitwise_red!("simd_reduce_any": vector_reduce_or, true);
13571281

13581282
if name == "simd_cast" {
13591283
require_simd!(ret_ty, "return");

‎src/librustc_typeck/check/intrinsic.rs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -362,9 +362,12 @@ pub fn check_platform_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
362362
"simd_extract" => (2, vec![param(0), tcx.types.u32], param(1)),
363363
"simd_cast" => (2, vec![param(0)], param(1)),
364364
"simd_reduce_all" | "simd_reduce_any" => (1, vec![param(0)], tcx.types.bool),
365-
"simd_reduce_add" | "simd_reduce_mul" |
365+
"simd_reduce_add_ordered" | "simd_reduce_mul_ordered"
366+
=> (2, vec![param(0), param(1)], param(1)),
367+
"simd_reduce_add_unordered" | "simd_reduce_mul_unordered" |
366368
"simd_reduce_and" | "simd_reduce_or" | "simd_reduce_xor" |
367-
"simd_reduce_min" | "simd_reduce_max"
369+
"simd_reduce_min" | "simd_reduce_max" |
370+
"simd_reduce_min_nanless" | "simd_reduce_max_nanless"
368371
=> (2, vec![param(0)], param(1)),
369372
name if name.starts_with("simd_shuffle") => {
370373
match name["simd_shuffle".len()..].parse() {

‎src/rustllvm/RustWrapper.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1397,6 +1397,7 @@ LLVMRustModuleCost(LLVMModuleRef M) {
13971397
}
13981398

13991399
// Vector reductions:
1400+
#if LLVM_VERSION_GE(6, 0)
14001401
extern "C" LLVMValueRef
14011402
LLVMRustBuildVectorReduceFAdd(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) {
14021403
return wrap(unwrap(B)->CreateFAddReduce(unwrap(Acc),unwrap(Src)));
@@ -1441,3 +1442,4 @@ extern "C" LLVMValueRef
14411442
LLVMRustBuildVectorReduceFMax(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) {
14421443
return wrap(unwrap(B)->CreateFPMaxReduce(unwrap(Src), NoNaN));
14431444
}
1445+
#endif

0 commit comments

Comments
 (0)