Skip to content

Commit

Permalink
Auto merge of #84274 - nagisa:fix-simd, r=nikic
Browse files Browse the repository at this point in the history 
Don't set fast-math for the SIMD operations we set it for previously

Instead of `fast-math`. `fast-math` implies things like functions not
being able to accept as an argument or return as a result, say, `inf`
which made these functions confusingly named or behaving incorrectly,
depending on how you interpret it. It seems that the intended behaviour
was to set a `afn` flag instead. In doing so we also renamed the
intrinsics to say `_approx` so that it is clear these are not precision
oriented and the users can act accordingly.

Fixes #84268
  • Loading branch information
@bors
bors committed Apr 18, 2021
2 parents b021bee + 487e273 commit ef88434
Show file tree
Hide file tree
Showing 19 changed files with 154 additions and 131 deletions.
18 changes: 9 additions & 9 deletions compiler/rustc_codegen_llvm/src/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,39 +261,39 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}

fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
Expand Down Expand Up @@ -1242,14 +1242,14 @@ impl Builder<'a, 'll, 'tcx> {
pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
unsafe {
let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
Expand Down Expand Up @@ -1282,15 +1282,15 @@ impl Builder<'a, 'll, 'tcx> {
unsafe {
let instr =
llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value {
unsafe {
let instr =
llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true);
llvm::LLVMRustSetHasUnsafeAlgebra(instr);
llvm::LLVMRustSetFastMath(instr);
instr
}
}
Expand Down
40 changes: 19 additions & 21 deletions compiler/rustc_codegen_llvm/src/intrinsic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1053,50 +1053,48 @@ fn generic_simd_intrinsic(
let vec_ty = bx.type_vector(elem_ty, in_len);

let (intr_name, fn_ty) = match name {
sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
_ => return_error!("unrecognized intrinsic `{}`", name),
};

let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
let f = bx.declare_cfn(&llvm_name, llvm::UnnamedAddr::No, fn_ty);
let c = bx.call(f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
unsafe { llvm::LLVMRustSetHasUnsafeAlgebra(c) };
Ok(c)
}

if std::matches!(
name,
sym::simd_fsqrt
| sym::simd_fsin
| sym::simd_fcos
sym::simd_ceil
| sym::simd_fabs
| sym::simd_ceil
| sym::simd_floor
| sym::simd_round
| sym::simd_trunc
| sym::simd_fexp
| sym::simd_fcos
| sym::simd_fexp2
| sym::simd_fexp
| sym::simd_flog10
| sym::simd_flog2
| sym::simd_flog
| sym::simd_fpowi
| sym::simd_fpow
| sym::simd_floor
| sym::simd_fma
| sym::simd_fpow
| sym::simd_fpowi
| sym::simd_fsin
| sym::simd_fsqrt
| sym::simd_round
| sym::simd_trunc
) {
return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
}
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1354,7 +1354,7 @@ extern "C" {
pub fn LLVMBuildNeg(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
pub fn LLVMBuildFNeg(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
pub fn LLVMBuildNot(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
pub fn LLVMRustSetHasUnsafeAlgebra(Instr: &Value);
pub fn LLVMRustSetFastMath(Instr: &Value);

// Memory
pub fn LLVMBuildAlloca(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value;
Expand Down
6 changes: 4 additions & 2 deletions compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,8 +349,10 @@ extern "C" void LLVMRustRemoveFunctionAttributes(LLVMValueRef Fn,
F->setAttributes(PALNew);
}

// enable fpmath flag UnsafeAlgebra
extern "C" void LLVMRustSetHasUnsafeAlgebra(LLVMValueRef V) {
// Enable a fast-math flag
//
// https://llvm.org/docs/LangRef.html#fast-math-flags
extern "C" void LLVMRustSetFastMath(LLVMValueRef V) {
if (auto I = dyn_cast<Instruction>(unwrap<Value>(V))) {
I->setFast(true);
}
Expand Down
23 changes: 23 additions & 0 deletions src/test/codegen/issue-84268.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// compile-flags: -O --crate-type=rlib
#![feature(platform_intrinsics, repr_simd)]

extern "platform-intrinsic" {
fn simd_fabs<T>(x: T) -> T;
fn simd_eq<T, U>(x: T, y: T) -> U;
}

#[repr(simd)]
pub struct V([f32; 4]);

#[repr(simd)]
pub struct M([i32; 4]);

#[no_mangle]
// CHECK-LABEL: @is_infinite
pub fn is_infinite(v: V) -> M {
// CHECK: fabs
// CHECK: cmp oeq
unsafe {
simd_eq(simd_fabs(v), V([f32::INFINITY; 4]))
}
}
14 changes: 7 additions & 7 deletions src/test/codegen/simd-intrinsic/simd-intrinsic-float-abs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,28 @@ extern "platform-intrinsic" {
// CHECK-LABEL: @fabs_32x2
#[no_mangle]
pub unsafe fn fabs_32x2(a: f32x2) -> f32x2 {
// CHECK: call fast <2 x float> @llvm.fabs.v2f32
// CHECK: call <2 x float> @llvm.fabs.v2f32
simd_fabs(a)
}

// CHECK-LABEL: @fabs_32x4
#[no_mangle]
pub unsafe fn fabs_32x4(a: f32x4) -> f32x4 {
// CHECK: call fast <4 x float> @llvm.fabs.v4f32
// CHECK: call <4 x float> @llvm.fabs.v4f32
simd_fabs(a)
}

// CHECK-LABEL: @fabs_32x8
#[no_mangle]
pub unsafe fn fabs_32x8(a: f32x8) -> f32x8 {
// CHECK: call fast <8 x float> @llvm.fabs.v8f32
// CHECK: call <8 x float> @llvm.fabs.v8f32
simd_fabs(a)
}

// CHECK-LABEL: @fabs_32x16
#[no_mangle]
pub unsafe fn fabs_32x16(a: f32x16) -> f32x16 {
// CHECK: call fast <16 x float> @llvm.fabs.v16f32
// CHECK: call <16 x float> @llvm.fabs.v16f32
simd_fabs(a)
}

Expand All @@ -73,20 +73,20 @@ pub struct f64x8(pub f64, pub f64, pub f64, pub f64,
// CHECK-LABEL: @fabs_64x4
#[no_mangle]
pub unsafe fn fabs_64x4(a: f64x4) -> f64x4 {
// CHECK: call fast <4 x double> @llvm.fabs.v4f64
// CHECK: call <4 x double> @llvm.fabs.v4f64
simd_fabs(a)
}

// CHECK-LABEL: @fabs_64x2
#[no_mangle]
pub unsafe fn fabs_64x2(a: f64x2) -> f64x2 {
// CHECK: call fast <2 x double> @llvm.fabs.v2f64
// CHECK: call <2 x double> @llvm.fabs.v2f64
simd_fabs(a)
}

// CHECK-LABEL: @fabs_64x8
#[no_mangle]
pub unsafe fn fabs_64x8(a: f64x8) -> f64x8 {
// CHECK: call fast <8 x double> @llvm.fabs.v8f64
// CHECK: call <8 x double> @llvm.fabs.v8f64
simd_fabs(a)
}
14 changes: 7 additions & 7 deletions src/test/codegen/simd-intrinsic/simd-intrinsic-float-ceil.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,28 @@ extern "platform-intrinsic" {
// CHECK-LABEL: @ceil_32x2
#[no_mangle]
pub unsafe fn ceil_32x2(a: f32x2) -> f32x2 {
// CHECK: call fast <2 x float> @llvm.ceil.v2f32
// CHECK: call <2 x float> @llvm.ceil.v2f32
simd_ceil(a)
}

// CHECK-LABEL: @ceil_32x4
#[no_mangle]
pub unsafe fn ceil_32x4(a: f32x4) -> f32x4 {
// CHECK: call fast <4 x float> @llvm.ceil.v4f32
// CHECK: call <4 x float> @llvm.ceil.v4f32
simd_ceil(a)
}

// CHECK-LABEL: @ceil_32x8
#[no_mangle]
pub unsafe fn ceil_32x8(a: f32x8) -> f32x8 {
// CHECK: call fast <8 x float> @llvm.ceil.v8f32
// CHECK: call <8 x float> @llvm.ceil.v8f32
simd_ceil(a)
}

// CHECK-LABEL: @ceil_32x16
#[no_mangle]
pub unsafe fn ceil_32x16(a: f32x16) -> f32x16 {
// CHECK: call fast <16 x float> @llvm.ceil.v16f32
// CHECK: call <16 x float> @llvm.ceil.v16f32
simd_ceil(a)
}

Expand All @@ -73,20 +73,20 @@ pub struct f64x8(pub f64, pub f64, pub f64, pub f64,
// CHECK-LABEL: @ceil_64x4
#[no_mangle]
pub unsafe fn ceil_64x4(a: f64x4) -> f64x4 {
// CHECK: call fast <4 x double> @llvm.ceil.v4f64
// CHECK: call <4 x double> @llvm.ceil.v4f64
simd_ceil(a)
}

// CHECK-LABEL: @ceil_64x2
#[no_mangle]
pub unsafe fn ceil_64x2(a: f64x2) -> f64x2 {
// CHECK: call fast <2 x double> @llvm.ceil.v2f64
// CHECK: call <2 x double> @llvm.ceil.v2f64
simd_ceil(a)
}

// CHECK-LABEL: @ceil_64x8
#[no_mangle]
pub unsafe fn ceil_64x8(a: f64x8) -> f64x8 {
// CHECK: call fast <8 x double> @llvm.ceil.v8f64
// CHECK: call <8 x double> @llvm.ceil.v8f64
simd_ceil(a)
}
14 changes: 7 additions & 7 deletions src/test/codegen/simd-intrinsic/simd-intrinsic-float-cos.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,28 @@ extern "platform-intrinsic" {
// CHECK-LABEL: @fcos_32x2
#[no_mangle]
pub unsafe fn fcos_32x2(a: f32x2) -> f32x2 {
// CHECK: call fast <2 x float> @llvm.cos.v2f32
// CHECK: call <2 x float> @llvm.cos.v2f32
simd_fcos(a)
}

// CHECK-LABEL: @fcos_32x4
#[no_mangle]
pub unsafe fn fcos_32x4(a: f32x4) -> f32x4 {
// CHECK: call fast <4 x float> @llvm.cos.v4f32
// CHECK: call <4 x float> @llvm.cos.v4f32
simd_fcos(a)
}

// CHECK-LABEL: @fcos_32x8
#[no_mangle]
pub unsafe fn fcos_32x8(a: f32x8) -> f32x8 {
// CHECK: call fast <8 x float> @llvm.cos.v8f32
// CHECK: call <8 x float> @llvm.cos.v8f32
simd_fcos(a)
}

// CHECK-LABEL: @fcos_32x16
#[no_mangle]
pub unsafe fn fcos_32x16(a: f32x16) -> f32x16 {
// CHECK: call fast <16 x float> @llvm.cos.v16f32
// CHECK: call <16 x float> @llvm.cos.v16f32
simd_fcos(a)
}

Expand All @@ -73,20 +73,20 @@ pub struct f64x8(pub f64, pub f64, pub f64, pub f64,
// CHECK-LABEL: @fcos_64x4
#[no_mangle]
pub unsafe fn fcos_64x4(a: f64x4) -> f64x4 {
// CHECK: call fast <4 x double> @llvm.cos.v4f64
// CHECK: call <4 x double> @llvm.cos.v4f64
simd_fcos(a)
}

// CHECK-LABEL: @fcos_64x2
#[no_mangle]
pub unsafe fn fcos_64x2(a: f64x2) -> f64x2 {
// CHECK: call fast <2 x double> @llvm.cos.v2f64
// CHECK: call <2 x double> @llvm.cos.v2f64
simd_fcos(a)
}

// CHECK-LABEL: @fcos_64x8
#[no_mangle]
pub unsafe fn fcos_64x8(a: f64x8) -> f64x8 {
// CHECK: call fast <8 x double> @llvm.cos.v8f64
// CHECK: call <8 x double> @llvm.cos.v8f64
simd_fcos(a)
}
Loading

0 comments on commit ef88434

Please sign in to comment.