diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 433046e98a01a..e7d74f303da0b 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -868,6 +868,54 @@ void Lowering::LowerHWIntrinsicCmpOp(GenTreeHWIntrinsic* node, genTreeOps cmpOp) GenTree* op1 = node->Op(1); GenTree* op2 = node->Op(2); + // Optimize comparison against Vector64/128<>.Zero via UMAX: + // + // bool eq = v == Vector128.Zero + // + // to: + // + // bool eq = AdvSimd.Arm64.MaxAcross(v.AsUInt16()).ToScalar() == 0; + // + GenTree* op = nullptr; + GenTree* opZero = nullptr; + if (op1->IsVectorZero()) + { + op = op2; + opZero = op1; + } + else if (op2->IsVectorZero()) + { + op = op1; + opZero = op2; + } + + if (!varTypeIsFloating(simdBaseType) && (op != nullptr)) + { + // Use USHORT for V64 and UINT for V128 due to better latency/TP on some CPUs + CorInfoType maxType = (simdSize == 8) ? CORINFO_TYPE_USHORT : CORINFO_TYPE_UINT; + GenTree* cmp = comp->gtNewSimdHWIntrinsicNode(simdType, op, NI_AdvSimd_Arm64_MaxAcross, maxType, simdSize); + BlockRange().InsertBefore(node, cmp); + LowerNode(cmp); + BlockRange().Remove(opZero); + + GenTree* val = comp->gtNewSimdHWIntrinsicNode(TYP_INT, cmp, NI_Vector128_ToScalar, CORINFO_TYPE_UINT, simdSize); + BlockRange().InsertAfter(cmp, val); + LowerNode(val); + + GenTree* cmpZeroCns = comp->gtNewIconNode(0, TYP_INT); + BlockRange().InsertAfter(val, cmpZeroCns); + + node->ChangeOper(cmpOp); + node->gtType = TYP_INT; + node->AsOp()->gtOp1 = val; + node->AsOp()->gtOp2 = cmpZeroCns; + LowerNodeCC(node, (cmpOp == GT_EQ) ? GenCondition::EQ : GenCondition::NE); + node->gtType = TYP_VOID; + node->ClearUnusedValue(); + LowerNode(node); + return; + } + NamedIntrinsic cmpIntrinsic; switch (simdBaseType) diff --git a/src/tests/JIT/HardwareIntrinsics/General/HwiOp/CompareVectorWithZero.cs b/src/tests/JIT/HardwareIntrinsics/General/HwiOp/CompareVectorWithZero.cs new file mode 100644 index 0000000000000..8024922d59a36 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/General/HwiOp/CompareVectorWithZero.cs @@ -0,0 +1,88 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +public class CompareVectorWithZero +{ + public static int Main() + { + Test(Vector128.Create(0)); + Test(Vector128.Create(0.0f)); + Test(Vector128.Create(-0.0f)); + Test(Vector128.Create(0.0)); + Test(Vector128.Create(-0.0)); + + TestReversed(Vector128.Create(0)); + TestReversed(Vector128.Create(0.0f)); + TestReversed(Vector128.Create(-0.0f)); + TestReversed(Vector128.Create(0.0)); + TestReversed(Vector128.Create(-0.0)); + + Test(Vector128.Create(-10)); + Test(Vector128.Create(10)); + Test(Vector128.Create((sbyte)-10)); + Test(Vector128.Create((ushort)10)); + Test(Vector64.Create(0)); + Test(Vector64.Create(0.0f)); + Test(Vector64.Create(-0.0f)); + Test(Vector64.Create(0.0)); + Test(Vector64.Create(-0.0)); + Test(Vector64.Create(-10)); + Test(Vector64.Create(10)); + Test(Vector64.Create((sbyte)-10)); + Test(Vector64.Create((ushort)10)); + Test(Vector128.Create(0, 0, 0, 0, 0, 0, 0, 1)); + Test(Vector128.Create(0, 0, 0, 0, 0, 0, 0, -1)); + Test(Vector64.Create(0, 0, 0, 0, 0, 0, 0, 1)); + Test(Vector64.Create(0, 0, 0, 0, 0, 0, 0, -1)); + + Test(Vector128.Create(0, 0, 0, 0, 0, 0, 1, 0)); + Test(Vector128.Create(0, 0, 0, 0, 0, 0, 1, 0)); + Test(Vector64.Create(0, 0, 0, 0, 0, 0, -1, 0)); + Test(Vector64.Create(0, 0, 0, 0, 0, 0, -1, 0)); + + Test(Vector128.Create(0, 0, 0, 1, 0, 0, 0, 1)); + Test(Vector128.Create(0, 0, 0, -1, 0, 0, 0, -1)); + Test(Vector64.Create(0, 0, 0, 1, 0, 0, 0, 1)); + Test(Vector64.Create(0, 0, 0, -1, 0, 0, 0, -1)); + + TestReversed(Vector128.Create(0, 0, 0, 1, 0, 0, 0, 1)); + TestReversed(Vector128.Create(0, 0, 0, -1, 0, 0, 0, -1)); + TestReversed(Vector64.Create(0, 0, 0, 1, 0, 0, 0, 1)); + TestReversed(Vector64.Create(0, 0, 0, -1, 0, 0, 0, -1)); + return 100; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static T ToVar(T t) => t; + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void AssertTrue(bool expr) + { + if (!expr) + throw new InvalidOperationException(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void Test(Vector128 v) where T : unmanaged => + AssertTrue((v == Vector128.Zero) == + (v == Vector128.Create(ToVar(default(T))))); + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void Test(Vector64 v) where T : unmanaged => + AssertTrue((v == Vector64.Zero) == + (v == Vector64.Create(ToVar(default(T))))); + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void TestReversed(Vector128 v) where T : unmanaged => + AssertTrue((Vector128.Zero == v) == + (v == Vector128.Create(ToVar(default(T))))); + + [MethodImpl(MethodImplOptions.NoInlining)] + public static void TestReversed(Vector64 v) where T : unmanaged => + AssertTrue((Vector64.Zero == v) == + (v == Vector64.Create(ToVar(default(T))))); +} diff --git a/src/tests/JIT/HardwareIntrinsics/General/HwiOp/CompareVectorWithZero.csproj b/src/tests/JIT/HardwareIntrinsics/General/HwiOp/CompareVectorWithZero.csproj new file mode 100644 index 0000000000000..6946bed81bfd5 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/General/HwiOp/CompareVectorWithZero.csproj @@ -0,0 +1,9 @@ + + + Exe + True + + + + + diff --git a/src/tests/issues.targets b/src/tests/issues.targets index 81424d1de65eb..c5e27cc88c43a 100644 --- a/src/tests/issues.targets +++ b/src/tests/issues.targets @@ -1449,6 +1449,10 @@ Crashes during LLVM AOT compilation. + + https://github.com/dotnet/runtime/pull/65632#issuecomment-1046294324 + + Doesn't pass after LLVM AOT compilation.