JIT: Faster vector == Vector128.Zero on arm64

src/coreclr/jit/lowerarmarch.cpp

@@ -868,6 +868,54 @@ void Lowering::LowerHWIntrinsicCmpOp(GenTreeHWIntrinsic* node, genTreeOps cmpOp)
     GenTree* op1 = node->Op(1);
     GenTree* op2 = node->Op(2);
 
+    // Optimize comparison against Vector64/128<>.Zero via UMAX:
+    //
+    //   bool eq = v == Vector128<integer>.Zero
+    //
+    // to:
+    //
+    //   bool eq = AdvSimd.Arm64.MaxAcross(v.AsUInt16()).ToScalar() == 0;
+    //
+    GenTree* op     = nullptr;
+    GenTree* opZero = nullptr;
+    if (op1->IsVectorZero())
+    {
+        op     = op2;
+        opZero = op1;
+    }
+    else if (op2->IsVectorZero())
+    {
+        op     = op1;
+        opZero = op2;
+    }
+
+    if (!varTypeIsFloating(simdBaseType) && (op != nullptr))
+    {
+        // Use USHORT for V64 and UINT for V128 due to better latency/TP on some CPUs
+        CorInfoType maxType = (simdSize == 8) ? CORINFO_TYPE_USHORT : CORINFO_TYPE_UINT;
+        GenTree*    cmp = comp->gtNewSimdHWIntrinsicNode(simdType, op, NI_AdvSimd_Arm64_MaxAcross, maxType, simdSize);
+        BlockRange().InsertBefore(node, cmp);
+        LowerNode(cmp);
+        BlockRange().Remove(opZero);
+
+        GenTree* val = comp->gtNewSimdHWIntrinsicNode(TYP_INT, cmp, NI_Vector128_ToScalar, CORINFO_TYPE_UINT, simdSize);
+        BlockRange().InsertAfter(cmp, val);
+        LowerNode(val);
+
+        GenTree* cmpZeroCns = comp->gtNewIconNode(0, TYP_INT);
+        BlockRange().InsertAfter(val, cmpZeroCns);
+
+        node->ChangeOper(cmpOp);
+        node->gtType        = TYP_INT;
+        node->AsOp()->gtOp1 = val;
+        node->AsOp()->gtOp2 = cmpZeroCns;
+        LowerNodeCC(node, (cmpOp == GT_EQ) ? GenCondition::EQ : GenCondition::NE);
+        node->gtType = TYP_VOID;
+        node->ClearUnusedValue();
+        LowerNode(node);
+        return;
+    }
+
     NamedIntrinsic cmpIntrinsic;
 
     switch (simdBaseType)

src/tests/JIT/HardwareIntrinsics/General/HwiOp/CompareVectorWithZero.cs

@@ -0,0 +1,88 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+
+public class CompareVectorWithZero
+{
+    public static int Main()
+    {
+        Test(Vector128.Create(0));
+        Test(Vector128.Create(0.0f));
+        Test(Vector128.Create(-0.0f));
+        Test(Vector128.Create(0.0));
+        Test(Vector128.Create(-0.0));
+
+        TestReversed(Vector128.Create(0));
+        TestReversed(Vector128.Create(0.0f));
+        TestReversed(Vector128.Create(-0.0f));
+        TestReversed(Vector128.Create(0.0));
+        TestReversed(Vector128.Create(-0.0));
+
+        Test(Vector128.Create(-10));
+        Test(Vector128.Create(10));
+        Test(Vector128.Create((sbyte)-10));
+        Test(Vector128.Create((ushort)10));
+        Test(Vector64.Create(0));
+        Test(Vector64.Create(0.0f));
+        Test(Vector64.Create(-0.0f));
+        Test(Vector64.Create(0.0));
+        Test(Vector64.Create(-0.0));
+        Test(Vector64.Create(-10));
+        Test(Vector64.Create(10));
+        Test(Vector64.Create((sbyte)-10));
+        Test(Vector64.Create((ushort)10));
+        Test(Vector128.Create(0, 0, 0, 0, 0, 0, 0, 1));
+        Test(Vector128.Create(0, 0, 0, 0, 0, 0, 0, -1));
+        Test(Vector64.Create(0, 0, 0, 0, 0, 0, 0, 1));
+        Test(Vector64.Create(0, 0, 0, 0, 0, 0, 0, -1));
+
+        Test(Vector128.Create(0, 0, 0, 0, 0, 0, 1, 0));
+        Test(Vector128.Create(0, 0, 0, 0, 0, 0, 1, 0));
+        Test(Vector64.Create(0, 0, 0, 0, 0, 0, -1, 0));
+        Test(Vector64.Create(0, 0, 0, 0, 0, 0, -1, 0));
+
+        Test(Vector128.Create(0, 0, 0, 1, 0, 0, 0, 1));
+        Test(Vector128.Create(0, 0, 0, -1, 0, 0, 0, -1));
+        Test(Vector64.Create(0, 0, 0, 1, 0, 0, 0, 1));
+        Test(Vector64.Create(0, 0, 0, -1, 0, 0, 0, -1));
+
+        TestReversed(Vector128.Create(0, 0, 0, 1, 0, 0, 0, 1));
+        TestReversed(Vector128.Create(0, 0, 0, -1, 0, 0, 0, -1));
+        TestReversed(Vector64.Create(0, 0, 0, 1, 0, 0, 0, 1));
+        TestReversed(Vector64.Create(0, 0, 0, -1, 0, 0, 0, -1));
+        return 100;
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static T ToVar<T>(T t) => t;
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static void AssertTrue(bool expr)
+    {
+        if (!expr)
+            throw new InvalidOperationException();
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static void Test<T>(Vector128<T> v) where T : unmanaged =>
+        AssertTrue((v == Vector128<T>.Zero) == 
+                   (v == Vector128.Create(ToVar(default(T)))));
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static void Test<T>(Vector64<T> v) where T : unmanaged =>
+        AssertTrue((v == Vector64<T>.Zero) == 
+                   (v == Vector64.Create(ToVar(default(T)))));
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static void TestReversed<T>(Vector128<T> v) where T : unmanaged =>
+        AssertTrue((Vector128<T>.Zero == v) == 
+                   (v == Vector128.Create(ToVar(default(T)))));
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static void TestReversed<T>(Vector64<T> v) where T : unmanaged =>
+        AssertTrue((Vector64<T>.Zero == v) == 
+                   (v == Vector64.Create(ToVar(default(T)))));
+}

src/tests/JIT/HardwareIntrinsics/General/HwiOp/CompareVectorWithZero.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <Optimize>True</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+</Project>

src/tests/issues.targets

@@ -1449,6 +1449,10 @@
             <Issue>Crashes during LLVM AOT compilation.</Issue>
         </ExcludeList>
 
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/HardwareIntrinsics/General/HwiOp/CompareVectorWithZero/**">
+            <Issue>https://github.com/dotnet/runtime/pull/65632#issuecomment-1046294324</Issue>
+        </ExcludeList>
+
         <ExcludeList Include="$(XunitTestBinBase)/JIT/opt/InstructionCombining/DivToMul/**">
             <Issue>Doesn't pass after LLVM AOT compilation.</Issue>
         </ExcludeList>