From b55a3155bcc5ebb8d0ce7c3facaba1eddcc383a5 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Tue, 26 Sep 2023 20:04:00 -0400 Subject: [PATCH] Vectorize TensorPrimitives.Min/Max{Magnitude} (#92618) * Vectorize TensorPrimitives.Min/Max{Magnitude} * Use AdvSimd.Max/Min * Rename some parameters/locals for consistency * Improve HorizontalAggregate * Move a few helpers * Avoid scalar path for returning found NaN --- .../Numerics/Tensors/TensorPrimitives.cs | 244 +------ .../Tensors/TensorPrimitives.netcore.cs | 638 ++++++++++++++++-- .../Tensors/TensorPrimitives.netstandard.cs | 255 ++++++- .../tests/TensorPrimitivesTests.cs | 194 +++++- 4 files changed, 1034 insertions(+), 297 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs index a4429a1eba080..48ff55b4f0bd2 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs @@ -408,44 +408,8 @@ public static void Log2(ReadOnlySpan x, Span destination) /// The tensor, represented as a span. /// The maximum element in . /// Length of '' must be greater than zero. - public static float Max(ReadOnlySpan x) - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - float result = float.NegativeInfinity; - - for (int i = 0; i < x.Length; i++) - { - // This matches the IEEE 754:2019 `maximum` function. - // It propagates NaN inputs back to the caller and - // otherwise returns the greater of the inputs. - // It treats +0 as greater than -0 as per the specification. - - float current = x[i]; - - if (current != result) - { - if (float.IsNaN(current)) - { - return current; - } - - if (result < current) - { - result = current; - } - } - else if (IsNegative(result)) - { - result = current; - } - } - - return result; - } + public static float Max(ReadOnlySpan x) => + MinMaxCore(x); /// Computes the element-wise result of: MathF.Max(, ). /// The first tensor, represented as a span. @@ -454,70 +418,15 @@ public static float Max(ReadOnlySpan x) /// Length of '' must be same as length of ''. /// Destination is too short. /// This method effectively does [i] = MathF.Max([i], [i]). - public static void Max(ReadOnlySpan x, ReadOnlySpan y, Span destination) - { - if (x.Length != y.Length) - { - ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); - } - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - for (int i = 0; i < x.Length; i++) - { - destination[i] = MathF.Max(x[i], y[i]); - } - } + public static void Max(ReadOnlySpan x, ReadOnlySpan y, Span destination) => + InvokeSpanSpanIntoSpan(x, y, destination); /// Computes the maximum magnitude of any element in . /// The tensor, represented as a span. /// The maximum magnitude of any element in . /// Length of '' must be greater than zero. - public static float MaxMagnitude(ReadOnlySpan x) - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - float result = float.NegativeInfinity; - float resultMag = float.NegativeInfinity; - - for (int i = 0; i < x.Length; i++) - { - // This matches the IEEE 754:2019 `maximumMagnitude` function. - // It propagates NaN inputs back to the caller and - // otherwise returns the input with a greater magnitude. - // It treats +0 as greater than -0 as per the specification. - - float current = x[i]; - float currentMag = Math.Abs(current); - - if (currentMag != resultMag) - { - if (float.IsNaN(currentMag)) - { - return currentMag; - } - - if (resultMag < currentMag) - { - result = current; - resultMag = currentMag; - } - } - else if (IsNegative(result)) - { - result = current; - resultMag = currentMag; - } - } - - return result; - } + public static float MaxMagnitude(ReadOnlySpan x) => + MinMaxCore(x); /// Computes the element-wise result of: MathF.MaxMagnitude(, ). /// The first tensor, represented as a span. @@ -526,66 +435,15 @@ public static float MaxMagnitude(ReadOnlySpan x) /// Length of '' must be same as length of ''. /// Destination is too short. /// This method effectively does [i] = MathF.MaxMagnitude([i], [i]). - public static void MaxMagnitude(ReadOnlySpan x, ReadOnlySpan y, Span destination) - { - if (x.Length != y.Length) - { - ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); - } - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - for (int i = 0; i < x.Length; i++) - { - destination[i] = MaxMagnitude(x[i], y[i]); - } - } + public static void MaxMagnitude(ReadOnlySpan x, ReadOnlySpan y, Span destination) => + InvokeSpanSpanIntoSpan(x, y, destination); /// Computes the minimum element in . /// The tensor, represented as a span. /// The minimum element in . /// Length of '' must be greater than zero. - public static float Min(ReadOnlySpan x) - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - float result = float.PositiveInfinity; - - for (int i = 0; i < x.Length; i++) - { - // This matches the IEEE 754:2019 `minimum` function - // It propagates NaN inputs back to the caller and - // otherwise returns the lesser of the inputs. - // It treats +0 as greater than -0 as per the specification. - - float current = x[i]; - - if (current != result) - { - if (float.IsNaN(current)) - { - return current; - } - - if (current < result) - { - result = current; - } - } - else if (IsNegative(current)) - { - result = current; - } - } - - return result; - } + public static float Min(ReadOnlySpan x) => + MinMaxCore(x); /// Computes the element-wise result of: MathF.Min(, ). /// The first tensor, represented as a span. @@ -594,70 +452,15 @@ public static float Min(ReadOnlySpan x) /// Length of '' must be same as length of ''. /// Destination is too short. /// This method effectively does [i] = MathF.Min([i], [i]). - public static void Min(ReadOnlySpan x, ReadOnlySpan y, Span destination) - { - if (x.Length != y.Length) - { - ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); - } - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - for (int i = 0; i < x.Length; i++) - { - destination[i] = MathF.Min(x[i], y[i]); - } - } + public static void Min(ReadOnlySpan x, ReadOnlySpan y, Span destination) => + InvokeSpanSpanIntoSpan(x, y, destination); /// Computes the minimum magnitude of any element in . /// The tensor, represented as a span. /// The minimum magnitude of any element in . /// Length of '' must be greater than zero. - public static float MinMagnitude(ReadOnlySpan x) - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - float result = float.PositiveInfinity; - float resultMag = float.PositiveInfinity; - - for (int i = 0; i < x.Length; i++) - { - // This matches the IEEE 754:2019 `minimumMagnitude` function. - // It propagates NaN inputs back to the caller and - // otherwise returns the input with a lesser magnitude. - // It treats +0 as greater than -0 as per the specification. - - float current = x[i]; - float currentMag = Math.Abs(current); - - if (currentMag != resultMag) - { - if (float.IsNaN(currentMag)) - { - return currentMag; - } - - if (currentMag < resultMag) - { - result = current; - resultMag = currentMag; - } - } - else if (IsNegative(current)) - { - result = current; - resultMag = currentMag; - } - } - - return result; - } + public static float MinMagnitude(ReadOnlySpan x) => + MinMaxCore(x); /// Computes the element-wise result of: MathF.MinMagnitude(, ). /// The first tensor, represented as a span. @@ -666,23 +469,8 @@ public static float MinMagnitude(ReadOnlySpan x) /// Length of '' must be same as length of ''. /// Destination is too short. /// This method effectively does [i] = MathF.MinMagnitude([i], [i]). - public static void MinMagnitude(ReadOnlySpan x, ReadOnlySpan y, Span destination) - { - if (x.Length != y.Length) - { - ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); - } - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - for (int i = 0; i < x.Length; i++) - { - destination[i] = MinMagnitude(x[i], y[i]); - } - } + public static void MinMagnitude(ReadOnlySpan x, ReadOnlySpan y, Span destination) => + InvokeSpanSpanIntoSpan(x, y, destination); /// Computes the element-wise result of: * . /// The first tensor, represented as a span. diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index 6323660cb75fd..e82f67b1f27d5 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -52,14 +52,6 @@ public static void ConvertToSingle(ReadOnlySpan source, Span destin } } - private static bool IsNegative(float f) => float.IsNegative(f); - - private static float MaxMagnitude(float x, float y) => MathF.MaxMagnitude(x, y); - - private static float MinMagnitude(float x, float y) => MathF.MinMagnitude(x, y); - - private static float Log2(float x) => MathF.Log2(x); - private static float CosineSimilarityCore(ReadOnlySpan x, ReadOnlySpan y) { // Compute the same as: @@ -177,8 +169,8 @@ private static float CosineSimilarityCore(ReadOnlySpan x, ReadOnlySpan( float identityValue, ReadOnlySpan x) - where TLoad : IUnaryOperator - where TAggregate : IBinaryOperator + where TLoad : struct, IUnaryOperator + where TAggregate : struct, IBinaryOperator { // Initialize the result to the identity value float result = identityValue; @@ -262,8 +254,8 @@ private static float Aggregate( private static float Aggregate( float identityValue, ReadOnlySpan x, ReadOnlySpan y) - where TBinary : IBinaryOperator - where TAggregate : IBinaryOperator + where TBinary : struct, IBinaryOperator + where TAggregate : struct, IBinaryOperator { // Initialize the result to the identity value float result = identityValue; @@ -348,9 +340,184 @@ private static float Aggregate( return result; } + /// + /// This is the same as , + /// except it early exits on NaN. + /// + private static float MinMaxCore(ReadOnlySpan x) where TMinMax : struct, IBinaryOperator + { + if (x.IsEmpty) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + // This matches the IEEE 754:2019 `maximum`/`minimum` functions. + // It propagates NaN inputs back to the caller and + // otherwise returns the greater of the inputs. + // It treats +0 as greater than -0 as per the specification. + + // Initialize the result to the identity value + float result = x[0]; + int i = 0; + +#if NET8_0_OR_GREATER + if (Vector512.IsHardwareAccelerated && x.Length >= Vector512.Count * 2) + { + ref float xRef = ref MemoryMarshal.GetReference(x); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector512 resultVector = Vector512.LoadUnsafe(ref xRef, 0), current; + if (!Vector512.EqualsAll(resultVector, resultVector)) + { + return GetFirstNaN(resultVector); + } + + int oneVectorFromEnd = x.Length - Vector512.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + i = Vector512.Count; + do + { + // Load the next vector, and early exit on NaN. + current = Vector512.LoadUnsafe(ref xRef, (uint)i); + if (!Vector512.EqualsAll(current, current)) + { + return GetFirstNaN(current); + } + + resultVector = TMinMax.Invoke(resultVector, current); + i += Vector512.Count; + } + while (i <= oneVectorFromEnd); + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); + if (!Vector512.EqualsAll(current, current)) + { + return GetFirstNaN(current); + } + + resultVector = TMinMax.Invoke(resultVector, current); + } + + // Aggregate the lanes in the vector to create the final scalar result. + return TMinMax.Invoke(resultVector); + } +#endif + + if (Vector256.IsHardwareAccelerated && x.Length >= Vector256.Count * 2) + { + ref float xRef = ref MemoryMarshal.GetReference(x); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector256 resultVector = Vector256.LoadUnsafe(ref xRef, 0), current; + if (!Vector256.EqualsAll(resultVector, resultVector)) + { + return GetFirstNaN(resultVector); + } + + int oneVectorFromEnd = x.Length - Vector256.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + i = Vector256.Count; + do + { + // Load the next vector, and early exit on NaN. + current = Vector256.LoadUnsafe(ref xRef, (uint)i); + if (!Vector256.EqualsAll(current, current)) + { + return GetFirstNaN(current); + } + + resultVector = TMinMax.Invoke(resultVector, current); + i += Vector256.Count; + } + while (i <= oneVectorFromEnd); + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); + if (!Vector256.EqualsAll(current, current)) + { + return GetFirstNaN(current); + } + + resultVector = TMinMax.Invoke(resultVector, current); + } + + // Aggregate the lanes in the vector to create the final scalar result. + return TMinMax.Invoke(resultVector); + } + + if (Vector128.IsHardwareAccelerated && x.Length >= Vector128.Count * 2) + { + ref float xRef = ref MemoryMarshal.GetReference(x); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector128 resultVector = Vector128.LoadUnsafe(ref xRef, 0), current; + if (!Vector128.EqualsAll(resultVector, resultVector)) + { + return GetFirstNaN(resultVector); + } + + int oneVectorFromEnd = x.Length - Vector128.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + i = Vector128.Count; + do + { + // Load the next vector, and early exit on NaN. + current = Vector128.LoadUnsafe(ref xRef, (uint)i); + if (!Vector128.EqualsAll(current, current)) + { + return GetFirstNaN(current); + } + + resultVector = TMinMax.Invoke(resultVector, current); + i += Vector128.Count; + } + while (i <= oneVectorFromEnd); + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); + if (!Vector128.EqualsAll(current, current)) + { + return GetFirstNaN(current); + } + + resultVector = TMinMax.Invoke(resultVector, current); + } + + // Aggregate the lanes in the vector to create the final scalar result. + return TMinMax.Invoke(resultVector); + } + + // Scalar path used when either vectorization is not supported or the input is too small to vectorize. + for (; (uint)i < (uint)x.Length; i++) + { + float current = x[i]; + if (float.IsNaN(current)) + { + return current; + } + + result = TMinMax.Invoke(result, current); + } + + return result; + } + private static unsafe void InvokeSpanIntoSpan( ReadOnlySpan x, Span destination) - where TUnaryOperator : IUnaryOperator + where TUnaryOperator : struct, IUnaryOperator { if (x.Length > destination.Length) { @@ -448,7 +615,7 @@ private static unsafe void InvokeSpanIntoSpan( private static unsafe void InvokeSpanSpanIntoSpan( ReadOnlySpan x, ReadOnlySpan y, Span destination) - where TBinaryOperator : IBinaryOperator + where TBinaryOperator : struct, IBinaryOperator { if (x.Length != y.Length) { @@ -559,7 +726,7 @@ private static unsafe void InvokeSpanSpanIntoSpan( private static unsafe void InvokeSpanScalarIntoSpan( ReadOnlySpan x, float y, Span destination) - where TBinaryOperator : IBinaryOperator + where TBinaryOperator : struct, IBinaryOperator { if (x.Length > destination.Length) { @@ -670,7 +837,7 @@ private static unsafe void InvokeSpanScalarIntoSpan( private static unsafe void InvokeSpanSpanSpanIntoSpan( ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan z, Span destination) - where TTernaryOperator : ITernaryOperator + where TTernaryOperator : struct, ITernaryOperator { if (x.Length != y.Length || x.Length != z.Length) { @@ -789,7 +956,7 @@ private static unsafe void InvokeSpanSpanSpanIntoSpan( private static unsafe void InvokeSpanSpanScalarIntoSpan( ReadOnlySpan x, ReadOnlySpan y, float z, Span destination) - where TTernaryOperator : ITernaryOperator + where TTernaryOperator : struct, ITernaryOperator { if (x.Length != y.Length) { @@ -913,7 +1080,7 @@ private static unsafe void InvokeSpanSpanScalarIntoSpan( private static unsafe void InvokeSpanScalarSpanIntoSpan( ReadOnlySpan x, float y, ReadOnlySpan z, Span destination) - where TTernaryOperator : ITernaryOperator + where TTernaryOperator : struct, ITernaryOperator { if (x.Length != z.Length) { @@ -1075,6 +1242,51 @@ private static Vector512 FusedMultiplyAdd(Vector512 x, Vector512(Vector128 x) where TAggregate : struct, IBinaryOperator => + TAggregate.Invoke( + TAggregate.Invoke(x[0], x[1]), + TAggregate.Invoke(x[2], x[3])); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static float HorizontalAggregate(Vector256 x) where TAggregate : struct, IBinaryOperator => + HorizontalAggregate(TAggregate.Invoke(x.GetLower(), x.GetUpper())); + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static float HorizontalAggregate(Vector512 x) where TAggregate : struct, IBinaryOperator => + HorizontalAggregate(TAggregate.Invoke(x.GetLower(), x.GetUpper())); +#endif + + private static bool IsNegative(float f) => float.IsNegative(f); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 IsNegative(Vector128 vector) => + Vector128.LessThan(vector.AsInt32(), Vector128.Zero).AsSingle(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 IsNegative(Vector256 vector) => + Vector256.LessThan(vector.AsInt32(), Vector256.Zero).AsSingle(); + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 IsNegative(Vector512 vector) => + Vector512.LessThan(vector.AsInt32(), Vector512.Zero).AsSingle(); +#endif + + private static float GetFirstNaN(Vector128 vector) => + vector[BitOperations.TrailingZeroCount((~Vector128.Equals(vector, vector)).ExtractMostSignificantBits())]; + + private static float GetFirstNaN(Vector256 vector) => + vector[BitOperations.TrailingZeroCount((~Vector256.Equals(vector, vector)).ExtractMostSignificantBits())]; + +#if NET8_0_OR_GREATER + private static float GetFirstNaN(Vector512 vector) => + vector[BitOperations.TrailingZeroCount((~Vector512.Equals(vector, vector)).ExtractMostSignificantBits())]; +#endif + + private static float Log2(float x) => MathF.Log2(x); + private readonly struct AddOperator : IBinaryOperator { public static float Invoke(float x, float y) => x + y; @@ -1139,49 +1351,399 @@ public static Vector512 Invoke(Vector512 x, Vector512 y) public static Vector512 Invoke(Vector512 x, Vector512 y) => x * y; #endif + public static float Invoke(Vector128 x) => HorizontalAggregate(x); + public static float Invoke(Vector256 x) => HorizontalAggregate(x); +#if NET8_0_OR_GREATER + public static float Invoke(Vector512 x) => HorizontalAggregate(x); +#endif + } + + private readonly struct DivideOperator : IBinaryOperator + { + public static float Invoke(float x, float y) => x / y; + public static Vector128 Invoke(Vector128 x, Vector128 y) => x / y; + public static Vector256 Invoke(Vector256 x, Vector256 y) => x / y; +#if NET8_0_OR_GREATER + public static Vector512 Invoke(Vector512 x, Vector512 y) => x / y; +#endif + } + + private readonly struct MaxOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Invoke(float x, float y) => + x == y ? + (IsNegative(x) ? y : x) : + (y > x ? y : x); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + if (AdvSimd.IsSupported) + { + return AdvSimd.Max(x, y); + } + + return + Vector128.ConditionalSelect(Vector128.Equals(x, y), + Vector128.ConditionalSelect(IsNegative(x), y, x), + Vector128.Max(x, y)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => + Vector256.ConditionalSelect(Vector256.Equals(x, y), + Vector256.ConditionalSelect(IsNegative(x), y, x), + Vector256.Max(x, y)); + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => + Vector512.ConditionalSelect(Vector512.Equals(x, y), + Vector512.ConditionalSelect(IsNegative(x), y, x), + Vector512.Max(x, y)); +#endif + + public static float Invoke(Vector128 x) => HorizontalAggregate(x); + public static float Invoke(Vector256 x) => HorizontalAggregate(x); +#if NET8_0_OR_GREATER + public static float Invoke(Vector512 x) => HorizontalAggregate(x); +#endif + } + + private readonly struct MaxPropagateNaNOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Invoke(float x, float y) => MathF.Max(x, y); + [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static float Invoke(Vector128 x) + public static Vector128 Invoke(Vector128 x, Vector128 y) { - float f = x[0]; - for (int i = 1; i < Vector128.Count; i++) + if (AdvSimd.IsSupported) { - f *= x[i]; + return AdvSimd.Max(x, y); } - return f; + + return + Vector128.ConditionalSelect(Vector128.Equals(x, x), + Vector128.ConditionalSelect(Vector128.Equals(y, y), + Vector128.ConditionalSelect(Vector128.Equals(x, y), + Vector128.ConditionalSelect(IsNegative(x), y, x), + Vector128.Max(x, y)), + y), + x); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static float Invoke(Vector256 x) + public static Vector256 Invoke(Vector256 x, Vector256 y) => + Vector256.ConditionalSelect(Vector256.Equals(x, x), + Vector256.ConditionalSelect(Vector256.Equals(y, y), + Vector256.ConditionalSelect(Vector256.Equals(x, y), + Vector256.ConditionalSelect(IsNegative(x), y, x), + Vector256.Max(x, y)), + y), + x); + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => + Vector512.ConditionalSelect(Vector512.Equals(x, x), + Vector512.ConditionalSelect(Vector512.Equals(y, y), + Vector512.ConditionalSelect(Vector512.Equals(x, y), + Vector512.ConditionalSelect(IsNegative(x), y, x), + Vector512.Max(x, y)), + y), + x); +#endif + } + + private readonly struct MaxMagnitudeOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Invoke(float x, float y) { - float f = x[0]; - for (int i = 1; i < Vector256.Count; i++) + float xMag = MathF.Abs(x), yMag = MathF.Abs(y); + return + xMag == yMag ? + (IsNegative(x) ? y : x) : + (xMag > yMag ? x : y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); + return + Vector128.ConditionalSelect(Vector128.Equals(xMag, yMag), + Vector128.ConditionalSelect(IsNegative(x), y, x), + Vector128.ConditionalSelect(Vector128.GreaterThan(xMag, yMag), x, y)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); + return + Vector256.ConditionalSelect(Vector256.Equals(xMag, yMag), + Vector256.ConditionalSelect(IsNegative(x), y, x), + Vector256.ConditionalSelect(Vector256.GreaterThan(xMag, yMag), x, y)); + } + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); + return + Vector512.ConditionalSelect(Vector512.Equals(xMag, yMag), + Vector512.ConditionalSelect(IsNegative(x), y, x), + Vector512.ConditionalSelect(Vector512.GreaterThan(xMag, yMag), x, y)); + } +#endif + + public static float Invoke(Vector128 x) => HorizontalAggregate(x); + public static float Invoke(Vector256 x) => HorizontalAggregate(x); +#if NET8_0_OR_GREATER + public static float Invoke(Vector512 x) => HorizontalAggregate(x); +#endif + } + + private readonly struct MaxMagnitudePropagateNaNOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Invoke(float x, float y) => MathF.MaxMagnitude(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); + return + Vector128.ConditionalSelect(Vector128.Equals(x, x), + Vector128.ConditionalSelect(Vector128.Equals(y, y), + Vector128.ConditionalSelect(Vector128.Equals(yMag, xMag), + Vector128.ConditionalSelect(IsNegative(x), y, x), + Vector128.ConditionalSelect(Vector128.GreaterThan(yMag, xMag), y, x)), + y), + x); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); + return + Vector256.ConditionalSelect(Vector256.Equals(x, x), + Vector256.ConditionalSelect(Vector256.Equals(y, y), + Vector256.ConditionalSelect(Vector256.Equals(xMag, yMag), + Vector256.ConditionalSelect(IsNegative(x), y, x), + Vector256.ConditionalSelect(Vector256.GreaterThan(xMag, yMag), x, y)), + y), + x); + } + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); + return + Vector512.ConditionalSelect(Vector512.Equals(x, x), + Vector512.ConditionalSelect(Vector512.Equals(y, y), + Vector512.ConditionalSelect(Vector512.Equals(xMag, yMag), + Vector512.ConditionalSelect(IsNegative(x), y, x), + Vector512.ConditionalSelect(Vector512.GreaterThan(xMag, yMag), x, y)), + y), + x); + } +#endif + } + + private readonly struct MinOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Invoke(float x, float y) => + x == y ? + (IsNegative(y) ? y : x) : + (y < x ? y : x); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + if (AdvSimd.IsSupported) { - f *= x[i]; + return AdvSimd.Min(x, y); } - return f; + + return + Vector128.ConditionalSelect(Vector128.Equals(x, y), + Vector128.ConditionalSelect(IsNegative(y), y, x), + Vector128.Min(x, y)); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => + Vector256.ConditionalSelect(Vector256.Equals(x, y), + Vector256.ConditionalSelect(IsNegative(y), y, x), + Vector256.Min(x, y)); + #if NET8_0_OR_GREATER [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static float Invoke(Vector512 x) + public static Vector512 Invoke(Vector512 x, Vector512 y) => + Vector512.ConditionalSelect(Vector512.Equals(x, y), + Vector512.ConditionalSelect(IsNegative(y), y, x), + Vector512.Min(x, y)); +#endif + + public static float Invoke(Vector128 x) => HorizontalAggregate(x); + public static float Invoke(Vector256 x) => HorizontalAggregate(x); +#if NET8_0_OR_GREATER + public static float Invoke(Vector512 x) => HorizontalAggregate(x); +#endif + } + + private readonly struct MinPropagateNaNOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Invoke(float x, float y) => MathF.Min(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) { - float f = x[0]; - for (int i = 1; i < Vector512.Count; i++) + if (AdvSimd.IsSupported) { - f *= x[i]; + return AdvSimd.Min(x, y); } - return f; + + return + Vector128.ConditionalSelect(Vector128.Equals(x, x), + Vector128.ConditionalSelect(Vector128.Equals(y, y), + Vector128.ConditionalSelect(Vector128.Equals(x, y), + Vector128.ConditionalSelect(IsNegative(x), x, y), + Vector128.Min(x, y)), + y), + x); } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => + Vector256.ConditionalSelect(Vector256.Equals(x, x), + Vector256.ConditionalSelect(Vector256.Equals(y, y), + Vector256.ConditionalSelect(Vector256.Equals(x, y), + Vector256.ConditionalSelect(IsNegative(x), x, y), + Vector256.Min(x, y)), + y), + x); + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => + Vector512.ConditionalSelect(Vector512.Equals(x, x), + Vector512.ConditionalSelect(Vector512.Equals(y, y), + Vector512.ConditionalSelect(Vector512.Equals(x, y), + Vector512.ConditionalSelect(IsNegative(x), x, y), + Vector512.Min(x, y)), + y), + x); #endif } - private readonly struct DivideOperator : IBinaryOperator + private readonly struct MinMagnitudeOperator : IBinaryOperator { - public static float Invoke(float x, float y) => x / y; - public static Vector128 Invoke(Vector128 x, Vector128 y) => x / y; - public static Vector256 Invoke(Vector256 x, Vector256 y) => x / y; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Invoke(float x, float y) + { + float xMag = MathF.Abs(x), yMag = MathF.Abs(y); + return xMag == yMag ? + (IsNegative(y) ? y : x) : + (yMag < xMag ? y : x); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); + return + Vector128.ConditionalSelect(Vector128.Equals(yMag, xMag), + Vector128.ConditionalSelect(IsNegative(y), y, x), + Vector128.ConditionalSelect(Vector128.LessThan(yMag, xMag), y, x)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); + return + Vector256.ConditionalSelect(Vector256.Equals(yMag, xMag), + Vector256.ConditionalSelect(IsNegative(y), y, x), + Vector256.ConditionalSelect(Vector256.LessThan(yMag, xMag), y, x)); + } + #if NET8_0_OR_GREATER - public static Vector512 Invoke(Vector512 x, Vector512 y) => x / y; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); + return + Vector512.ConditionalSelect(Vector512.Equals(yMag, xMag), + Vector512.ConditionalSelect(IsNegative(y), y, x), + Vector512.ConditionalSelect(Vector512.LessThan(yMag, xMag), y, x)); + } +#endif + + public static float Invoke(Vector128 x) => HorizontalAggregate(x); + public static float Invoke(Vector256 x) => HorizontalAggregate(x); +#if NET8_0_OR_GREATER + public static float Invoke(Vector512 x) => HorizontalAggregate(x); +#endif + } + + private readonly struct MinMagnitudePropagateNaNOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static float Invoke(float x, float y) => MathF.MinMagnitude(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); + return + Vector128.ConditionalSelect(Vector128.Equals(x, x), + Vector128.ConditionalSelect(Vector128.Equals(y, y), + Vector128.ConditionalSelect(Vector128.Equals(yMag, xMag), + Vector128.ConditionalSelect(IsNegative(x), x, y), + Vector128.ConditionalSelect(Vector128.LessThan(xMag, yMag), x, y)), + y), + x); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); + return + Vector256.ConditionalSelect(Vector256.Equals(x, x), + Vector256.ConditionalSelect(Vector256.Equals(y, y), + Vector256.ConditionalSelect(Vector256.Equals(yMag, xMag), + Vector256.ConditionalSelect(IsNegative(x), x, y), + Vector256.ConditionalSelect(Vector256.LessThan(xMag, yMag), x, y)), + y), + x); + } + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); + return + Vector512.ConditionalSelect(Vector512.Equals(x, x), + Vector512.ConditionalSelect(Vector512.Equals(y, y), + Vector512.ConditionalSelect(Vector512.Equals(yMag, xMag), + Vector512.ConditionalSelect(IsNegative(x), x, y), + Vector512.ConditionalSelect(Vector512.LessThan(xMag, yMag), x, y)), + y), + x); + } #endif } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs index 1ffd5d30683e9..a19f2529ab99c 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs @@ -8,14 +8,6 @@ namespace System.Numerics.Tensors { public static partial class TensorPrimitives { - private static unsafe bool IsNegative(float f) => *(int*)&f < 0; - - private static float MaxMagnitude(float x, float y) => MathF.Abs(x) >= MathF.Abs(y) ? x : y; - - private static float MinMagnitude(float x, float y) => MathF.Abs(x) < MathF.Abs(y) ? x : y; - - private static float Log2(float x) => MathF.Log(x, 2); - private static float CosineSimilarityCore(ReadOnlySpan x, ReadOnlySpan y) { // Compute the same as: @@ -160,6 +152,89 @@ private static float Aggregate( return result; } + private static float MinMaxCore(ReadOnlySpan x, TMinMax minMax = default) where TMinMax : struct, IBinaryOperator + { + if (x.IsEmpty) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + // This matches the IEEE 754:2019 `maximum`/`minimum` functions. + // It propagates NaN inputs back to the caller and + // otherwise returns the greater of the inputs. + // It treats +0 as greater than -0 as per the specification. + + // Initialize the result to the identity value + float result = x[0]; + int i = 0; + + if (Vector.IsHardwareAccelerated && x.Length >= Vector.Count * 2) + { + ref float xRef = ref MemoryMarshal.GetReference(x); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector resultVector = AsVector(ref xRef, 0), current; + if (Vector.EqualsAll(resultVector, resultVector)) + { + int oneVectorFromEnd = x.Length - Vector.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + i = Vector.Count; + do + { + // Load the next vector, and early exit on NaN. + current = AsVector(ref xRef, i); + if (!Vector.EqualsAll(current, current)) + { + goto Scalar; + } + + resultVector = minMax.Invoke(resultVector, current); + i += Vector.Count; + } + while (i <= oneVectorFromEnd); + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = AsVector(ref xRef, x.Length - Vector.Count); + if (!Vector.EqualsAll(current, current)) + { + goto Scalar; + } + + resultVector = minMax.Invoke(resultVector, current); + } + + // Aggregate the lanes in the vector to create the final scalar result. + for (int f = 0; f < Vector.Count; f++) + { + result = minMax.Invoke(result, resultVector[f]); + } + + return result; + } + } + + // Scalar path used when either vectorization is not supported, the input is too small to vectorize, + // or a NaN is encountered. + Scalar: + for (; (uint)i < (uint)x.Length; i++) + { + float current = x[i]; + + if (float.IsNaN(current)) + { + return current; + } + + result = minMax.Invoke(result, current); + } + + return result; + } + private static void InvokeSpanIntoSpan( ReadOnlySpan x, Span destination, TUnaryOperator op = default) where TUnaryOperator : struct, IUnaryOperator @@ -500,6 +575,13 @@ private static ref Vector AsVector(ref float start, int offset) => ref Unsafe.As>( ref Unsafe.Add(ref start, offset)); + private static unsafe bool IsNegative(float f) => *(int*)&f < 0; + + private static unsafe Vector IsNegative(Vector f) => + (Vector)Vector.LessThan((Vector)f, Vector.Zero); + + private static float Log2(float x) => MathF.Log(x, 2); + private readonly struct AddOperator : IBinaryOperator { public float Invoke(float x, float y) => x + y; @@ -539,6 +621,163 @@ public Vector Invoke(Vector x, Vector y) public Vector Invoke(Vector x, Vector y) => x / y; } + private readonly struct MaxOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public float Invoke(float x, float y) => + x == y ? + (IsNegative(x) ? y : x) : + (y > x ? y : x); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Vector Invoke(Vector x, Vector y) => + Vector.ConditionalSelect(Vector.Equals(x, y), + Vector.ConditionalSelect(IsNegative(x), y, x), + Vector.Max(x, y)); + } + + private readonly struct MaxPropagateNaNOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public float Invoke(float x, float y) => MathF.Max(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Vector Invoke(Vector x, Vector y) => + Vector.ConditionalSelect(Vector.Equals(x, x), + Vector.ConditionalSelect(Vector.Equals(y, y), + Vector.ConditionalSelect(Vector.Equals(x, y), + Vector.ConditionalSelect(IsNegative(x), y, x), + Vector.Max(x, y)), + y), + x); + } + + private readonly struct MaxMagnitudeOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public float Invoke(float x, float y) + { + float xMag = MathF.Abs(x), yMag = MathF.Abs(y); + return + yMag == xMag ? + (IsNegative(x) ? y : x) : + (xMag > yMag ? x : y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Vector Invoke(Vector x, Vector y) + { + Vector xMag = Vector.Abs(x), yMag = Vector.Abs(y); + return + Vector.ConditionalSelect(Vector.Equals(xMag, yMag), + Vector.ConditionalSelect(IsNegative(x), y, x), + Vector.ConditionalSelect(Vector.GreaterThan(xMag, yMag), x, y)); + } + } + + private readonly struct MaxMagnitudePropagateNaNOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public float Invoke(float x, float y) + { + float xMag = MathF.Abs(x), yMag = MathF.Abs(y); + return xMag > yMag || float.IsNaN(xMag) || (xMag == yMag && !IsNegative(x)) ? x : y; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Vector Invoke(Vector x, Vector y) + { + Vector xMag = Vector.Abs(x), yMag = Vector.Abs(y); + return + Vector.ConditionalSelect(Vector.Equals(x, x), + Vector.ConditionalSelect(Vector.Equals(y, y), + Vector.ConditionalSelect(Vector.Equals(xMag, yMag), + Vector.ConditionalSelect(IsNegative(x), y, x), + Vector.ConditionalSelect(Vector.GreaterThan(xMag, yMag), x, y)), + y), + x); + } + } + + private readonly struct MinOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public float Invoke(float x, float y) => + x == y ? + (IsNegative(y) ? y : x) : + (y < x ? y : x); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Vector Invoke(Vector x, Vector y) => + Vector.ConditionalSelect(Vector.Equals(x, y), + Vector.ConditionalSelect(IsNegative(y), y, x), + Vector.Min(x, y)); + } + + private readonly struct MinPropagateNaNOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public float Invoke(float x, float y) => MathF.Min(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Vector Invoke(Vector x, Vector y) => + Vector.ConditionalSelect(Vector.Equals(x, x), + Vector.ConditionalSelect(Vector.Equals(y, y), + Vector.ConditionalSelect(Vector.Equals(x, y), + Vector.ConditionalSelect(IsNegative(x), x, y), + Vector.Min(x, y)), + y), + x); + } + + private readonly struct MinMagnitudeOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public float Invoke(float x, float y) + { + float xMag = MathF.Abs(x), yMag = MathF.Abs(y); + return + yMag == xMag ? + (IsNegative(y) ? y : x) : + (yMag < xMag ? y : x); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Vector Invoke(Vector x, Vector y) + { + Vector xMag = Vector.Abs(x), yMag = Vector.Abs(y); + return + Vector.ConditionalSelect(Vector.Equals(yMag, xMag), + Vector.ConditionalSelect(IsNegative(y), y, x), + Vector.ConditionalSelect(Vector.LessThan(yMag, xMag), y, x)); + } + } + + private readonly struct MinMagnitudePropagateNaNOperator : IBinaryOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public float Invoke(float x, float y) + { + float xMag = MathF.Abs(x), yMag = MathF.Abs(y); + return xMag < yMag || float.IsNaN(xMag) || (xMag == yMag && IsNegative(x)) ? x : y; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Vector Invoke(Vector x, Vector y) + { + Vector xMag = Vector.Abs(x), yMag = Vector.Abs(y); + + return + Vector.ConditionalSelect(Vector.Equals(x, x), + Vector.ConditionalSelect(Vector.Equals(y, y), + Vector.ConditionalSelect(Vector.Equals(yMag, xMag), + Vector.ConditionalSelect(IsNegative(x), x, y), + Vector.ConditionalSelect(Vector.LessThan(xMag, yMag), x, y)), + y), + x); + } + } + private readonly struct NegateOperator : IUnaryOperator { public float Invoke(float x) => -x; diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs index 1f2fc37ad1f08..edcebe8eb4775 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs @@ -42,10 +42,20 @@ private static void FillTensor(Span tensor) } } - private static float NextSingle() - { + private static float NextSingle() => // For testing purposes, get a mix of negative and positive values. - return (float)((s_random.NextDouble() * 2) - 1); + (float)((s_random.NextDouble() * 2) - 1); + + private static unsafe float MathFMaxMagnitude(float x, float y) + { + float ax = MathF.Abs(x), ay = MathF.Abs(y); + return (ax > ay) || float.IsNaN(ax) || (ax == ay && *(int*)&x >= 0) ? x : y; + } + + private static unsafe float MathFMinMagnitude(float x, float y) + { + float ax = MathF.Abs(x), ay = MathF.Abs(y); + return (ax < ay) || float.IsNaN(ax) || (ax == ay && *(int*)&x < 0) ? x : y; } #endregion @@ -790,9 +800,10 @@ public static void Max_Tensor(int tensorLength) [MemberData(nameof(TensorLengths))] public static void Max_Tensor_NanReturned(int tensorLength) { - using BoundedMemory x = CreateAndFillTensor(tensorLength); + using BoundedMemory x = CreateTensor(tensorLength); foreach (int expected in new[] { 0, tensorLength / 2, tensorLength - 1 }) { + FillTensor(x); x[expected] = float.NaN; Assert.Equal(float.NaN, TensorPrimitives.Max(x)); } @@ -823,6 +834,41 @@ public static void Max_TwoTensors(int tensorLength) } } + [Theory] + [MemberData(nameof(TensorLengths))] + public static void Max_TwoTensors_SpecialValues(int tensorLength) + { + using BoundedMemory x = CreateAndFillTensor(tensorLength); + using BoundedMemory y = CreateAndFillTensor(tensorLength); + using BoundedMemory destination = CreateTensor(tensorLength); + + // NaNs + x[s_random.Next(x.Length)] = float.NaN; + y[s_random.Next(y.Length)] = float.NaN; + + // Same magnitude, opposite sign + int pos = s_random.Next(x.Length); + x[pos] = -5f; + y[pos] = 5f; + + // Positive and negative 0s + pos = s_random.Next(x.Length); + x[pos] = 0f; + y[pos] = -0f; + + TensorPrimitives.Max(x, y, destination); + for (int i = 0; i < tensorLength; i++) + { + Assert.Equal(MathF.Max(x[i], y[i]), destination[i], Tolerance); + } + + TensorPrimitives.Max(y, x, destination); + for (int i = 0; i < tensorLength; i++) + { + Assert.Equal(MathF.Max(y[i], x[i]), destination[i], Tolerance); + } + } + [Theory] [MemberData(nameof(TensorLengths))] public static void Max_TwoTensors_ThrowsForMismatchedLengths(int tensorLength) @@ -860,25 +906,23 @@ public static void MaxMagnitude_Tensor(int tensorLength) { using BoundedMemory x = CreateAndFillTensor(tensorLength); - int index = 0; - for (int i = 0; i < x.Length; i++) + float maxMagnitude = x[0]; + foreach (float i in x.Span) { - if (MathF.Abs(x[i]) >= MathF.Abs(x[index])) - { - index = i; - } + maxMagnitude = MathFMaxMagnitude(maxMagnitude, i); } - Assert.Equal(x[index], TensorPrimitives.MaxMagnitude(x), Tolerance); + Assert.Equal(maxMagnitude, TensorPrimitives.MaxMagnitude(x), Tolerance); } [Theory] [MemberData(nameof(TensorLengths))] public static void MaxMagnitude_Tensor_NanReturned(int tensorLength) { - using BoundedMemory x = CreateAndFillTensor(tensorLength); + using BoundedMemory x = CreateTensor(tensorLength); foreach (int expected in new[] { 0, tensorLength / 2, tensorLength - 1 }) { + FillTensor(x); x[expected] = float.NaN; Assert.Equal(float.NaN, TensorPrimitives.MaxMagnitude(x)); } @@ -907,7 +951,42 @@ public static void MaxMagnitude_TwoTensors(int tensorLength) for (int i = 0; i < tensorLength; i++) { - Assert.Equal(MathF.Abs(x[i]) >= MathF.Abs(y[i]) ? x[i] : y[i], destination[i], Tolerance); + Assert.Equal(MathFMaxMagnitude(x[i], y[i]), destination[i], Tolerance); + } + } + + [Theory] + [MemberData(nameof(TensorLengths))] + public static void MaxMagnitude_TwoTensors_SpecialValues(int tensorLength) + { + using BoundedMemory x = CreateAndFillTensor(tensorLength); + using BoundedMemory y = CreateAndFillTensor(tensorLength); + using BoundedMemory destination = CreateTensor(tensorLength); + + // NaNs + x[s_random.Next(x.Length)] = float.NaN; + y[s_random.Next(y.Length)] = float.NaN; + + // Same magnitude, opposite sign + int pos = s_random.Next(x.Length); + x[pos] = -5f; + y[pos] = 5f; + + // Positive and negative 0s + pos = s_random.Next(x.Length); + x[pos] = 0f; + y[pos] = -0f; + + TensorPrimitives.MaxMagnitude(x, y, destination); + for (int i = 0; i < tensorLength; i++) + { + Assert.Equal(MathFMaxMagnitude(x[i], y[i]), destination[i], Tolerance); + } + + TensorPrimitives.MaxMagnitude(y, x, destination); + for (int i = 0; i < tensorLength; i++) + { + Assert.Equal(MathFMaxMagnitude(y[i], x[i]), destination[i], Tolerance); } } @@ -962,9 +1041,10 @@ public static void Min_Tensor(int tensorLength) [MemberData(nameof(TensorLengths))] public static void Min_Tensor_NanReturned(int tensorLength) { - using BoundedMemory x = CreateAndFillTensor(tensorLength); + using BoundedMemory x = CreateTensor(tensorLength); foreach (int expected in new[] { 0, tensorLength / 2, tensorLength - 1 }) { + FillTensor(x); x[expected] = float.NaN; Assert.Equal(float.NaN, TensorPrimitives.Min(x)); } @@ -995,6 +1075,41 @@ public static void Min_TwoTensors(int tensorLength) } } + [Theory] + [MemberData(nameof(TensorLengths))] + public static void Min_TwoTensors_SpecialValues(int tensorLength) + { + using BoundedMemory x = CreateAndFillTensor(tensorLength); + using BoundedMemory y = CreateAndFillTensor(tensorLength); + using BoundedMemory destination = CreateTensor(tensorLength); + + // NaNs + x[s_random.Next(x.Length)] = float.NaN; + y[s_random.Next(y.Length)] = float.NaN; + + // Same magnitude, opposite sign + int pos = s_random.Next(x.Length); + x[pos] = -5f; + y[pos] = 5f; + + // Positive and negative 0s + pos = s_random.Next(x.Length); + x[pos] = 0f; + y[pos] = -0f; + + TensorPrimitives.Min(x, y, destination); + for (int i = 0; i < tensorLength; i++) + { + Assert.Equal(MathF.Min(x[i], y[i]), destination[i], Tolerance); + } + + TensorPrimitives.Min(y, x, destination); + for (int i = 0; i < tensorLength; i++) + { + Assert.Equal(MathF.Min(y[i], x[i]), destination[i], Tolerance); + } + } + [Theory] [MemberData(nameof(TensorLengths))] public static void Min_TwoTensors_ThrowsForMismatchedLengths(int tensorLength) @@ -1032,25 +1147,23 @@ public static void MinMagnitude_Tensor(int tensorLength) { using BoundedMemory x = CreateAndFillTensor(tensorLength); - int index = 0; - for (int i = 0; i < x.Length; i++) + float minMagnitude = x[0]; + foreach (float i in x.Span) { - if (MathF.Abs(x[i]) < MathF.Abs(x[index])) - { - index = i; - } + minMagnitude = MathFMinMagnitude(minMagnitude, i); } - Assert.Equal(x[index], TensorPrimitives.MinMagnitude(x), Tolerance); + Assert.Equal(minMagnitude, TensorPrimitives.MinMagnitude(x), Tolerance); } [Theory] [MemberData(nameof(TensorLengths))] public static void MinMagnitude_Tensor_NanReturned(int tensorLength) { - using BoundedMemory x = CreateAndFillTensor(tensorLength); + using BoundedMemory x = CreateTensor(tensorLength); foreach (int expected in new[] { 0, tensorLength / 2, tensorLength - 1 }) { + FillTensor(x); x[expected] = float.NaN; Assert.Equal(float.NaN, TensorPrimitives.MinMagnitude(x)); } @@ -1077,7 +1190,42 @@ public static void MinMagnitude_TwoTensors(int tensorLength) for (int i = 0; i < tensorLength; i++) { - Assert.Equal(MathF.Abs(x[i]) < MathF.Abs(y[i]) ? x[i] : y[i], destination[i], Tolerance); + Assert.Equal(MathFMinMagnitude(x[i], y[i]), destination[i], Tolerance); + } + } + + [Theory] + [MemberData(nameof(TensorLengths))] + public static void MinMagnitude_TwoTensors_SpecialValues(int tensorLength) + { + using BoundedMemory x = CreateAndFillTensor(tensorLength); + using BoundedMemory y = CreateAndFillTensor(tensorLength); + using BoundedMemory destination = CreateTensor(tensorLength); + + // NaNs + x[s_random.Next(x.Length)] = float.NaN; + y[s_random.Next(y.Length)] = float.NaN; + + // Same magnitude, opposite sign + int pos = s_random.Next(x.Length); + x[pos] = -5f; + y[pos] = 5f; + + // Positive and negative 0s + pos = s_random.Next(x.Length); + x[pos] = 0f; + y[pos] = -0f; + + TensorPrimitives.MinMagnitude(x, y, destination); + for (int i = 0; i < tensorLength; i++) + { + Assert.Equal(MathFMinMagnitude(x[i], y[i]), destination[i], Tolerance); + } + + TensorPrimitives.MinMagnitude(y, x, destination); + for (int i = 0; i < tensorLength; i++) + { + Assert.Equal(MathFMinMagnitude(y[i], x[i]), destination[i], Tolerance); } }