Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add vectorized implementation of hex encoding/decoding #39702

Closed
wants to merge 2 commits into from
Closed

Add vectorized implementation of hex encoding/decoding #39702

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9c30e8a
Add vectorized implementation of hex encoding/decoding
tkp1n Jul 21, 2020
3ed41b5
Address initial review feedback
tkp1n Jul 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 238 additions & 7 deletions src/libraries/Common/src/System/HexConverter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@

using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;

#if SYSTEM_PRIVATE_CORELIB
using Internal.Runtime.CompilerServices;
#endif

namespace System
{
Expand Down Expand Up @@ -83,16 +90,85 @@ public static void ToCharsBuffer(byte value, Span<char> buffer, int startingInde
buffer[startingIndex] = (char)(packedResult >> 8);
}

public static void EncodeToUtf16(ReadOnlySpan<byte> bytes, Span<char> chars, Casing casing = Casing.Upper)
private static unsafe int ToCharsBufferAvx2(byte* bytes, int bytesCount, char* chars, Casing casing)
{
Debug.Assert(chars.Length >= bytes.Length * 2);
Debug.Assert(Avx2.IsSupported);
Debug.Assert(bytesCount >= 32);

Vector256<byte> x00 = Vector256<byte>.Zero;
Vector256<byte> x0F = Vector256.Create((byte)0x0F);
Vector256<byte> hexLookupTable = casing == Casing.Lower ?
CreateVector(LowerHexLookupTable) :
CreateVector(UpperHexLookupTable);

int bytesToRead = RoundDownToNext32(bytesCount);
byte* eof = bytes + bytesToRead;
byte* charsAsByte = (byte*)chars;
do
{
Vector256<byte> value = Avx.LoadVector256(bytes);
bytes += 32;

Vector256<byte> hiShift = Avx2.ShiftRightLogical(value.AsInt16(), 4).AsByte();
Vector256<byte> loHalf = Avx2.And(value, x0F);
Vector256<byte> hiHalf = Avx2.And(hiShift, x0F);
Vector256<byte> lo02 = Avx2.UnpackLow(hiHalf, loHalf);
Vector256<byte> hi13 = Avx2.UnpackHigh(hiHalf, loHalf);

Vector256<byte> resLo = Avx2.Shuffle(hexLookupTable, lo02);
Vector256<byte> resHi = Avx2.Shuffle(hexLookupTable, hi13);

Vector256<byte> ae = Avx2.UnpackLow(resLo, x00);
Vector256<byte> bf = Avx2.UnpackHigh(resLo, x00);
Vector256<byte> cg = Avx2.UnpackLow(resHi, x00);
Vector256<byte> dh = Avx2.UnpackHigh(resHi, x00);

Vector256<byte> ab = Avx2.Permute2x128(ae, bf, 0b0010_0000);
Vector256<byte> ef = Avx2.Permute2x128(ae, bf, 0b0011_0001);
Vector256<byte> cd = Avx2.Permute2x128(cg, dh, 0b0010_0000);
Vector256<byte> gh = Avx2.Permute2x128(cg, dh, 0b0011_0001);

Avx.Store(charsAsByte, ab);
charsAsByte += 32;
Avx.Store(charsAsByte, cd);
charsAsByte += 32;
Avx.Store(charsAsByte, ef);
charsAsByte += 32;
Avx.Store(charsAsByte, gh);
charsAsByte += 32;
} while (bytes != eof);

return bytesToRead;
}

for (int pos = 0; pos < bytes.Length; ++pos)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void EncodeToUtf16Core(byte* bytes, int bytesCount, Span<char> chars, Casing casing)
{
int pos = 0;
if (Avx2.IsSupported && bytesCount >= 32)
{
Debug.Assert(!chars.IsEmpty);
fixed (char* charPtr = &MemoryMarshal.GetReference(chars))
{
pos = ToCharsBufferAvx2(bytes, bytesCount, charPtr, casing);
}
Debug.Assert(pos == (bytesCount / 32) * 32);
}
for (; pos < bytesCount; ++pos)
{
ToCharsBuffer(bytes[pos], chars, pos * 2, casing);
}
}

public static unsafe void EncodeToUtf16(ReadOnlySpan<byte> bytes, Span<char> chars, Casing casing = Casing.Upper)
{
Debug.Assert(chars.Length >= bytes.Length * 2);
fixed (byte* bytesPtr = bytes)
{
EncodeToUtf16Core(bytesPtr, bytes.Length, chars, casing);
}
}

#if ALLOW_PARTIALLY_TRUSTED_CALLERS
[System.Security.SecuritySafeCriticalAttribute]
#endif
Expand Down Expand Up @@ -120,10 +196,9 @@ public static unsafe string ToString(ReadOnlySpan<byte> bytes, Casing casing = C
#else
fixed (byte* bytesPtr = bytes)
{
return string.Create(bytes.Length * 2, (Ptr: (IntPtr)bytesPtr, bytes.Length, casing), static (chars, args) =>
return string.Create(bytes.Length * 2, (Ptr: (IntPtr)bytesPtr, bytes.Length, casing), (chars, args) =>
{
var ros = new ReadOnlySpan<byte>((byte*)args.Ptr, args.Length);
EncodeToUtf16(ros, chars, args.casing);
EncodeToUtf16Core((byte*)args.Ptr, args.Length, chars, args.casing);
});
}
#endif
Expand Down Expand Up @@ -162,13 +237,25 @@ public static bool TryDecodeFromUtf16(ReadOnlySpan<char> chars, Span<byte> bytes
return TryDecodeFromUtf16(chars, bytes, out _);
}

public static bool TryDecodeFromUtf16(ReadOnlySpan<char> chars, Span<byte> bytes, out int charsProcessed)
public static unsafe bool TryDecodeFromUtf16(ReadOnlySpan<char> chars, Span<byte> bytes, out int charsProcessed)
{
Debug.Assert(chars.Length % 2 == 0, "Un-even number of characters provided");
Debug.Assert(chars.Length / 2 == bytes.Length, "Target buffer not right-sized for provided characters");

int i = 0;
int j = 0;
if (Avx2.IsSupported && bytes.Length > 32)
{
fixed (char* charPtr = &MemoryMarshal.GetReference(chars))
fixed (byte* bytePtr = &MemoryMarshal.GetReference(bytes))
{
j = DecodeFromUtf16Avx2(charPtr, bytePtr, bytes.Length);
Debug.Assert(j % 32 == 0);
Debug.Assert(j <= bytes.Length);
i = j * 2;
}
}

int byteLo = 0;
int byteHi = 0;
while (j < bytes.Length)
Expand All @@ -192,6 +279,92 @@ public static bool TryDecodeFromUtf16(ReadOnlySpan<char> chars, Span<byte> bytes
return (byteLo | byteHi) != 0xFF;
}

private static unsafe int DecodeFromUtf16Avx2(char* chars, byte* bytes, int bytesCount)
{
Debug.Assert(Avx2.IsSupported);

Vector256<byte> x0F = Vector256.Create((byte) 0x0F);
Vector256<byte> xF0 = Vector256.Create((byte) 0xF0);
Vector256<byte> digHexSelector = CreateVector(UpperLowerDigHexSelector);
Vector256<byte> digits = CreateVector(Digits);
Vector256<byte> hexs = CreateVector(Hexs);
Vector256<byte> evenBytes = CreateVector(EvenBytes);
Vector256<byte> oddBytes = CreateVector(OddBytes);

int bytesToWrite = RoundDownToNext32(bytesCount);
byte* eof = bytes + bytesToWrite;
byte* dest = bytes;
byte* charsAsByte = (byte*)chars;
int leftOk, rightOk;
while (dest != eof)
{
Vector256<short> a = Avx.LoadVector256(charsAsByte).AsInt16();
charsAsByte += 32;
Vector256<short> b = Avx.LoadVector256(charsAsByte).AsInt16();
charsAsByte += 32;
Vector256<short> c = Avx.LoadVector256(charsAsByte).AsInt16();
charsAsByte += 32;
Vector256<short> d = Avx.LoadVector256(charsAsByte).AsInt16();
charsAsByte += 32;

Vector256<byte> ab = Avx2.PackUnsignedSaturate(a, b);
Vector256<byte> cd = Avx2.PackUnsignedSaturate(c, d);

Vector256<byte> inputLeft = Avx2.Permute4x64(ab.AsUInt64(), 0b11_01_10_00).AsByte();
Vector256<byte> inputRight = Avx2.Permute4x64(cd.AsUInt64(), 0b11_01_10_00).AsByte();

Vector256<byte> loNibbleLeft = Avx2.And(inputLeft, x0F);
Vector256<byte> loNibbleRight = Avx2.And(inputRight, x0F);

Vector256<byte> hiNibbleLeft = Avx2.And(inputLeft, xF0);
Vector256<byte> hiNibbleRight = Avx2.And(inputRight, xF0);

Vector256<byte> leftDigits = Avx2.Shuffle(digits, loNibbleLeft);
Vector256<byte> leftHex = Avx2.Shuffle(hexs, loNibbleLeft);

Vector256<byte> hiNibbleShLeft = Avx2.ShiftRightLogical(hiNibbleLeft.AsInt16(), 4).AsByte();
Vector256<byte> hiNibbleShRight = Avx2.ShiftRightLogical(hiNibbleRight.AsInt16(), 4).AsByte();

Vector256<byte> rightDigits = Avx2.Shuffle(digits, loNibbleRight);
Vector256<byte> rightHex = Avx2.Shuffle(hexs, loNibbleRight);

Vector256<byte> magicLeft = Avx2.Shuffle(digHexSelector, hiNibbleShLeft);
Vector256<byte> magicRight = Avx2.Shuffle(digHexSelector, hiNibbleShRight);

Vector256<byte> valueLeft = Avx2.BlendVariable(leftDigits, leftHex, magicLeft);
Vector256<byte> valueRight = Avx2.BlendVariable(rightDigits, rightHex, magicRight);

Vector256<byte> errLeft = Avx2.ShiftLeftLogical(magicLeft.AsInt16(), 7).AsByte();
Vector256<byte> errRight = Avx2.ShiftLeftLogical(magicRight.AsInt16(), 7).AsByte();

Vector256<byte> evenBytesLeft = Avx2.Shuffle(valueLeft, evenBytes);
Vector256<byte> oddBytesLeft = Avx2.Shuffle(valueLeft, oddBytes);
Vector256<byte> evenBytesRight = Avx2.Shuffle(valueRight, evenBytes);
Vector256<byte> oddBytesRight = Avx2.Shuffle(valueRight, oddBytes);

evenBytesLeft = Avx2.ShiftLeftLogical(evenBytesLeft.AsUInt16(), 4).AsByte();
evenBytesRight = Avx2.ShiftLeftLogical(evenBytesRight.AsUInt16(), 4).AsByte();

evenBytesLeft = Avx2.Or(evenBytesLeft, oddBytesLeft);
evenBytesRight = Avx2.Or(evenBytesRight, oddBytesRight);

Vector256<byte> result = Merge(evenBytesLeft, evenBytesRight);

Vector256<byte> validationResultLeft = Avx2.Or(errLeft, valueLeft);
Vector256<byte> validationResultRight = Avx2.Or(errRight, valueRight);

leftOk = Avx2.MoveMask(validationResultLeft);
rightOk = Avx2.MoveMask(validationResultRight);

if ((leftOk | rightOk) != 0) break;

Avx.Store(dest, result);
dest += 32;
}

return bytesToWrite - (int) (eof - dest);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int FromChar(int c)
{
Expand Down Expand Up @@ -254,5 +427,63 @@ public static bool IsHexLowerChar(int c)
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 239
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 255
};

private static ReadOnlySpan<byte> LowerHexLookupTable => new byte[]
{
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

** not relevant yet -- only when SSE2 comes into play **

Had a quick look at HexMate, and saw that (as expected) the constants for AVX2 just doubles SSE2 in the upper lane.

So in order to save some (disk) space for the assembly, one could use this outline:

Vector128<byte> sseConstant = ReadVector(Sse2ContantData);
Vector256<byte> avxConstant = Vector256.Create(sseConstant, sseConstant);

In essence it's about avoiding to have duplicated data in the "text-section" of the DLL.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, I added your suggestion to the HexMate repo (here) 👍

I let this open as a reminder in case we get to SSE2 in this PR.

0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66
};

private static ReadOnlySpan<byte> UpperHexLookupTable => new byte[]
{
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46
};

private static ReadOnlySpan<byte> UpperLowerDigHexSelector => new byte[]
{
0x01, 0x01, 0x01, 0x00, 0x80, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x00, 0x80, 0x01, 0x80, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01
};

private static ReadOnlySpan<byte> Digits => new byte[]
{
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
};

private static ReadOnlySpan<byte> Hexs => new byte[]
{
0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
};

private static ReadOnlySpan<byte> EvenBytes => new byte[]
{
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
tkp1n marked this conversation as resolved.
Show resolved Hide resolved
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
};

private static ReadOnlySpan<byte> OddBytes => new byte[]
{
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
};

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int RoundDownToNext32(int x)
=> x & 0x7FFFFFE0;

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<byte> CreateVector(ReadOnlySpan<byte> data)
=> Unsafe.ReadUnaligned<Vector256<byte>>(ref MemoryMarshal.GetReference(data));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<byte> Merge(Vector256<byte> a, Vector256<byte> b)
{
Vector256<ulong> a1 = Avx2.Permute4x64(a.AsUInt64(), 0b11_10_10_00);
Vector256<ulong> b1 = Avx2.Permute4x64(b.AsUInt64(), 0b11_00_01_00);
return Avx2.Blend(a1.AsUInt32(), b1.AsUInt32(), 0b1111_0000).AsByte();
}
}
}