Add support for reading and writing the .NET Half type

cpp/LogicalType.cpp

@@ -106,6 +106,11 @@ extern "C"
 		TRYCATCH(*logical_type = new std::shared_ptr(LogicalType::UUID());)
 	}
 
+	PARQUETSHARP_EXPORT ExceptionInfo* LogicalType_Float16(const std::shared_ptr<const LogicalType>** logical_type)
+	{
+		TRYCATCH(*logical_type = new std::shared_ptr(LogicalType::Float16());)
+	}
+
 	PARQUETSHARP_EXPORT ExceptionInfo* LogicalType_None(const std::shared_ptr<const LogicalType>** logical_type)
 	{
 		TRYCATCH(*logical_type = new std::shared_ptr(LogicalType::None());)

csharp.test/TestLogicalTypeRoundtrip.cs

@@ -1632,6 +1632,30 @@
                     HasStatistics = false
                 },
                 new ExpectedColumn
+                {
+                    Name = "half_field",
+                    PhysicalType = PhysicalType.FixedLenByteArray,
+                    LogicalType = LogicalType.Float16(),
+                    Length = 2,
+                    Values = Enumerable.Range(0, NumRows).Select(i => i % 5 == 0 ? Half.NaN : (Half) Math.Sqrt(i)).ToArray(),
+                    Min = (Half) 1.0,
+                    Max = (Half) Math.Sqrt(NumRows - 1),
+                    Converter = (v, _) => LogicalRead.ToHalf((FixedLenByteArray) v)
+                },
+                new ExpectedColumn
+                {
+                    Name = "half?_field",
+                    PhysicalType = PhysicalType.FixedLenByteArray,
+                    LogicalType = LogicalType.Float16(),
+                    Length = 2,
+                    Values = Enumerable.Range(0, NumRows).Select(i => i % 11 == 0 ? (Half?) null : i % 5 == 0 ? Half.NaN : (Half) Math.Sqrt(i)).ToArray(),
+                    NullCount = (NumRows + 10) / 11,
+                    NumValues = NumRows - (NumRows + 10) / 11,
+                    Min = (Half) 1.0,
+                    Max = (Half) Math.Sqrt(NumRows - 1),
+                    Converter = (v, _) => LogicalRead.ToHalf((FixedLenByteArray) v)
+                },
+                new ExpectedColumn
                 {
                     Name = "float_field",
                     PhysicalType = PhysicalType.Float,

csharp/AadPrefixVerifier.cs

@@ -26,7 +26,7 @@ internal IntPtr CreateGcHandle()
 
         internal static AadPrefixVerifier GetGcHandleTarget(IntPtr handle)
         {
-            return (AadPrefixVerifier) GCHandle.FromIntPtr(handle).Target;
+            return (AadPrefixVerifier) GCHandle.FromIntPtr(handle).Target!;
         }
 
         internal delegate void FreeGcHandleFunc(IntPtr handle);
@@ -46,7 +46,7 @@ private static void Verify(IntPtr handle, IntPtr aadPrefix, out string? exceptio
 
             try
             {
-                var obj = (AadPrefixVerifier) GCHandle.FromIntPtr(handle).Target;
+                var obj = (AadPrefixVerifier) GCHandle.FromIntPtr(handle).Target!;
                 var aadPrefixStr = StringUtil.PtrToStringUtf8(aadPrefix);
                 obj.Verify(aadPrefixStr);
             }

csharp/ByteArray.cs

@@ -23,7 +23,7 @@ public bool Equals(ByteArray other)
             return Length == other.Length && Pointer == other.Pointer;
         }
 
-        public override bool Equals(object obj)
+        public override bool Equals(object? obj)
         {
             return obj is ByteArray other && Equals(other);
         }

csharp/ByteArrayReaderCache.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Runtime.CompilerServices;
 
@@ -40,7 +41,11 @@ public TLogical Add(TPhysical physical, TLogical logical)
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#if (NETSTANDARD2_1_OR_GREATER || NET5_0_OR_GREATER)
+        public bool TryGetValue(TPhysical physical, [MaybeNullWhen(false)] out TLogical logical)
+#else
         public bool TryGetValue(TPhysical physical, out TLogical logical)
+#endif
         {
             if (_map == null) throw new InvalidOperationException("cache is not in a usable state");
             return _map.TryGetValue(physical, out logical);

csharp/Column.cs

@@ -21,10 +21,15 @@ public Column(Type logicalSystemType, string name, LogicalType? logicalTypeOverr
         {
             var isDecimal = logicalSystemType == typeof(decimal) || logicalSystemType == typeof(decimal?);
             var isUuid = logicalSystemType == typeof(Guid) || logicalSystemType == typeof(Guid?);
+#if NET5_0_OR_GREATER
+            var isHalf = logicalSystemType == typeof(Half) || logicalSystemType == typeof(Half?);
+#else
+            var isHalf = false;
+#endif
 
-            if (length != -1 && !isDecimal && !isUuid)
+            if (length != -1 && !(isDecimal || isUuid || isHalf))
             {
-                throw new ArgumentException("length can only be set with the decimal or Guid type");
+                throw new ArgumentException("length can only be set with the decimal, Guid or Half type");
             }
 
             if (isDecimal && !(logicalTypeOverride is DecimalLogicalType))
@@ -106,6 +111,13 @@ private static unsafe int GetTypeLength(Type logicalSystemType)
                 return 16;
             }
 
+#if NET5_0_OR_GREATER
+            if (logicalSystemType == typeof(Half) || logicalSystemType == typeof(Half?))
+            {
+                return 2;
+            }
+#endif
+
             return -1;
         }
 
@@ -119,7 +131,7 @@ private static Node CreateSchemaNode(LogicalTypeFactory logicalTypeFactory, Type
 
             if (type.IsArray)
             {
-                var item = CreateSchemaNode(logicalTypeFactory, type.GetElementType(), "item", logicalTypeOverride, length);
+                var item = CreateSchemaNode(logicalTypeFactory, type.GetElementType()!, "item", logicalTypeOverride, length);
                 var list = new GroupNode("list", Repetition.Repeated, new[] {item});
 
                 try

csharp/Date.cs

@@ -39,7 +39,7 @@ public bool Equals(Date other)
             return Days == other.Days;
         }
 
-        public override bool Equals(object obj)
+        public override bool Equals(object? obj)
         {
             return obj is Date date && Equals(date);
         }
@@ -49,7 +49,7 @@ public override int GetHashCode()
             return Days;
         }
 
-        public int CompareTo(object obj)
+        public int CompareTo(object? obj)
         {
             return obj switch
             {

csharp/DateTimeNanos.cs

@@ -35,7 +35,7 @@ public bool Equals(DateTimeNanos other)
             return Ticks == other.Ticks;
         }
 
-        public override bool Equals(object obj)
+        public override bool Equals(object? obj)
         {
             return obj is DateTimeNanos date && Equals(date);
         }
@@ -45,7 +45,7 @@ public override int GetHashCode()
             return Ticks.GetHashCode();
         }
 
-        public int CompareTo(object obj)
+        public int CompareTo(object? obj)
         {
             return obj switch
             {

csharp/DecryptionKeyRetriever.cs

@@ -25,7 +25,7 @@ internal IntPtr CreateGcHandle()
 
         internal static DecryptionKeyRetriever GetGcHandleTarget(IntPtr handle)
         {
-            return (DecryptionKeyRetriever) GCHandle.FromIntPtr(handle).Target;
+            return (DecryptionKeyRetriever) GCHandle.FromIntPtr(handle).Target!;
         }
 
         internal delegate void FreeGcHandleFunc(IntPtr handle);
@@ -45,7 +45,7 @@ private static void GetKey(IntPtr handle, IntPtr keyMetadata, out AesKey key, ou
 
             try
             {
-                var obj = (DecryptionKeyRetriever) GCHandle.FromIntPtr(handle).Target;
+                var obj = (DecryptionKeyRetriever) GCHandle.FromIntPtr(handle).Target!;
                 var keyMetadataStr = StringUtil.PtrToStringUtf8(keyMetadata);
                 key = new AesKey(obj.GetKey(keyMetadataStr));
             }

csharp/FixedLenByteArray.cs

@@ -21,7 +21,7 @@ public bool Equals(FixedLenByteArray other)
             return Pointer == other.Pointer;
         }
 
-        public override bool Equals(object obj)
+        public override bool Equals(object? obj)
         {
             return obj is FixedLenByteArray other && Equals(other);
         }

csharp/LogicalBatchReader/LogicalBatchReaderFactory.cs

@@ -168,7 +168,7 @@ private ILogicalBatchReader<TElement> MakeArrayReader<TElement>(
 
             var arrayReaderType = typeof(ArrayReader<,,>).MakeGenericType(typeof(TPhysical), typeof(TLogical), containedType);
             return (ILogicalBatchReader<TElement>) Activator.CreateInstance(
-                arrayReaderType, innerReader, _bufferedReader!, arrayDefinitionLevel, repetitionLevel, innerNodeIsOptional);
+                arrayReaderType, innerReader, _bufferedReader!, arrayDefinitionLevel, repetitionLevel, innerNodeIsOptional)!;
         }
 
         /// <summary>
@@ -185,7 +185,7 @@ private ILogicalBatchReader<TElement> MakeNestedReader<TElement>(
             var innerReader = MakeGenericReader(nestedType, innerSchema, definitionLevel, repetitionLevel);
 
             var nestedReaderType = typeof(NestedReader<>).MakeGenericType(nestedType);
-            return (ILogicalBatchReader<TElement>) Activator.CreateInstance(nestedReaderType, innerReader, _buffers.Length);
+            return (ILogicalBatchReader<TElement>) Activator.CreateInstance(nestedReaderType, innerReader, _buffers.Length)!;
         }
 
         /// <summary>
@@ -205,7 +205,7 @@ private ILogicalBatchReader<TElement> MakeNestedOptionalReader<TElement>(
             var optionalNestedReaderType = typeof(OptionalNestedReader<,,>).MakeGenericType(
                 typeof(TPhysical), typeof(TLogical), nestedType);
             return (ILogicalBatchReader<TElement>) Activator.CreateInstance(
-                optionalNestedReaderType, innerReader, _bufferedReader!, definitionLevel);
+                optionalNestedReaderType, innerReader, _bufferedReader!, definitionLevel)!;
         }
 
         /// <summary>
@@ -223,7 +223,7 @@ private ILogicalBatchReader<TElement> MakeOptionalReader<TElement>(
             var optionalReaderType = typeof(OptionalReader<,,>).MakeGenericType(
                 typeof(TPhysical), typeof(TLogical), innerType);
             return (ILogicalBatchReader<TElement>) Activator.CreateInstance(
-                optionalReaderType, innerReader, _bufferedReader!, definitionLevel);
+                optionalReaderType, innerReader, _bufferedReader!, definitionLevel)!;
         }
 
         /// <summary>
@@ -246,7 +246,7 @@ private object MakeGenericReader(
             return genericMethod.MakeGenericMethod(elementType).Invoke(this, new object[]
             {
                 schemaNodes, nullDefinitionLevel, repetitionLevel
-            });
+            })!;
         }
 
         private readonly ColumnReader<TPhysical> _physicalReader;

csharp/LogicalBatchWriter/LogicalBatchWriterFactory.cs

@@ -121,7 +121,7 @@ private ILogicalBatchWriter<TElement> MakeArrayWriter<TElement>(
             var arrayWriterType = typeof(ArrayWriter<,>).MakeGenericType(containedType, typeof(TPhysical));
             return (ILogicalBatchWriter<TElement>) Activator.CreateInstance(
                 arrayWriterType, writer0, writer1, _physicalWriter, optional,
-                arrayDefinitionLevel, repetitionLevel, firstRepetitionLevel);
+                arrayDefinitionLevel, repetitionLevel, firstRepetitionLevel)!;
         }
 
         /// <summary>
@@ -141,7 +141,7 @@ private ILogicalBatchWriter<TElement> MakeNestedWriter<TElement>(
 
             var nestedWriterType = typeof(NestedWriter<>).MakeGenericType(nestedType);
             return (ILogicalBatchWriter<TElement>) Activator.CreateInstance(
-                nestedWriterType, firstInnerWriter, innerWriter, _buffers.Length);
+                nestedWriterType, firstInnerWriter, innerWriter, _buffers.Length)!;
         }
 
         /// <summary>
@@ -163,7 +163,7 @@ private ILogicalBatchWriter<TElement> MakeNestedOptionalWriter<TElement>(
             var optionalNestedWriterType = typeof(OptionalNestedWriter<,>).MakeGenericType(nestedType, typeof(TPhysical));
             return (ILogicalBatchWriter<TElement>) Activator.CreateInstance(
                 optionalNestedWriterType, firstInnerWriter, innerWriter, _physicalWriter, _buffers,
-                definitionLevel, repetitionLevel, firstRepetitionLevel);
+                definitionLevel, repetitionLevel, firstRepetitionLevel)!;
         }
 
         /// <summary>
@@ -187,7 +187,7 @@ private object MakeGenericWriter(
             return genericMethod.MakeGenericMethod(elementType).Invoke(this, new object[]
             {
                 schemaNodes, nullDefinitionLevel, repetitionLevel, firstRepetitionLevel
-            });
+            })!;
         }
 
         private readonly ByteBuffer? _byteBuffer;

csharp/LogicalRead.cs

@@ -167,6 +167,18 @@ public static Delegate GetConverter(ColumnDescriptor columnDescriptor, ColumnChu
                 return (LogicalRead<Guid?, FixedLenByteArray>.Converter) LogicalRead.ConvertUuid;
             }
 
+#if NET5_0_OR_GREATER
+            if (typeof(TLogical) == typeof(Half))
+            {
+                return (LogicalRead<Half, FixedLenByteArray>.Converter) ((s, _, d, _) => LogicalRead.ConvertHalf(s, d));
+            }
+
+            if (typeof(TLogical) == typeof(Half?))
+            {
+                return (LogicalRead<Half?, FixedLenByteArray>.Converter) LogicalRead.ConvertHalf;
+            }
+#endif
+
             if (typeof(TLogical) == typeof(Date))
             {
                 return LogicalRead.GetNativeConverter<Date, int>();
@@ -478,6 +490,24 @@ public static void ConvertUuid(ReadOnlySpan<FixedLenByteArray> source, ReadOnlyS
             }
         }
 
+#if NET5_0_OR_GREATER
+        public static void ConvertHalf(ReadOnlySpan<FixedLenByteArray> source, Span<Half> destination)
+        {
+            for (int i = 0; i < destination.Length; ++i)
+            {
+                destination[i] = ToHalf(source[i]);
+            }
+        }
+
+        public static void ConvertHalf(ReadOnlySpan<FixedLenByteArray> source, ReadOnlySpan<short> defLevels, Span<Half?> destination, short definedLevel)
+        {
+            for (int i = 0, src = 0; i < destination.Length; ++i)
+            {
+                destination[i] = defLevels[i] != definedLevel ? default(Half?) : ToHalf(source[src++]);
+            }
+        }
+#endif
+
         public static void ConvertDateTimeMicros(ReadOnlySpan<long> source, Span<DateTime> destination, DateTimeKind kind = DateTimeKind.Unspecified)
         {
             for (int i = 0; i < destination.Length; ++i)
@@ -630,6 +660,27 @@ public static unsafe Guid ToUuid(FixedLenByteArray source)
             }
         }
 
+#if NET5_0_OR_GREATER
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static unsafe Half ToHalf(FixedLenByteArray source)
+        {
+            if (BitConverter.IsLittleEndian)
+            {
+                return *(Half*) source.Pointer;
+            }
+            else
+            {
+                // Float-16 values are always stored in little-endian order
+                var value = (Half) 0.0f;
+                var dest = (byte*) &value;
+                var src = (byte*) source.Pointer;
+                dest[1] = src[0];
+                dest[0] = src[1];
+                return value;
+            }
+        }
+#endif
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static DateTime ToDateTimeMicros(long source)
         {

csharp/LogicalType.cs

@@ -68,6 +68,15 @@ public override string ToString()
         public static LogicalType Json() => Create(ExceptionInfo.Return<IntPtr>(LogicalType_JSON));
         public static LogicalType Bson() => Create(ExceptionInfo.Return<IntPtr>(LogicalType_BSON));
         public static LogicalType Uuid() => Create(ExceptionInfo.Return<IntPtr>(LogicalType_UUID));
+
+        /// <summary>
+        /// Create a Float16 LogicalType
+        ///
+        /// This can be used to annotate a 2-byte fixed-length byte array
+        /// </summary>
+        /// <returns>New Float16 LogicalType</returns>
+        public static LogicalType Float16() => Create(ExceptionInfo.Return<IntPtr>(LogicalType_Float16));
+
         public static LogicalType None() => Create(ExceptionInfo.Return<IntPtr>(LogicalType_None));
 
         internal static LogicalType Create(IntPtr handle)
@@ -95,6 +104,7 @@ internal static LogicalType Create(IntPtr handle)
                 LogicalTypeEnum.Json => new JsonLogicalType(handle),
                 LogicalTypeEnum.Bson => new BsonLogicalType(handle),
                 LogicalTypeEnum.Uuid => new UuidLogicalType(handle),
+                LogicalTypeEnum.Float16 => new Float16LogicalType(handle),
                 LogicalTypeEnum.None => new NoneLogicalType(handle),
                 _ => throw new ArgumentOutOfRangeException($"unknown logical type {type}")
             };
@@ -157,6 +167,9 @@ internal static LogicalType Create(IntPtr handle)
         [DllImport(ParquetDll.Name)]
         private static extern IntPtr LogicalType_UUID(out IntPtr logicalType);
 
+        [DllImport(ParquetDll.Name)]
+        private static extern IntPtr LogicalType_Float16(out IntPtr logicalType);
+
         [DllImport(ParquetDll.Name)]
         private static extern IntPtr LogicalType_None(out IntPtr logicalType);
 
@@ -277,6 +290,11 @@ public sealed class UuidLogicalType : LogicalType
         internal UuidLogicalType(IntPtr handle) : base(handle) { }
     }
 
+    public sealed class Float16LogicalType : LogicalType
+    {
+        internal Float16LogicalType(IntPtr handle) : base(handle) { }
+    }
+
     public sealed class NoneLogicalType : LogicalType
     {
         internal NoneLogicalType(IntPtr handle) : base(handle) { }