Add ReadOnlySequence support to Utf8JsonReader with unit tests

src/System.Text.Json/ref/System.Text.Json.cs

@@ -31,6 +31,7 @@ public partial struct JsonReaderState
         public JsonReaderState(int maxDepth = 64, System.Text.Json.JsonReaderOptions options = default(System.Text.Json.JsonReaderOptions)) { throw null; }
         public long BytesConsumed { get { throw null; } }
         public int MaxDepth { get { throw null; } }
+        public SequencePosition Position { get { throw null; } }
         public System.Text.Json.JsonReaderOptions Options { get { throw null; } }
     }
     public enum JsonTokenType : byte
@@ -52,11 +53,15 @@ public ref partial struct Utf8JsonReader
     {
         private object _dummy;
         public Utf8JsonReader(System.ReadOnlySpan<byte> jsonData, bool isFinalBlock, System.Text.Json.JsonReaderState state) { throw null; }
+        public Utf8JsonReader(in System.Buffers.ReadOnlySequence<byte> jsonData, bool isFinalBlock, System.Text.Json.JsonReaderState state) { throw null; }
         public long BytesConsumed { get { throw null; } }
         public int CurrentDepth { get { throw null; } }
+        public bool HasValueSequence { get { throw null; } }
         public System.Text.Json.JsonReaderState CurrentState { get { throw null; } }
+        public SequencePosition Position { get { throw null; } }
         public System.Text.Json.JsonTokenType TokenType { get { throw null; } }
         public System.ReadOnlySpan<byte> ValueSpan { get { throw null; } }
+        public System.Buffers.ReadOnlySequence<byte> ValueSequence { get { throw null; } }
         public bool GetBooleanValue() { throw null; }
         public string GetStringValue() { throw null; }
         public bool Read() { throw null; }

src/System.Text.Json/src/System.Text.Json.csproj

@@ -9,6 +9,7 @@
   <ItemGroup>
     <Compile Include="System\Text\Json\ConsumeNumberResult.cs" />
     <Compile Include="System\Text\Json\ConsumeTokenResult.cs" />
+    <Compile Include="System\Text\Json\CustomUncheckedBitArray.cs" />
     <Compile Include="System\Text\Json\JsonCommentHandling.cs" />
     <Compile Include="System\Text\Json\JsonConstants.cs" />
     <Compile Include="System\Text\Json\JsonReaderException.cs" />
@@ -18,6 +19,7 @@
     <Compile Include="System\Text\Json\JsonTokenType.cs" />
     <Compile Include="System\Text\Json\ThrowHelper.cs" />
     <Compile Include="System\Text\Json\Utf8JsonReader.cs" />
+    <Compile Include="System\Text\Json\Utf8JsonReader.MultiSegment.cs" />
     <Compile Include="System\Text\Json\Utf8JsonReader.TryGet.cs" />
   </ItemGroup>
   <ItemGroup>

src/System.Text.Json/src/System/Text/Json/CustomUncheckedBitArray.cs

@@ -0,0 +1,122 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+
+namespace System.Text.Json
+{
+    // Inspired by BitArray
+    // https://github.com/dotnet/corefx/blob/master/src/System.Collections/src/System/Collections/BitArray.cs
+    internal struct CustomUncheckedBitArray
+    {
+        private int[] _array;
+
+        public int Length { get; private set; }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public CustomUncheckedBitArray(int bitLength, int integerLength)
+        {
+            Debug.Assert(bitLength > 0, $"bitLength: {bitLength}, integerLength: {integerLength}");
+            Debug.Assert(integerLength > 0 && integerLength <= int.MaxValue / 32 + 1, $"bitLength: {bitLength}, integerLength: {integerLength}");
+            Debug.Assert(bitLength <= (long)integerLength * 32, $"bitLength: {bitLength}, integerLength: {integerLength}");
+
+            _array = new int[integerLength];
+            Length = bitLength;
+        }
+
+        public bool this[int index]
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get
+            {
+                Debug.Assert(index >= 0 && index < Length, $"index: {index}, Length: {Length}");
+
+                int elementIndex = Div32Rem(index, out int extraBits);
+
+                Debug.Assert(elementIndex < _array.Length, $"index: {index}, Length: {Length}, elementIndex: {elementIndex}, arrayLength: {_array.Length}, extraBits: {extraBits}");
+
+                return (_array[elementIndex] & (1 << extraBits)) != 0;
+            }
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            set
+            {
+                Debug.Assert(index >= 0, $"index: {index}, Length: {Length}");
+
+                // Grow the array when setting a bit if it isn't big enough
+                // This way the caller doesn't have to check.
+                if (index >= Length)
+                {
+                    Grow(index);
+                }
+
+                int elementIndex = Div32Rem(index, out int extraBits);
+
+                Debug.Assert(elementIndex < _array.Length, $"index: {index}, Length: {Length}, elementIndex: {elementIndex}, arrayLength: {_array.Length}, extraBits: {extraBits}");
+
+                int newValue = _array[elementIndex];
+                if (value)
+                {
+                    newValue |= 1 << extraBits;
+                }
+                else
+                {
+                    newValue &= ~(1 << extraBits);
+                }
+                _array[elementIndex] = newValue;
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private void Grow(int index)
+        {
+            Debug.Assert(index >= Length, $"index: {index}, Length: {Length}");
+
+            // If index is a multiple of 32, add 1.
+            int newints = (index & 31) == 0 ? 1 : 0;
+
+            if ((uint)index / 2 < int.MaxValue)
+            {
+                index *= 2; // Grow by doubling, if possible.
+            }
+            else
+            {
+                index++;
+            }
+
+            newints += ((index - 1) >> 5) + 1;   // (value + 31) / 32 without overflow
+
+            if (newints > _array.Length)
+            {
+                var newArray = new int[newints];
+                _array.AsSpan().CopyTo(newArray);
+                _array = newArray;
+            }
+
+            if (index > Length)
+            {
+                // clear high bit values in the last int
+                int last = (Length - 1) >> 5;
+                Div32Rem(Length, out int bits);
+                if (bits > 0)
+                {
+                    _array[last] &= (1 << bits) - 1;
+                }
+
+                // clear remaining int values
+                _array.AsSpan(last + 1, newints - last - 1).Clear();
+            }
+
+            Length = index;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int Div32Rem(int number, out int remainder)
+        {
+            uint quotient = (uint)number / 32;
+            remainder = number & 31;    // number & 31 == number % 32
+            return (int)quotient;
+        }
+    }
+}

src/System.Text.Json/src/System/Text/Json/JsonReaderState.cs

@@ -17,16 +17,16 @@ namespace System.Text.Json
     public struct JsonReaderState
     {
         // We are using a ulong to represent our nested state, so we can only
-        // go 64 levels deep without having to allocate a stack.
-        internal const int StackFreeMaxDepth = sizeof(ulong) * 8;
+        // go 64 levels deep without having to allocate.
+        internal const int AllocationFreeMaxDepth = sizeof(ulong) * 8;
 
         // This ulong container represents a tiny stack to track the state during nested transitions.
         // The first bit represents the state of the current depth (1 == object, 0 == array).
         // Each subsequent bit is the parent / containing type (object or array). Since this
         // reader does a linear scan, we only need to keep a single path as we go through the data.
-        // This is primarily used as an optimization to avoid having to allocate a stack object for
+        // This is primarily used as an optimization to avoid having to allocate an object for
         // depths up to 64 (which is the default max depth).
-        internal ulong _stackFreeContainer;
+        internal ulong _allocationFreeContainer;
         internal long _lineNumber;
         internal long _bytePositionInLine;
         internal long _bytesConsumed;
@@ -37,14 +37,22 @@ public struct JsonReaderState
         internal JsonTokenType _tokenType;
         internal JsonTokenType _previousTokenType;
         internal JsonReaderOptions _readerOptions;
-        internal Stack<JsonTokenType> _stack;
+        internal CustomUncheckedBitArray _bitArray;
+        internal SequencePosition _sequencePosition;
 
         /// <summary>
         /// Returns the total amount of bytes consumed by the <see cref="Utf8JsonReader"/> so far
         /// for the given UTF-8 encoded input text.
         /// </summary>
         public long BytesConsumed => _bytesConsumed;
 
+        /// <summary>
+        /// Returns the current <see cref="SequencePosition"/> within the provided UTF-8 encoded
+        /// input ReadOnlySequence&lt;byte&gt;. If the <see cref="Utf8JsonReader"/> was constructed
+        /// with a ReadOnlySpan&lt;byte&gt; instead, this will always return a default <see cref="SequencePosition"/>.
+        /// </summary>
+        public SequencePosition Position => _sequencePosition;
+
         /// <summary>
         /// Constructs a new <see cref="JsonReaderState"/> instance.
         /// </summary>
@@ -62,12 +70,12 @@ public struct JsonReaderState
         /// across async/await boundaries and hence this type is required to provide support for reading
         /// in more data asynchronously before continuing with a new instance of the <see cref="Utf8JsonReader"/>.
         /// </remarks>
-        public JsonReaderState(int maxDepth = StackFreeMaxDepth, JsonReaderOptions options = default)
+        public JsonReaderState(int maxDepth = AllocationFreeMaxDepth, JsonReaderOptions options = default)
         {
             if (maxDepth <= 0)
                 throw ThrowHelper.GetArgumentException_MaxDepthMustBePositive();
 
-            _stackFreeContainer = default;
+            _allocationFreeContainer = default;
             _lineNumber = default;
             _bytePositionInLine = default;
             _bytesConsumed = default;
@@ -79,9 +87,11 @@ public JsonReaderState(int maxDepth = StackFreeMaxDepth, JsonReaderOptions optio
             _previousTokenType = default;
             _readerOptions = options;
 
-            // Only allocate the stack if the user reads a JSON payload beyond the depth that the _stackFreeContainer can handle.
+            // Only allocate if the user reads a JSON payload beyond the depth that the _allocationFreeContainer can handle.
             // This way we avoid allocations in the common, default cases, and allocate lazily.
-            _stack = null;
+            _bitArray = default;
+
+            _sequencePosition = default;
         }
 
         /// <summary>