diff --git a/stdlib/public/core/ArrayBuffer.swift b/stdlib/public/core/ArrayBuffer.swift index e0d8931ce2174..6d7ebf9adda8f 100644 --- a/stdlib/public/core/ArrayBuffer.swift +++ b/stdlib/public/core/ArrayBuffer.swift @@ -387,8 +387,8 @@ extension _ArrayBuffer { } else { var refCopy = self - refCopy.replace( - subRange: i..<(i + 1), + refCopy.replaceSubrange( + i..<(i + 1), with: 1, elementsOf: CollectionOfOne(newValue)) } diff --git a/stdlib/public/core/ArrayBufferProtocol.swift b/stdlib/public/core/ArrayBufferProtocol.swift index b70f483a73ffc..f6982d1e454da 100644 --- a/stdlib/public/core/ArrayBufferProtocol.swift +++ b/stdlib/public/core/ArrayBufferProtocol.swift @@ -72,8 +72,8 @@ internal protocol _ArrayBufferProtocol /// /// - Precondition: This buffer is backed by a uniquely-referenced /// `_ContiguousArrayBuffer`. - mutating func replace( - subRange: Range, + mutating func replaceSubrange( + _ subrange: Range, with newCount: Int, elementsOf newValues: C ) where C : Collection, C.Iterator.Element == Element @@ -124,41 +124,42 @@ internal protocol _ArrayBufferProtocol var identity: UnsafeRawPointer { get } var startIndex: Int { get } + var endIndex: Int { get } } -extension _ArrayBufferProtocol where Index == Int { +extension _ArrayBufferProtocol { internal var subscriptBaseAddress: UnsafeMutablePointer { return firstElementAddress } - internal mutating func replace( - subRange: Range, + internal mutating func replaceSubrange( + _ subrange: Range, with newCount: Int, elementsOf newValues: C ) where C : Collection, C.Iterator.Element == Element { _sanityCheck(startIndex == 0, "_SliceBuffer should override this function.") let oldCount = self.count - let eraseCount = subRange.count + let eraseCount = subrange.count let growth = newCount - eraseCount self.count = oldCount + growth let elements = self.subscriptBaseAddress - let oldTailIndex = subRange.upperBound + let oldTailIndex = subrange.upperBound let oldTailStart = elements + oldTailIndex let newTailIndex = oldTailIndex + growth let newTailStart = oldTailStart + growth - let tailCount = oldCount - subRange.upperBound + let tailCount = oldCount - subrange.upperBound if growth > 0 { // Slide the tail part of the buffer forwards, in reverse order // so as not to self-clobber. newTailStart.moveInitialize(from: oldTailStart, count: tailCount) - // Assign over the original subRange + // Assign over the original subrange var i = newValues.startIndex - for j in CountableRange(subRange) { + for j in CountableRange(subrange) { elements[j] = newValues[i] newValues.formIndex(after: &i) } @@ -170,12 +171,12 @@ extension _ArrayBufferProtocol where Index == Int { _expectEnd(i, newValues) } else { // We're not growing the buffer - // Assign all the new elements into the start of the subRange - var i = subRange.lowerBound + // Assign all the new elements into the start of the subrange + var i = subrange.lowerBound var j = newValues.startIndex for _ in 0.. R { let count = self.count // Ensure unique storage - _outlinedMakeUniqueBuffer(&_buffer, bufferCount: count) + _buffer._outlinedMakeUniqueBuffer(bufferCount: count) // Ensure that body can't invalidate the storage or its bounds by // moving self into a temporary working array. @@ -1666,29 +1666,25 @@ internal struct _InitializeMemoryFromCollection< var newValues: C } -// FIXME(ABI)#14 : add argument labels. -@inline(never) -internal func _arrayOutOfPlaceReplace( - _ source: inout B, - _ bounds: Range, - _ newValues: C, - _ insertCount: Int -) where - B : _ArrayBufferProtocol, - C : Collection, - C.Iterator.Element == B.Element, - B.Index == Int { - - let growth = insertCount - bounds.count - let newCount = source.count + growth - var newBuffer = _forceCreateUniqueMutableBuffer( - &source, newCount: newCount, requiredCapacity: newCount) - - _arrayOutOfPlaceUpdate( - &source, &newBuffer, - bounds.lowerBound - source.startIndex, insertCount, - _InitializeMemoryFromCollection(newValues) - ) +extension _ArrayBufferProtocol { + @inline(never) + internal mutating func _arrayOutOfPlaceReplace( + _ bounds: Range, + with newValues: C, + count insertCount: Int + ) where C.Iterator.Element == Element { + + let growth = insertCount - bounds.count + let newCount = self.count + growth + var newBuffer = _forceCreateUniqueMutableBuffer( + newCount: newCount, requiredCapacity: newCount) + + _arrayOutOfPlaceUpdate( + &newBuffer, + bounds.lowerBound - startIndex, insertCount, + _InitializeMemoryFromCollection(newValues) + ) + } } /// A _debugPrecondition check that `i` has exactly reached the end of @@ -1749,25 +1745,25 @@ extension ${Self} { _precondition(subrange.lowerBound >= self._buffer.startIndex, "${Self} replace: subrange start is negative") % else: - _precondition(subrange.lowerBound >= self._buffer.startIndex, + _precondition(subrange.lowerBound >= _buffer.startIndex, "${Self} replace: subrange start is before the startIndex") % end - _precondition(subrange.upperBound <= self._buffer.endIndex, + _precondition(subrange.upperBound <= _buffer.endIndex, "${Self} replace: subrange extends past the end") - let oldCount = self._buffer.count + let oldCount = _buffer.count let eraseCount = subrange.count let insertCount = numericCast(newElements.count) as Int let growth = insertCount - eraseCount - if self._buffer.requestUniqueMutableBackingBuffer( + if _buffer.requestUniqueMutableBackingBuffer( minimumCapacity: oldCount + growth) != nil { - self._buffer.replace( - subRange: subrange, with: insertCount, elementsOf: newElements) + _buffer.replaceSubrange( + subrange, with: insertCount, elementsOf: newElements) } else { - _arrayOutOfPlaceReplace(&self._buffer, subrange, newElements, insertCount) + _buffer._arrayOutOfPlaceReplace(subrange, with: newElements, count: insertCount) } } } @@ -1823,58 +1819,59 @@ public func += < //===--- generic helpers --------------------------------------------------===// -/// Create a unique mutable buffer that has enough capacity to hold 'newCount' -/// elements and at least 'requiredCapacity' elements. Set the count of the new -/// buffer to 'newCount'. The content of the buffer is uninitialized. -/// The formula used to compute the new buffers capacity is: -/// max(requiredCapacity, source.capacity) if newCount <= source.capacity -/// max(requiredCapacity, _growArrayCapacity(source.capacity)) otherwise -@inline(never) -internal func _forceCreateUniqueMutableBuffer<_Buffer : _ArrayBufferProtocol>( - _ source: inout _Buffer, newCount: Int, requiredCapacity: Int -) -> _ContiguousArrayBuffer<_Buffer.Element> { - return _forceCreateUniqueMutableBufferImpl( - &source, countForBuffer: newCount, minNewCapacity: newCount, - requiredCapacity: requiredCapacity) -} - -/// Create a unique mutable buffer that has enough capacity to hold -/// 'minNewCapacity' elements and set the count of the new buffer to -/// 'countForNewBuffer'. The content of the buffer uninitialized. -/// The formula used to compute the new buffers capacity is: -/// max(minNewCapacity, source.capacity) if minNewCapacity <= source.capacity -/// max(minNewCapacity, _growArrayCapacity(source.capacity)) otherwise -@inline(never) -internal -func _forceCreateUniqueMutableBuffer<_Buffer : _ArrayBufferProtocol>( - _ source: inout _Buffer, countForNewBuffer: Int, minNewCapacity: Int -) -> _ContiguousArrayBuffer<_Buffer.Element> { - return _forceCreateUniqueMutableBufferImpl( - &source, countForBuffer: countForNewBuffer, minNewCapacity: minNewCapacity, - requiredCapacity: minNewCapacity) -} - -/// Create a unique mutable buffer that has enough capacity to hold -/// 'minNewCapacity' elements and at least 'requiredCapacity' elements and set -/// the count of the new buffer to 'countForBuffer'. The content of the buffer -/// uninitialized. -/// The formula used to compute the new capacity is: -/// max(requiredCapacity, source.capacity) if minNewCapacity <= source.capacity -/// max(requiredCapacity, _growArrayCapacity(source.capacity)) otherwise -internal func _forceCreateUniqueMutableBufferImpl<_Buffer : _ArrayBufferProtocol>( - _ source: inout _Buffer, countForBuffer: Int, minNewCapacity: Int, - requiredCapacity: Int -) -> _ContiguousArrayBuffer<_Buffer.Element> { - _sanityCheck(countForBuffer >= 0) - _sanityCheck(requiredCapacity >= countForBuffer) - _sanityCheck(minNewCapacity >= countForBuffer) - - let minimumCapacity = max( - requiredCapacity, minNewCapacity > source.capacity - ? _growArrayCapacity(source.capacity) : source.capacity) - - return _ContiguousArrayBuffer( - _uninitializedCount: countForBuffer, minimumCapacity: minimumCapacity) +extension _ArrayBufferProtocol { + /// Create a unique mutable buffer that has enough capacity to hold 'newCount' + /// elements and at least 'requiredCapacity' elements. Set the count of the new + /// buffer to 'newCount'. The content of the buffer is uninitialized. + /// The formula used to compute the new buffers capacity is: + /// max(requiredCapacity, source.capacity) if newCount <= source.capacity + /// max(requiredCapacity, _growArrayCapacity(source.capacity)) otherwise + @inline(never) + internal func _forceCreateUniqueMutableBuffer( + newCount: Int, requiredCapacity: Int + ) -> _ContiguousArrayBuffer { + return _forceCreateUniqueMutableBufferImpl( + countForBuffer: newCount, minNewCapacity: newCount, + requiredCapacity: requiredCapacity) + } + + /// Create a unique mutable buffer that has enough capacity to hold + /// 'minNewCapacity' elements and set the count of the new buffer to + /// 'countForNewBuffer'. The content of the buffer uninitialized. + /// The formula used to compute the new buffers capacity is: + /// max(minNewCapacity, source.capacity) if minNewCapacity <= source.capacity + /// max(minNewCapacity, _growArrayCapacity(source.capacity)) otherwise + @inline(never) + internal func _forceCreateUniqueMutableBuffer( + countForNewBuffer: Int, minNewCapacity: Int + ) -> _ContiguousArrayBuffer { + return _forceCreateUniqueMutableBufferImpl( + countForBuffer: countForNewBuffer, minNewCapacity: minNewCapacity, + requiredCapacity: minNewCapacity) + } + + /// Create a unique mutable buffer that has enough capacity to hold + /// 'minNewCapacity' elements and at least 'requiredCapacity' elements and set + /// the count of the new buffer to 'countForBuffer'. The content of the buffer + /// uninitialized. + /// The formula used to compute the new capacity is: + /// max(requiredCapacity, source.capacity) if minNewCapacity <= source.capacity + /// max(requiredCapacity, _growArrayCapacity(source.capacity)) otherwise + internal func _forceCreateUniqueMutableBufferImpl( + countForBuffer: Int, minNewCapacity: Int, + requiredCapacity: Int + ) -> _ContiguousArrayBuffer { + _sanityCheck(countForBuffer >= 0) + _sanityCheck(requiredCapacity >= countForBuffer) + _sanityCheck(minNewCapacity >= countForBuffer) + + let minimumCapacity = Swift.max(requiredCapacity, + minNewCapacity > capacity + ? _growArrayCapacity(capacity) : capacity) + + return _ContiguousArrayBuffer( + _uninitializedCount: countForBuffer, minimumCapacity: minimumCapacity) + } } internal protocol _PointerFunction { @@ -1882,82 +1879,81 @@ internal protocol _PointerFunction { func call(_: UnsafeMutablePointer, count: Int) } -/// Initialize the elements of dest by copying the first headCount -/// items from source, calling initializeNewElements on the next -/// uninitialized element, and finally by copying the last N items -/// from source into the N remaining uninitialized elements of dest. -/// -/// As an optimization, may move elements out of source rather than -/// copying when it isUniquelyReferenced. -@inline(never) -internal func _arrayOutOfPlaceUpdate<_Buffer, Initializer>( - _ source: inout _Buffer, - _ dest: inout _ContiguousArrayBuffer<_Buffer.Element>, - _ headCount: Int, // Count of initial source elements to copy/move - _ newCount: Int, // Number of new elements to insert - _ initializeNewElements: Initializer -) where - _Buffer : _ArrayBufferProtocol, - Initializer : _PointerFunction, - Initializer.Element == _Buffer.Element, - _Buffer.Index == Int { - - _sanityCheck(headCount >= 0) - _sanityCheck(newCount >= 0) - - // Count of trailing source elements to copy/move - let tailCount = dest.count - headCount - newCount - _sanityCheck(headCount + tailCount <= source.count) - - let sourceCount = source.count - let oldCount = sourceCount - headCount - tailCount - let destStart = dest.firstElementAddress - let newStart = destStart + headCount - let newEnd = newStart + newCount - - // Check to see if we have storage we can move from - if let backing = source.requestUniqueMutableBackingBuffer( - minimumCapacity: sourceCount) { - - let sourceStart = source.firstElementAddress - let oldStart = sourceStart + headCount - - // Destroy any items that may be lurking in a _SliceBuffer before - // its real first element - let backingStart = backing.firstElementAddress - let sourceOffset = sourceStart - backingStart - backingStart.deinitialize(count: sourceOffset) - - // Move the head items - destStart.moveInitialize(from: sourceStart, count: headCount) - - // Destroy unused source items - oldStart.deinitialize(count: oldCount) - - initializeNewElements.call(newStart, count: newCount) - - // Move the tail items - newEnd.moveInitialize(from: oldStart + oldCount, count: tailCount) - - // Destroy any items that may be lurking in a _SliceBuffer after - // its real last element - let backingEnd = backingStart + backing.count - let sourceEnd = sourceStart + sourceCount - sourceEnd.deinitialize(count: backingEnd - sourceEnd) - backing.count = 0 - } - else { - let headStart = source.startIndex - let headEnd = headStart + headCount - let newStart = source._copyContents( - subRange: headStart..( + _ dest: inout _ContiguousArrayBuffer, + _ headCount: Int, // Count of initial source elements to copy/move + _ newCount: Int, // Number of new elements to insert + _ initializeNewElements: Initializer + ) where + Initializer : _PointerFunction, + Initializer.Element == Element { + + _sanityCheck(headCount >= 0) + _sanityCheck(newCount >= 0) + + // Count of trailing source elements to copy/move + let sourceCount = self.count + let tailCount = dest.count - headCount - newCount + _sanityCheck(headCount + tailCount <= sourceCount) + + let oldCount = sourceCount - headCount - tailCount + let destStart = dest.firstElementAddress + let newStart = destStart + headCount + let newEnd = newStart + newCount + + // Check to see if we have storage we can move from + if let backing = requestUniqueMutableBackingBuffer( + minimumCapacity: sourceCount) { + + let sourceStart = firstElementAddress + let oldStart = sourceStart + headCount + + // Destroy any items that may be lurking in a _SliceBuffer before + // its real first element + let backingStart = backing.firstElementAddress + let sourceOffset = sourceStart - backingStart + backingStart.deinitialize(count: sourceOffset) + + // Move the head items + destStart.moveInitialize(from: sourceStart, count: headCount) + + // Destroy unused source items + oldStart.deinitialize(count: oldCount) + + initializeNewElements.call(newStart, count: newCount) + + // Move the tail items + newEnd.moveInitialize(from: oldStart + oldCount, count: tailCount) + + // Destroy any items that may be lurking in a _SliceBuffer after + // its real last element + let backingEnd = backingStart + backing.count + let sourceEnd = sourceStart + sourceCount + sourceEnd.deinitialize(count: backingEnd - sourceEnd) + backing.count = 0 + } + else { + let headStart = startIndex + let headEnd = headStart + headCount + let newStart = _copyContents( + subRange: headStart.. : _PointerFunction { @@ -1981,78 +1977,69 @@ internal struct _IgnorePointer : _PointerFunction { } } -@_versioned -@inline(never) -internal func _outlinedMakeUniqueBuffer<_Buffer>( - _ buffer: inout _Buffer, bufferCount: Int -) where - _Buffer : _ArrayBufferProtocol, - _Buffer.Index == Int { +extension _ArrayBufferProtocol { + @_versioned + @inline(never) + internal mutating func _outlinedMakeUniqueBuffer(bufferCount: Int) { + + if _fastPath( + requestUniqueMutableBackingBuffer(minimumCapacity: bufferCount) != nil) { + return + } - if _fastPath( - buffer.requestUniqueMutableBackingBuffer(minimumCapacity: bufferCount) != nil) { - return + var newBuffer = _forceCreateUniqueMutableBuffer( + newCount: bufferCount, requiredCapacity: bufferCount) + _arrayOutOfPlaceUpdate(&newBuffer, bufferCount, 0, _IgnorePointer()) } - var newBuffer = _forceCreateUniqueMutableBuffer( - &buffer, newCount: bufferCount, requiredCapacity: bufferCount) - _arrayOutOfPlaceUpdate(&buffer, &newBuffer, bufferCount, 0, _IgnorePointer()) -} + internal mutating func _arrayReserve(_ minimumCapacity: Int) { -internal func _arrayReserve<_Buffer>( - _ buffer: inout _Buffer, _ minimumCapacity: Int -) where - _Buffer : _ArrayBufferProtocol, - _Buffer.Index == Int { + let oldCount = self.count + let requiredCapacity = Swift.max(count, minimumCapacity) + + if _fastPath( + requestUniqueMutableBackingBuffer( + minimumCapacity: requiredCapacity) != nil + ) { + return + } - let count = buffer.count - let requiredCapacity = max(count, minimumCapacity) + var newBuffer = _forceCreateUniqueMutableBuffer( + newCount: oldCount, requiredCapacity: requiredCapacity) - if _fastPath( - buffer.requestUniqueMutableBackingBuffer( - minimumCapacity: requiredCapacity) != nil - ) { - return + _arrayOutOfPlaceUpdate(&newBuffer, oldCount, 0, _IgnorePointer()) } - var newBuffer = _forceCreateUniqueMutableBuffer( - &buffer, newCount: count, requiredCapacity: requiredCapacity) - _arrayOutOfPlaceUpdate(&buffer, &newBuffer, count, 0, _IgnorePointer()) -} + /// Append items from `newItems` to a buffer. + internal mutating func _arrayAppendSequence( + _ newItems: S + ) where S.Iterator.Element == Element { + + var stream = newItems.makeIterator() + var nextItem = stream.next() -/// Append items from `newItems` to `buffer`. -internal func _arrayAppendSequence( - _ buffer: inout Buffer, _ newItems: S -) where - Buffer : _ArrayBufferProtocol, - S : Sequence, - S.Iterator.Element == Buffer.Element, - Buffer.Index == Int { - - var stream = newItems.makeIterator() - var nextItem = stream.next() - - if nextItem == nil { - return - } - - // This will force uniqueness - var count = buffer.count - _arrayReserve(&buffer, count + 1) - while true { - let capacity = buffer.capacity - let base = buffer.firstElementAddress - - while (nextItem != nil) && count < capacity { - (base + count).initialize(to: nextItem!) - count += 1 - nextItem = stream.next() - } - buffer.count = count if nextItem == nil { return } - _arrayReserve(&buffer, _growArrayCapacity(capacity)) + + // This will force uniqueness + var newCount = self.count + _arrayReserve(newCount + 1) + while true { + let currentCapacity = self.capacity + let base = self.firstElementAddress + + while (nextItem != nil) && newCount < currentCapacity { + (base + newCount).initialize(to: nextItem!) + newCount += 1 + nextItem = stream.next() + } + self.count = newCount + if nextItem == nil { + return + } + self._arrayReserve(_growArrayCapacity(currentCapacity)) + } } } diff --git a/stdlib/public/core/SliceBuffer.swift b/stdlib/public/core/SliceBuffer.swift index c369cb5c346bf..e601ea5fd2620 100644 --- a/stdlib/public/core/SliceBuffer.swift +++ b/stdlib/public/core/SliceBuffer.swift @@ -79,8 +79,8 @@ internal struct _SliceBuffer /// - Precondition: This buffer is backed by a uniquely-referenced /// `_ContiguousArrayBuffer` and /// `insertCount <= numericCast(newValues.count)`. - internal mutating func replace( - subRange: Range, + internal mutating func replaceSubrange( + _ subrange: Range, with insertCount: Int, elementsOf newValues: C ) where C : Collection, C.Iterator.Element == Element { @@ -90,7 +90,7 @@ internal struct _SliceBuffer _sanityCheck(_hasNativeBuffer && isUniquelyReferenced()) - let eraseCount = subRange.count + let eraseCount = subrange.count let growth = insertCount - eraseCount let oldCount = count @@ -99,10 +99,10 @@ internal struct _SliceBuffer _sanityCheck(native.count + growth <= native.capacity) - let start = subRange.lowerBound - startIndex + hiddenElementCount - let end = subRange.upperBound - startIndex + hiddenElementCount - native.replace( - subRange: start.. let myCount = count if _slowPath(backingCount > myCount + offset) { - native.replace( - subRange: (myCount+offset)..