From 1fb71ad3adf842f3b3f55a93e0a4cb35b82c623c Mon Sep 17 00:00:00 2001 From: Guillaume Blanc Date: Wed, 22 Nov 2023 18:29:06 +0100 Subject: [PATCH] Implements a span stream, using a external buffer provided at construction time --- CHANGES.md | 6 +++ include/ozz/base/io/stream.h | 81 +++++++++++++++++++++--------------- src/base/io/stream.cc | 74 ++++++++++++++++---------------- test/base/io/stream_tests.cc | 24 +++++++++-- 4 files changed, 109 insertions(+), 76 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4c7a99e53..a0fe5760e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +Release version next +---------------------- + +* Library + - [base] Implements a span stream, using a external buffer provided at construction time. + Release version 0.14.2 ---------------------- diff --git a/include/ozz/base/io/stream.h b/include/ozz/base/io/stream.h index faeba0844..03af7dd45 100644 --- a/include/ozz/base/io/stream.h +++ b/include/ozz/base/io/stream.h @@ -33,7 +33,9 @@ #include +#include "ozz/base/containers/vector.h" #include "ozz/base/platform.h" +#include "ozz/base/span.h" namespace ozz { namespace io { @@ -103,86 +105,99 @@ class OZZ_BASE_DLL File : public Stream { explicit File(void* _file); // Close the file if it is opened. - virtual ~File(); + virtual ~File() override; // Close the file if it is opened. void Close(); // See Stream::opened for details. - virtual bool opened() const; + virtual bool opened() const override; // See Stream::Read for details. - virtual size_t Read(void* _buffer, size_t _size); + virtual size_t Read(void* _buffer, size_t _size) override; // See Stream::Write for details. - virtual size_t Write(const void* _buffer, size_t _size); + virtual size_t Write(const void* _buffer, size_t _size) override; // See Stream::Seek for details. - virtual int Seek(int _offset, Origin _origin); + virtual int Seek(int _offset, Origin _origin) override; // See Stream::Tell for details. - virtual int Tell() const; + virtual int Tell() const override; // See Stream::Tell for details. - virtual size_t Size() const; + virtual size_t Size() const override; private: // The CRT file pointer. void* file_; }; -// Implements an in-memory Stream. Allows to use a memory buffer as a Stream. -// The opening mode is equivalent to fopen w+b (binary read/write). -class OZZ_BASE_DLL MemoryStream : public Stream { +// Implements an in-memory Stream. Buffer is provided at construction time and +// can be reallocated if overriden. The opening mode is equivalent to fopen w+b +// (binary read/write). +class OZZ_BASE_DLL SpanStream : public Stream { public: // Construct an empty memory stream opened in w+b mode. - MemoryStream(); + SpanStream(ozz::span _buffer); // Closes the stream and deallocates memory buffer. - virtual ~MemoryStream(); + virtual ~SpanStream(); // See Stream::opened for details. - virtual bool opened() const; + virtual bool opened() const override; // See Stream::Read for details. - virtual size_t Read(void* _buffer, size_t _size); + virtual size_t Read(void* _buffer, size_t _size) override; // See Stream::Write for details. - virtual size_t Write(const void* _buffer, size_t _size); + virtual size_t Write(const void* _buffer, size_t _size) override; // See Stream::Seek for details. - virtual int Seek(int _offset, Origin _origin); + virtual int Seek(int _offset, Origin _origin) override; // See Stream::Tell for details. - virtual int Tell() const; + virtual int Tell() const override; // See Stream::Tell for details. - virtual size_t Size() const; + virtual size_t Size() const override; private: - // Resizes buffers size to _size bytes. If _size is less than the actual - // buffer size, then it remains unchanged. - // Returns true if the buffer can contains _size bytes. - bool Resize(size_t _size); - - // Size of the buffer increment. - static const size_t kBufferSizeIncrement; - // Maximum stream size. static const size_t kMaxSize; - // Buffer of data. - byte* buffer_; + // Asked to resizes buffers _size bytes. + // Returns true if the buffer can contains _size bytes. + virtual bool Resize(size_t _size, ozz::span& _buffer); - // The size of the buffer, which is greater or equal to the size of the data - // it contains (end_). - size_t alloc_size_; + // Buffer of data. + ozz::span buffer_; // The effective size of the data in the buffer. - int end_; + int end_ = 0; // The cursor position in the buffer of data. - int tell_; + int tell_ = 0; +}; + +// Implements a self-allocated in-memory Stream. +// The opening mode is equivalent to fopen w+b (binary read/write). +class OZZ_BASE_DLL MemoryStream : public SpanStream { + public: + // Construct an empty memory stream opened in w+b mode. + MemoryStream(); + + // Closes the stream and deallocates memory buffer. + virtual ~MemoryStream(); + + private: + // Resizes buffers size to _size bytes. If _size is less than the actual + // buffer size, then it remains unchanged. + // Returns true if the buffer can contains _size bytes. + virtual bool Resize(size_t _size, ozz::span& _buffer) override; + + // Buffer storage + ozz::vector allocation_; }; } // namespace io } // namespace ozz diff --git a/src/base/io/stream.cc b/src/base/io/stream.cc index 414b0a40a..d7d9c5266 100644 --- a/src/base/io/stream.cc +++ b/src/base/io/stream.cc @@ -107,21 +107,18 @@ size_t File::Size() const { return static_cast(end); } -// Starts MemoryStream implementation. -const size_t MemoryStream::kBufferSizeIncrement = 16 << 10; -const size_t MemoryStream::kMaxSize = std::numeric_limits::max(); +// SpanStream implementation. +// ---------------------------------------------------------------------------- -MemoryStream::MemoryStream() - : buffer_(nullptr), alloc_size_(0), end_(0), tell_(0) {} +const size_t SpanStream::kMaxSize = std::numeric_limits::max(); -MemoryStream::~MemoryStream() { - ozz::memory::default_allocator()->Deallocate(buffer_); - buffer_ = nullptr; -} +SpanStream::SpanStream(ozz::span _buffer) : buffer_(_buffer) {} + +SpanStream::~SpanStream() {} -bool MemoryStream::opened() const { return true; } +bool SpanStream::opened() const { return true; } -size_t MemoryStream::Read(void* _buffer, size_t _size) { +size_t SpanStream::Read(void* _buffer, size_t _size) { // A read cannot set file position beyond the end of the file. // A read cannot exceed the maximum Stream size. if (tell_ > end_ || _size > kMaxSize) { @@ -129,12 +126,12 @@ size_t MemoryStream::Read(void* _buffer, size_t _size) { } const int read_size = math::Min(end_ - tell_, static_cast(_size)); - std::memcpy(_buffer, buffer_ + tell_, read_size); + std::memcpy(_buffer, buffer_.data() + tell_, read_size); tell_ += read_size; return read_size; } -size_t MemoryStream::Write(const void* _buffer, size_t _size) { +size_t SpanStream::Write(const void* _buffer, size_t _size) { if (_size > kMaxSize || tell_ > static_cast(kMaxSize - _size)) { // A write cannot exceed the maximum Stream size. return 0; @@ -144,27 +141,27 @@ size_t MemoryStream::Write(const void* _buffer, size_t _size) { // beyond the end of existing data in the file. If data is later written at // this point, subsequent reads of data in the gap shall return bytes with // the value 0 until data is actually written into the gap. - if (!Resize(tell_)) { + if (!Resize(tell_, buffer_)) { return 0; } // Fills the gap with 0's. const size_t gap = tell_ - end_; - std::memset(buffer_ + end_, 0, gap); + std::memset(buffer_.data() + end_, 0, gap); end_ = tell_; } const int size = static_cast(_size); const int tell_end = tell_ + size; - if (Resize(tell_end)) { + if (Resize(tell_end, buffer_)) { end_ = math::Max(tell_end, end_); - std::memcpy(buffer_ + tell_, _buffer, _size); + std::memcpy(buffer_.data() + tell_, _buffer, _size); tell_ += size; return _size; } return 0; } -int MemoryStream::Seek(int _offset, Origin _origin) { +int SpanStream::Seek(int _offset, Origin _origin) { int origin; switch (_origin) { case kCurrent: @@ -192,28 +189,27 @@ int MemoryStream::Seek(int _offset, Origin _origin) { return 0; } -int MemoryStream::Tell() const { return tell_; } - -size_t MemoryStream::Size() const { return static_cast(end_); } - -bool MemoryStream::Resize(size_t _size) { - if (_size > alloc_size_) { - // Resize to the next multiple of kBufferSizeIncrement, requires - // kBufferSizeIncrement to be a power of 2. - static_assert( - (MemoryStream::kBufferSizeIncrement & (kBufferSizeIncrement - 1)) == 0, - "kBufferSizeIncrement must be a power of 2"); - const size_t new_size = ozz::Align(_size, kBufferSizeIncrement); - byte* new_buffer = reinterpret_cast( - ozz::memory::default_allocator()->Allocate(new_size, 16)); - if (buffer_ != nullptr) { - std::memcpy(new_buffer, buffer_, alloc_size_); - } - ozz::memory::default_allocator()->Deallocate(buffer_); - buffer_ = new_buffer; - alloc_size_ = new_size; +int SpanStream::Tell() const { return tell_; } + +size_t SpanStream::Size() const { return static_cast(end_); } + +bool SpanStream::Resize(size_t _size, ozz::span& _buffer) { + return _size <= _buffer.size(); +} + +// MemoryStream implementation. +// ---------------------------------------------------------------------------- + +MemoryStream::MemoryStream() : SpanStream({}) {} + +MemoryStream::~MemoryStream() {} + +bool MemoryStream::Resize(size_t _size, ozz::span& _buffer) { + if (_size > _buffer.size()) { + allocation_.resize(_size); + _buffer = ozz::make_span(allocation_); } - return _size == 0 || buffer_ != nullptr; + return true; } } // namespace io } // namespace ozz diff --git a/test/base/io/stream_tests.cc b/test/base/io/stream_tests.cc index 5b9dd4b44..b61d5a7ba 100644 --- a/test/base/io/stream_tests.cc +++ b/test/base/io/stream_tests.cc @@ -25,13 +25,13 @@ // // //----------------------------------------------------------------------------// -#include "ozz/base/io/stream.h" - #include + #include #include "gtest/gtest.h" - +#include "ozz/base/containers/array.h" +#include "ozz/base/io/stream.h" #include "ozz/base/platform.h" void TestStream(ozz::io::Stream* _stream) { @@ -79,7 +79,7 @@ void TestSeek(ozz::io::Stream* _stream) { EXPECT_EQ(_stream->Tell(), static_cast(sizeof(Type))); EXPECT_EQ(_stream->Size(), sizeof(Type)); - const int64_t kEnd = 465827; + const int64_t kEnd = 46582; // Force file length to kEnd but do not write to the stream. EXPECT_EQ(_stream->Seek(kEnd - _stream->Tell(), ozz::io::Stream::kCurrent), 0); @@ -171,6 +171,22 @@ TEST(File, Stream) { { EXPECT_TRUE(ozz::io::File::Exist("test.bin")); } } +TEST(SpanStream, Stream) { + ozz::array buffer; + { + ozz::io::SpanStream stream(ozz::make_span(buffer)); + TestStream(&stream); + } + { + ozz::io::SpanStream stream(ozz::make_span(buffer)); + TestSeek(&stream); + } + { + ozz::io::SpanStream stream(ozz::make_span(buffer)); + TestTooBigStream(&stream); + } +} + TEST(MemoryStream, Stream) { { ozz::io::MemoryStream stream;