diff --git a/nautilus_trader/persistence/external/readers.py b/nautilus_trader/persistence/external/readers.py index 6ce0ee59c84e..595381b5bb70 100644 --- a/nautilus_trader/persistence/external/readers.py +++ b/nautilus_trader/persistence/external/readers.py @@ -182,6 +182,7 @@ def __init__( line_preprocessor: LinePreprocessor = None, instrument_provider: Optional[InstrumentProvider] = None, instrument_provider_update: Optional[Callable] = None, + newline=b"\n", ): assert line_preprocessor is None or isinstance(line_preprocessor, LinePreprocessor) super().__init__( @@ -190,11 +191,12 @@ def __init__( instrument_provider=instrument_provider, ) self.line_preprocessor = line_preprocessor or LinePreprocessor() + self.newline = newline def parse(self, block) -> Generator: # noqa: C901 self.buffer += block if b"\n" in block: - process, self.buffer = self.buffer.rsplit(b"\n", maxsplit=1) + process, self.buffer = self.buffer.rsplit(self.newline, maxsplit=1) else: process, self.buffer = block, b"" if process: @@ -245,6 +247,8 @@ def __init__( chunked=True, as_dataframe=True, separator=",", + newline=b"\n", + encoding="utf-8", ): super().__init__( instrument_provider=instrument_provider, @@ -256,15 +260,17 @@ def __init__( self.chunked = chunked self.as_dataframe = as_dataframe self.separator = separator + self.newline = newline + self.encoding = encoding def parse(self, block: bytes) -> Generator: if self.header is None: header, block = block.split(b"\n", maxsplit=1) - self.header = header.decode().split(self.separator) + self.header = header.decode(self.encoding).split(self.separator) self.buffer += block if b"\n" in block: - process, self.buffer = self.buffer.rsplit(b"\n", maxsplit=1) + process, self.buffer = self.buffer.rsplit(self.newline, maxsplit=1) else: process, self.buffer = block, b"" diff --git a/tests/unit_tests/persistence/test_metadata.py b/tests/unit_tests/persistence/test_metadata.py index 23abf219d7f2..68ac801c7700 100644 --- a/tests/unit_tests/persistence/test_metadata.py +++ b/tests/unit_tests/persistence/test_metadata.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------------------------- +import sys import fsspec +import pytest from nautilus_trader.backtest.data.providers import TestInstrumentProvider from nautilus_trader.model.identifiers import Venue @@ -28,6 +30,7 @@ TEST_DATA_DIR = PACKAGE_ROOT + "/data" +@pytest.mark.skipif(sys.platform == "win32", reason="test path broken on windows") class TestPersistenceBatching: def setup(self): data_catalog_setup()