Skip windows broken tests, add fix for #563

nautilus_trader/persistence/external/readers.py

@@ -182,6 +182,7 @@ def __init__(
         line_preprocessor: LinePreprocessor = None,
         instrument_provider: Optional[InstrumentProvider] = None,
         instrument_provider_update: Optional[Callable] = None,
+        newline=b"\n",
     ):
         assert line_preprocessor is None or isinstance(line_preprocessor, LinePreprocessor)
         super().__init__(
@@ -190,11 +191,12 @@ def __init__(
             instrument_provider=instrument_provider,
         )
         self.line_preprocessor = line_preprocessor or LinePreprocessor()
+        self.newline = newline
 
     def parse(self, block) -> Generator:  # noqa: C901
         self.buffer += block
         if b"\n" in block:
-            process, self.buffer = self.buffer.rsplit(b"\n", maxsplit=1)
+            process, self.buffer = self.buffer.rsplit(self.newline, maxsplit=1)
         else:
             process, self.buffer = block, b""
         if process:
@@ -245,6 +247,8 @@ def __init__(
         chunked=True,
         as_dataframe=True,
         separator=",",
+        newline=b"\n",
+        encoding="utf-8",
     ):
         super().__init__(
             instrument_provider=instrument_provider,
@@ -256,15 +260,17 @@ def __init__(
         self.chunked = chunked
         self.as_dataframe = as_dataframe
         self.separator = separator
+        self.newline = newline
+        self.encoding = encoding
 
     def parse(self, block: bytes) -> Generator:
         if self.header is None:
             header, block = block.split(b"\n", maxsplit=1)
-            self.header = header.decode().split(self.separator)
+            self.header = header.decode(self.encoding).split(self.separator)
 
         self.buffer += block
         if b"\n" in block:
-            process, self.buffer = self.buffer.rsplit(b"\n", maxsplit=1)
+            process, self.buffer = self.buffer.rsplit(self.newline, maxsplit=1)
         else:
             process, self.buffer = block, b""
 

tests/unit_tests/persistence/test_metadata.py

@@ -12,8 +12,10 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 # -------------------------------------------------------------------------------------------------
+import sys
 
 import fsspec
+import pytest
 
 from nautilus_trader.backtest.data.providers import TestInstrumentProvider
 from nautilus_trader.model.identifiers import Venue
@@ -28,6 +30,7 @@
 TEST_DATA_DIR = PACKAGE_ROOT + "/data"
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="test path broken on windows")
 class TestPersistenceBatching:
     def setup(self):
         data_catalog_setup()