Integration tests, benchmark updates, subclass Buffer (#83)

Will close #30

Add integration tests with data compressed by third party implementations
Update benchmarks for use with igizp
Allow subclassing cramjam.Buffer

.gitignore

@@ -136,3 +136,4 @@ dmypy.json
 #Added by cargo
 
 /target
+.vscode/

benchmark-requirements.txt

@@ -3,5 +3,6 @@ python-snappy==0.5.4
 lz4==3.1.0
 brotlipy==0.7.0
 zstd==1.5.0.2
+isal==0.11.1
 numpy
 memory-profiler

benchmarks/test_bench.py

@@ -121,27 +121,35 @@ def test_cramjam_snappy_de_compress_into(benchmark, op, file):
     )
 
 
-@pytest.mark.parametrize(
-    "use_cramjam", (True, False), ids=lambda val: "cramjam" if val else "gzip"
-)
+@pytest.mark.parametrize("lib", ("gzip", "cramjam", "isal"), ids=lambda val: val)
 @pytest.mark.parametrize("file", FILES, ids=lambda val: val.name)
-def test_gzip(benchmark, file, use_cramjam: bool):
+def test_gzip(benchmark, file, lib):
+    from isal import igzip
+
     data = file.read_bytes()
-    if use_cramjam:
+    if lib == "cramjam":
         benchmark(
             round_trip,
             compress=cramjam.gzip.compress,
             decompress=cramjam.gzip.decompress,
             data=data,
-            level=9,
+            level=3,
         )
-    else:
+    elif lib == "gzip":
         benchmark(
             round_trip,
             compress=gzip.compress,
             decompress=gzip.decompress,
             data=data,
-            compresslevel=9,
+            compresslevel=3,
+        )
+    else:
+        benchmark(
+            round_trip,
+            compress=igzip.compress,
+            decompress=igzip.decompress,
+            data=data,
+            compresslevel=igzip._COMPRESS_LEVEL_BEST,  # 3
         )
 
 

dev-requirements.txt

@@ -1,4 +1,4 @@
-black==21.5b1
+black==22.3.0
 maturin
 numpy
 pytest>=5.3.0

src/io.rs

@@ -350,7 +350,7 @@ impl RustyFile {
 /// b'bytes'
 /// ```
 ///
-#[pyclass(name = "Buffer")]
+#[pyclass(subclass, name = "Buffer")]
 #[derive(Default)]
 pub struct RustyBuffer {
     pub(crate) inner: Cursor<Vec<u8>>,

tests/data/integration/plaintext.txt

@@ -0,0 +1,21 @@
+The Zen of Python, by Tim Peters
+
+Beautiful is better than ugly.
+Explicit is better than implicit.
+Simple is better than complex.
+Complex is better than complicated.
+Flat is better than nested.
+Sparse is better than dense.
+Readability counts.
+Special cases aren't special enough to break the rules.
+Although practicality beats purity.
+Errors should never pass silently.
+Unless explicitly silenced.
+In the face of ambiguity, refuse the temptation to guess.
+There should be one-- and preferably only one --obvious way to do it.
+Although that way may not be obvious at first unless you're Dutch.
+Now is better than never.
+Although never is often better than *right* now.
+If the implementation is hard to explain, it's a bad idea.
+If the implementation is easy to explain, it may be a good idea.
+Namespaces are one honking great idea -- let's do more of those!

tests/test_integration.py

@@ -0,0 +1,41 @@
+"""
+Test decompressing files which have been compressed from 
+main stream third party implementations, separate from this project.
+"""
+import sys
+import pathlib
+from collections import namedtuple
+
+import pytest
+import cramjam
+
+
+@pytest.fixture
+def integration_dir():
+    return pathlib.Path(__file__).parent.joinpath("data/integration")
+
+
+@pytest.fixture
+def plaintext(integration_dir):
+    return integration_dir.joinpath("plaintext.txt").read_bytes()
+
+
+Variant = namedtuple("Variant", ("name", "suffix"))
+
+
+@pytest.mark.skipif(sys.platform.startswith("win"), reason="Bytes comparison fails on windows")
+@pytest.mark.parametrize(
+    "variant",
+    (
+        Variant("gzip", "gz"),
+        Variant("bzip2", "bz2"),
+        Variant("zstd", "zst"),
+        Variant("brotli", "br"),
+        Variant("lz4", "lz4"),
+        Variant("snappy", "snappy"),
+    ),
+)
+def test_variant(variant: Variant, integration_dir: pathlib.Path, plaintext: bytes):
+    file = integration_dir.joinpath(f"plaintext.txt.{variant.suffix}")
+    decompress = getattr(cramjam, variant.name).decompress
+    assert bytes(decompress(file.read_bytes())) == plaintext