From d559efb2a9a31d3513fde03e277c2c5035f2953c Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 6 Feb 2024 09:44:53 +0100 Subject: [PATCH 01/13] compile blosc2 filter with its plugins --- setup.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index a5a320ee..852efc9c 100644 --- a/setup.py +++ b/setup.py @@ -865,11 +865,28 @@ def get_blosc2_plugin(): """ hdf5_blosc2_dir = 'src/PyTables/hdf5-blosc2/src' blosc2_dir = 'src/c-blosc2' + plugins_dir = f'{blosc2_dir}/plugins' # blosc sources sources = glob(f'{blosc2_dir}/blosc/*.c') - include_dirs = [blosc2_dir, f'{blosc2_dir}/blosc', f'{blosc2_dir}/include'] - define_macros = [('SHUFFLE_AVX512_ENABLED', 1), ('SHUFFLE_NEON_ENABLED', 1)] + sources += [ # Add embedded codecs, filters and tuners + src_file + for src_file in glob(f'{plugins_dir}/*.c') + glob(f'{plugins_dir}/*/*.c') + glob(f'{plugins_dir}/*/*/*.c') + if not os.path.basename(src_file).startswith("test") + ] + sources += glob(f'{plugins_dir}/codecs/zfp/src/*.c') # Add ZFP embedded sources + + include_dirs = [ + blosc2_dir, + f'{blosc2_dir}/blosc', + f'{blosc2_dir}/include', + f'{blosc2_dir}/plugins/codecs/zfp/include', + ] + define_macros = [ + ('HAVE_PLUGINS', 1), + ('SHUFFLE_AVX512_ENABLED', 1), + ('SHUFFLE_NEON_ENABLED', 1), + ] extra_compile_args = [] extra_link_args = [] libraries = [] @@ -902,8 +919,7 @@ def get_blosc2_plugin(): include_dirs += get_zstd_clib('include_dirs') define_macros.append(('HAVE_ZSTD', 1)) - extra_compile_args += ['-std=gnu99'] # Needed to build manylinux1 wheels - extra_compile_args += ['-O3', '-ffast-math'] + extra_compile_args += ['-O3', '-ffast-math', '-std=gnu99'] extra_compile_args += ['/Ox', '/fp:fast'] extra_compile_args += ['-pthread'] extra_link_args += ['-pthread'] From 4ef02604aad53d500c80cadb7827245775a8486c Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 6 Feb 2024 14:52:43 +0100 Subject: [PATCH 02/13] Add some tests of the blosc2 plugins --- setup.py | 7 ++- src/hdf5plugin/test.py | 103 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 852efc9c..6373c393 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ # coding: utf-8 # /*########################################################################## # -# Copyright (c) 2016-2022 European Synchrotron Radiation Facility +# Copyright (c) 2016-2024 European Synchrotron Radiation Facility # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -1305,7 +1305,10 @@ def get_version(): ext_modules=extensions, install_requires=['h5py'], setup_requires=['setuptools', 'wheel'], - extras_require={'dev': ['sphinx', 'sphinx_rtd_theme']}, + extras_require={ + 'dev': ['sphinx', 'sphinx_rtd_theme'], + 'test': ['blosc2', 'blosc2-grok'], + }, cmdclass=cmdclass, libraries=libraries, zip_safe=False, diff --git a/src/hdf5plugin/test.py b/src/hdf5plugin/test.py index f9ed5b03..4e3f8809 100644 --- a/src/hdf5plugin/test.py +++ b/src/hdf5plugin/test.py @@ -1,7 +1,7 @@ # coding: utf-8 # /*########################################################################## # -# Copyright (c) 2019-2023 European Synchrotron Radiation Facility +# Copyright (c) 2019-2024 European Synchrotron Radiation Facility # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -23,6 +23,10 @@ # # ###########################################################################*/ """Provides tests """ +from __future__ import annotations + +import importlib.util +import io import os import shutil import tempfile @@ -31,6 +35,10 @@ import h5py import hdf5plugin +try: + import blosc2 +except ImportError: + blosc2 = None from hdf5plugin import _filters @@ -465,9 +473,100 @@ def testAbsoluteMode(self): ) +class TestBlosc2Plugins(unittest.TestCase): + """Specific tests for Blosc2 compression with Blosc2 plugins""" + + def setUp(self): + if not should_test("blosc2"): + self.skipTest("Blosc2 filter not available") + if blosc2 is None: + self.skipTest("Blosc2 package not available") + + def _readback_hdf5_blosc2_dataset( + self, + data: numpy.ndarray, + blocks: tuple[int, ...] | None = None, + **cparams + ) -> numpy.ndarray: + """Compress data with blosc2, write it as HDF5 file with direct chunk write and read it back with h5py + + :param data: data array to compress + :param blocks: Blosc2 block shape + :param cparams: Blosc2 compression parameters + """ + # Convert data to a blosc2 array: This is where compression happens + blosc_array = blosc2.asarray( + data, + chunks=data.shape, + blocks=blocks, + cparams=cparams, + ) + + # Write blosc2 array as a hdf5 dataset + with io.BytesIO() as buffer: + with h5py.File(buffer, 'w') as f: + dataset = f.create_dataset( + 'data', + shape=data.shape, + dtype=data.dtype, + chunks=data.shape, + compression=hdf5plugin.Blosc2(), + ) + dataset.id.write_direct_chunk( + (0,) * data.ndim, + blosc_array.schunk.to_cframe(), + ) + f.flush() + + return dataset[()] + + def test_blosc2_filter_int_trunc(self): + """Read blosc2 dataset written with int truncate filter plugin""" + data = numpy.arange(2**16, dtype=numpy.int16) + + removed_bits = 2 + read_data = self._readback_hdf5_blosc2_dataset( + data, + codec=blosc2.Codec.ZSTD, + filters=[blosc2.Filter.INT_TRUNC], + filters_meta=[-removed_bits], + ) + assert numpy.allclose(read_data, data, rtol=0.0, atol=2**removed_bits) + + def test_blosc2_codec_zfp(self): + """Read blosc2 dataset written with zfp codec plugin""" + data = numpy.outer(numpy.arange(128), numpy.arange(128)).astype(numpy.float32) + + read_data = self._readback_hdf5_blosc2_dataset( + data, + codec=blosc2.Codec.ZFP_PREC, + codec_meta=8, + filters=[], + filters_meta=[], + splitmode=blosc2.SplitMode.NEVER_SPLIT, + ) + assert numpy.allclose(read_data, data, rtol=1e-3, atol=0) + + @unittest.skipIf(importlib.util.find_spec("blosc2_grok") is None, "blosc2_grok package is not available") + def test_blosc2_codec_grok(self): + """Read blosc2 dataset written with blosc2-grok external codec plugin""" + shape = 10, 128, 128 + data = numpy.arange(numpy.prod(shape), dtype=numpy.uint16).reshape(shape) + + read_data = self._readback_hdf5_blosc2_dataset( + data, + blocks=(1,) + data.shape[1:], # 1 block per slice + codec=blosc2.Codec.GROK, + # Disable the filters and the splitmode, because these don't work with grok. + filters=[], + splitmode=blosc2.SplitMode.NEVER_SPLIT, + ) + assert numpy.array_equal(read_data, data) + + def suite(): test_suite = unittest.TestSuite() - for cls in (TestHDF5PluginRW, TestPackage, TestRegisterFilter, TestGetFilters, TestSZ): + for cls in (TestHDF5PluginRW, TestPackage, TestRegisterFilter, TestGetFilters, TestSZ, TestBlosc2Plugins): test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(cls)) return test_suite From 7e437c0a303c8a3ccc90a99bbae46be247a84c23 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 6 Feb 2024 15:31:11 +0100 Subject: [PATCH 03/13] Set minimal version of blosc2 to use for tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6373c393..d8abb240 100644 --- a/setup.py +++ b/setup.py @@ -1307,7 +1307,7 @@ def get_version(): setup_requires=['setuptools', 'wheel'], extras_require={ 'dev': ['sphinx', 'sphinx_rtd_theme'], - 'test': ['blosc2', 'blosc2-grok'], + 'test': ['blosc2>=2.5.1', 'blosc2-grok>=0.2.2'], }, cmdclass=cmdclass, libraries=libraries, From c9ca5529a64c090541057d771daedd3adb4c2a29 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 6 Feb 2024 15:57:50 +0100 Subject: [PATCH 04/13] update ci to test with blosc2 --- .github/workflows/ci.yml | 4 ++-- .github/workflows/release.yml | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acece783..2e9a4a89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,8 +96,8 @@ jobs: ls dist - name: Install from source package - run: - pip install --pre dist/hdf5plugin* + run: | + pip install --pre "$(ls dist/hdf5plugin-*)[test]" --only-binary blosc2 || pip install --pre dist/hdf5plugin-* - name: Print python info run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 45e1a701..877441aa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,7 +45,7 @@ jobs: name: cibw-sdist path: dist - name: Install sdist - run: pip install --pre dist/hdf5plugin*.tar.gz + run: pip install --pre "$(ls dist/hdf5plugin*.tar.gz)[test]" - name: Run tests run: python test/test.py @@ -118,8 +118,10 @@ jobs: include: - python-version: '3.7' OLDEST_DEPENDENCIES: 'h5py==2.8.0' + EXTRAS: '' - python-version: '3.12' OLDEST_DEPENDENCIES: 'h5py==3.10.0' + EXTRAS: '[test]' steps: - uses: actions/checkout@v4 @@ -135,7 +137,7 @@ jobs: - name: Install h5py and hdf5plugin run: | pip install h5py - pip install --no-index --no-cache --find-links=./dist hdf5plugin --only-binary hdf5plugin + pip install --no-index --no-cache --find-links=./dist "hdf5plugin${{ matrix.EXTRAS }}" --only-binary hdf5plugin - name: Run test with latest h5py run: python test/test.py - name: Run test with oldest h5py From 23b7603ecb4405e432c0642599ae3e3a1ed5e6bf Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 6 Feb 2024 16:07:47 +0100 Subject: [PATCH 05/13] remove pre-release warning --- src/hdf5plugin/_filters.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hdf5plugin/_filters.py b/src/hdf5plugin/_filters.py index 39885b7e..60816ef0 100644 --- a/src/hdf5plugin/_filters.py +++ b/src/hdf5plugin/_filters.py @@ -221,8 +221,6 @@ def __init__(self, cname='lz4', clevel=5, shuffle=SHUFFLE): class Blosc2(_FilterRefClass): """``h5py.Group.create_dataset``'s compression arguments for using blosc2 filter. - WARNING: This is a pre-release version of the HDF5 filter, only for testing purpose. - It can be passed as keyword arguments: .. code-block:: python From 6fda9e6f680cef2fd7c9a03461b7e0fbee8fdee7 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 6 Feb 2024 16:08:05 +0100 Subject: [PATCH 06/13] add a note about Blosc2 plugins --- doc/usage.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/usage.rst b/doc/usage.rst index 7fc1c1e9..6a7d1411 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -18,6 +18,10 @@ In order to read compressed dataset with `h5py`_, use: It registers ``hdf5plugin`` supported compression filters with the HDF5 library used by `h5py`_. Hence, HDF5 compressed datasets can be read as any other dataset (see `h5py documentation `_). +.. note:: + + HDF5 datasets compressed with `Blosc2`_ can require additional plugins to enable decompression, such as `blosc2-grok `_ or `blosc2-openhtj2k `_. + Write compressed datasets +++++++++++++++++++++++++ From 8be92e283d5c921d75747da212d5953a54755164 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 7 Feb 2024 09:53:41 +0100 Subject: [PATCH 07/13] Add HDF5PLUGIN_SSSE3 env. var. --- .github/workflows/ci.yml | 1 + .github/workflows/release.yml | 3 ++- doc/install.rst | 4 ++++ package/debian11/rules | 1 + package/debian12/rules | 1 + setup.py | 31 +++++++++++++++++++++++++++---- 6 files changed, 36 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e9a4a89..97031f24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,5 +129,6 @@ jobs: CIBW_BUILD: cp39-macosx_* CIBW_ARCHS_MACOS: arm64 HDF5PLUGIN_SSE2: False + HDF5PLUGIN_SSSE3: False HDF5PLUGIN_AVX2: False HDF5PLUGIN_NATIVE: False diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 877441aa..5f371032 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -85,13 +85,14 @@ jobs: HDF5PLUGIN_OPENMP: "False" HDF5PLUGIN_NATIVE: "False" HDF5PLUGIN_SSE2: ${{ matrix.with_sse2 && 'True' || 'False' }} + HDF5PLUGIN_SSSE3: "False" HDF5PLUGIN_AVX2: "False" HDF5PLUGIN_AVX512: "False" HDF5PLUGIN_BMI2: "False" HDF5PLUGIN_CPP11: "True" HDF5PLUGIN_CPP14: "True" - CIBW_ENVIRONMENT_PASS_LINUX: HDF5PLUGIN_OPENMP HDF5PLUGIN_NATIVE HDF5PLUGIN_SSE2 HDF5PLUGIN_AVX2 HDF5PLUGIN_AVX512 HDF5PLUGIN_BMI2 HDF5PLUGIN_CPP11 HDF5PLUGIN_CPP14 + CIBW_ENVIRONMENT_PASS_LINUX: HDF5PLUGIN_OPENMP HDF5PLUGIN_NATIVE HDF5PLUGIN_SSE2 HDF5PLUGIN_SSSE3 HDF5PLUGIN_AVX2 HDF5PLUGIN_AVX512 HDF5PLUGIN_BMI2 HDF5PLUGIN_CPP11 HDF5PLUGIN_CPP14 # Use Python3.11 to build wheels that are compatible with all supported version of Python CIBW_BUILD: cp311-* diff --git a/doc/install.rst b/doc/install.rst index 22e74548..3435f08b 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -50,6 +50,9 @@ Available options * - ``HDF5PLUGIN_SSE2`` - Whether or not to compile with `SSE2`_ support. Default: True on ppc64le and when probed on x86, False otherwise + * - ``HDF5PLUGIN_SSSE3`` + - Whether or not to compile with `SSSE3`_ support. + Default: True when probed on x86, False otherwise * - ``HDF5PLUGIN_AVX2`` - Whether or not to compile with `AVX2`_ support. It requires enabling `SSE2`_ support. @@ -80,4 +83,5 @@ Note: Boolean options are passed as ``True`` or ``False``. .. _BMI2: https://en.wikipedia.org/wiki/X86_Bit_manipulation_instruction_set .. _IPP: https://en.wikipedia.org/wiki/Integrated_Performance_Primitives .. _SSE2: https://en.wikipedia.org/wiki/SSE2 +.. _SSSE3: https://en.wikipedia.org/wiki/SSSE3 .. _OpenMP: https://www.openmp.org/ diff --git a/package/debian11/rules b/package/debian11/rules index 6f74de2c..f02e7ce3 100755 --- a/package/debian11/rules +++ b/package/debian11/rules @@ -6,6 +6,7 @@ export PYBUILD_NAME=hdf5plugin # Build options export HDF5PLUGIN_NATIVE=False export HDF5PLUGIN_SSE2=True +export HDF5PLUGIN_SSSE3=False export HDF5PLUGIN_AVX2=False export HDF5PLUGIN_OPENMP=True export HDF5PLUGIN_CPP11=True diff --git a/package/debian12/rules b/package/debian12/rules index ff8cb157..9f604f83 100755 --- a/package/debian12/rules +++ b/package/debian12/rules @@ -6,6 +6,7 @@ export PYBUILD_NAME=hdf5plugin # Build options export HDF5PLUGIN_NATIVE=False export HDF5PLUGIN_SSE2=True +export HDF5PLUGIN_SSSE3=False export HDF5PLUGIN_AVX2=False export HDF5PLUGIN_OPENMP=True export HDF5PLUGIN_CPP11=True diff --git a/setup.py b/setup.py index d8abb240..9ab8710a 100644 --- a/setup.py +++ b/setup.py @@ -158,6 +158,11 @@ def __init__(self, compiler=None): else: self.sse2_compile_args = () + if self.ARCH in ('X86_32', 'X86_64'): + self.ssse3_compile_args = ('-mssse3',) # There is no /arch:SSSE3 + else: + self.ssse3_compile_args = () + if self.ARCH in ('X86_32', 'X86_64'): self.avx2_compile_args = ('-mavx2', '/arch:AVX2') else: @@ -201,6 +206,16 @@ def has_sse2(self) -> bool: return check_compile_flags(self.__compiler, "-msse2") return False # Disabled by default + def has_ssse3(self) -> bool: + """Check SSSE3 availability on host""" + if self.ARCH in ('X86_32', 'X86_64'): + if not has_cpu_flag('ssse3'): + return False # SSSE3 not available on host + if self.__compiler.compiler_type == "msvc": + return True + return check_compile_flags(self.__compiler, "-mssse3") + return False # Disabled by default + def has_avx2(self) -> bool: """Check AVX2 availability on host""" if self.ARCH in ('X86_32', 'X86_64'): @@ -273,20 +288,24 @@ def __init__( use_sse2 = host_config.has_sse2() if env_sse2 is None else env_sse2 == "True" self.__use_sse2 = bool(use_sse2) + env_ssse3 = os.environ.get("HDF5PLUGIN_SSSE3", None) + use_ssse3 = host_config.has_ssse3() if env_ssse3 is None else env_ssse3 == "True" + self.__use_ssse3 = bool(use_ssse3) + if use_avx2 is None: env_avx2 = os.environ.get("HDF5PLUGIN_AVX2", None) use_avx2 = host_config.has_avx2() if env_avx2 is None else env_avx2 == "True" - if use_avx2 and not use_sse2: + if use_avx2 and not (use_sse2 and use_ssse3): logger.error( - "use_avx2=True disabled: incompatible with use_sse2=False") + "use_avx2=True disabled: incompatible with use_sse2=False and use_ssse3=False") use_avx2 = False self.__use_avx2 = bool(use_avx2) env_avx512 = os.environ.get("HDF5PLUGIN_AVX512", None) use_avx512 = host_config.has_avx512() if env_avx512 is None else env_avx512 == "True" - if use_avx512 and not (use_sse2 and use_avx2): + if use_avx512 and not (use_sse2 and use_ssse3 and use_avx2): logger.error( - "use_avx512=True disabled: incompatible with use_sse2=False or use_avx2=False") + "use_avx512=True disabled: incompatible with use_sse2=False, use_ssse3=False and use_avx2=False") use_avx512 = False self.__use_avx512 = bool(use_avx512) @@ -304,6 +323,8 @@ def __init__( compile_args = [] if self.__use_sse2: compile_args.extend(host_config.sse2_compile_args) + if self.__use_ssse3: + compile_args.extend(host_config.ssse3_compile_args) if self.__use_avx2: compile_args.extend(host_config.avx2_compile_args) if self.__use_avx512: @@ -319,6 +340,7 @@ def __init__( use_cpp11 = property(lambda self: self.__use_cpp11) use_cpp14 = property(lambda self: self.__use_cpp14) use_sse2 = property(lambda self: self.__use_sse2) + use_ssse3 = property(lambda self: self.__use_ssse3) use_avx2 = property(lambda self: self.__use_avx2) use_avx512 = property(lambda self: self.__use_avx512) use_openmp = property(lambda self: self.__use_openmp) @@ -346,6 +368,7 @@ def get_config_string(self): 'native': self.use_native, 'bmi2': self.USE_BMI2, 'sse2': self.use_sse2, + 'ssse3': self.use_ssse3, 'avx2': self.use_avx2, 'avx512': self.use_avx512, 'cpp11': self.use_cpp11, From 5279e99c69c5fd8f380467548fb217a73092996a Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 7 Feb 2024 09:55:23 +0100 Subject: [PATCH 08/13] add missing HDF5PLUGIN_AVX512 to ci and packaging config --- .github/workflows/ci.yml | 1 + package/debian11/rules | 1 + package/debian12/rules | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97031f24..043eb1bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,4 +131,5 @@ jobs: HDF5PLUGIN_SSE2: False HDF5PLUGIN_SSSE3: False HDF5PLUGIN_AVX2: False + HDF5PLUGIN_AVX512: False HDF5PLUGIN_NATIVE: False diff --git a/package/debian11/rules b/package/debian11/rules index f02e7ce3..153134a8 100755 --- a/package/debian11/rules +++ b/package/debian11/rules @@ -8,6 +8,7 @@ export HDF5PLUGIN_NATIVE=False export HDF5PLUGIN_SSE2=True export HDF5PLUGIN_SSSE3=False export HDF5PLUGIN_AVX2=False +export HDF5PLUGIN_AVX512=False export HDF5PLUGIN_OPENMP=True export HDF5PLUGIN_CPP11=True diff --git a/package/debian12/rules b/package/debian12/rules index 9f604f83..6aec2c65 100755 --- a/package/debian12/rules +++ b/package/debian12/rules @@ -8,6 +8,7 @@ export HDF5PLUGIN_NATIVE=False export HDF5PLUGIN_SSE2=True export HDF5PLUGIN_SSSE3=False export HDF5PLUGIN_AVX2=False +export HDF5PLUGIN_AVX512=False export HDF5PLUGIN_OPENMP=True export HDF5PLUGIN_CPP11=True From 0c6b8df66f3ceb7bb7c7a71d6899d2b3d8dbd695 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 7 Feb 2024 16:24:17 +0100 Subject: [PATCH 09/13] Squashed 'src/c-blosc2/' changes from edcf96d4f..85a8f7391 ed9002c99 Getting ready for release 2.13.2 e0918a19b Document global registered filters and codecs 8c519b428 rely on __SSSE3__ macro rather that on architecture macros f0183767d Quodana does not seem designed for C (yet) 103e17cb5 Initialization file for quodana 472dc12f2 Add code quality assessment with Qodana ebfb27d36 Fixed some warnings found by clang-tidy 9f6eaf6f2 Fixed a couple of issues introduced in last PR a4d52c6db Adding decompress_file example, in order to have a counterpart of compress_file example 98ba23b25 Fix a compiler warning a52b8debf Add private functions section bd3d21365 Post 2.13.1 release actions done git-subtree-dir: src/c-blosc2 git-subtree-split: 85a8f739158cff6ce92e8d9d750366e18ce3172d --- ANNOUNCE.md | 8 ++- RELEASE_NOTES.md | 11 ++++ blosc/frame.c | 11 +++- doc/Doxyfile | 2 +- doc/reference/utility_variables.rst | 20 ++++++ examples/CMakeLists.txt | 10 ++- examples/decompress_file.c | 89 +++++++++++++++++++++++++++ include/blosc2.h | 13 ++-- include/blosc2/codecs-registry.h | 11 ++++ include/blosc2/filters-registry.h | 16 ++++- plugins/filters/bytedelta/bytedelta.c | 10 ++- 11 files changed, 184 insertions(+), 17 deletions(-) create mode 100644 examples/decompress_file.c diff --git a/ANNOUNCE.md b/ANNOUNCE.md index 634a3aa3..f0a271ee 100644 --- a/ANNOUNCE.md +++ b/ANNOUNCE.md @@ -1,9 +1,13 @@ -# Announcing C-Blosc2 2.13.1 +# Announcing C-Blosc2 2.13.2 A fast, compressed and persistent binary data store library for C. ## What is new? -This is a patch release for fixing a bug regarding the included files in `b2nd.h`. +This is a patch release for improving of SSSE3 detection on Visual Studio. +Also, documentation for the globally registered filters and codecs has been +added: +https://www.blosc.org/c-blosc2/reference/utility_variables.html#codes-for-filters +https://www.blosc.org/c-blosc2/reference/utility_variables.html#compressor-codecs For more info, please see the release notes in: diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 36f8441d..137cf357 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,17 @@ Release notes for C-Blosc2 ========================== +Changes from 2.13.1 to 2.13.2 +============================= + +* Better checking for `SSSE3` availability in Visual Studio. Probably fixes #546 too. + Thanks to @t20100 (Thomas Vincent) for the PR (#586). + +* Documented the globally registered filters and codecs. See: + https://www.blosc.org/c-blosc2/reference/utility_variables.html#codes-for-filters + https://www.blosc.org/c-blosc2/reference/utility_variables.html#compressor-codecs + + Changes from 2.13.0 to 2.13.1 ============================= diff --git a/blosc/frame.c b/blosc/frame.c index 1c5290b3..eada94fd 100644 --- a/blosc/frame.c +++ b/blosc/frame.c @@ -1358,7 +1358,7 @@ static int get_meta_from_header(blosc2_frame_s* frame, blosc2_schunk* schunk, ui schunk->metalayers[nmetalayer] = metalayer; // Populate the metalayer string - int8_t nslen = *idxp & (uint8_t)0x1F; + uint8_t nslen = *idxp & (uint8_t)0x1F; idxp += 1; header_pos += nslen; if (header_len < header_pos) { @@ -1538,7 +1538,7 @@ static int get_vlmeta_from_trailer(blosc2_frame_s* frame, blosc2_schunk* schunk, schunk->vlmetalayers[nmetalayer] = metalayer; // Populate the metalayer string - int8_t nslen = *idxp & (uint8_t)0x1F; + uint8_t nslen = *idxp & (uint8_t)0x1F; idxp += 1; trailer_pos += nslen; if (trailer_len < trailer_pos) { @@ -1885,6 +1885,11 @@ blosc2_schunk* frame_to_schunk(blosc2_frame_s* frame, bool copy, const blosc2_io } if (chunk_cbytes > (int32_t)prev_alloc) { data_chunk = realloc(data_chunk, chunk_cbytes); + if (data_chunk == NULL) { + BLOSC_TRACE_ERROR("Cannot realloc space for the data_chunk."); + rc = BLOSC2_ERROR_MEMORY_ALLOC; + break; + } prev_alloc = chunk_cbytes; } if (!frame->sframe) { @@ -3119,7 +3124,7 @@ void* frame_update_chunk(blosc2_frame_s* frame, int64_t nchunk, void* chunk, blo } // Add the new offset - int64_t sframe_chunk_id; + int64_t sframe_chunk_id = -1; if (frame->sframe) { if (offsets[nchunk] < 0) { sframe_chunk_id = -1; diff --git a/doc/Doxyfile b/doc/Doxyfile index df232aa6..3b56e428 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -7,7 +7,7 @@ GENERATE_RTF = NO CASE_SENSE_NAMES = NO GENERATE_HTML = NO GENERATE_XML = YES -RECURSIVE = NO +RECURSIVE = YES QUIET = YES JAVADOC_AUTOBRIEF = YES WARN_IF_UNDOCUMENTED = NO diff --git a/doc/reference/utility_variables.rst b/doc/reference/utility_variables.rst index 767a27a7..c0383315 100644 --- a/doc/reference/utility_variables.rst +++ b/doc/reference/utility_variables.rst @@ -33,6 +33,14 @@ Codes for filters .. doxygenenumvalue:: BLOSC_TRUNC_PREC +.. doxygenenumvalue:: BLOSC_FILTER_NDCELL + +.. doxygenenumvalue:: BLOSC_FILTER_NDMEAN + +.. doxygenenumvalue:: BLOSC_FILTER_BYTEDELTA + +.. doxygenenumvalue:: BLOSC_FILTER_INT_TRUNC + Compressor codecs ----------------- @@ -46,6 +54,18 @@ Compressor codecs .. doxygenenumvalue:: BLOSC_ZSTD +.. doxygenenumvalue:: BLOSC_CODEC_NDLZ + +.. doxygenenumvalue:: BLOSC_CODEC_ZFP_FIXED_ACCURACY + +.. doxygenenumvalue:: BLOSC_CODEC_ZFP_FIXED_PRECISION + +.. doxygenenumvalue:: BLOSC_CODEC_ZFP_FIXED_RATE + +.. doxygenenumvalue:: BLOSC_CODEC_OPENHTJ2K + +.. doxygenenumvalue:: BLOSC_CODEC_GROK + Compressor names ---------------- diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 385030f7..3d5c5479 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,7 +1,7 @@ # Examples with correspondingly named source files set(EXAMPLES contexts instrument_codec delta_schunk_ex multithread simple frame_metalayers noinit find_roots schunk_simple frame_simple schunk_postfilter urcodecs urfilters frame_vlmetalayers - sframe_simple frame_backed_schunk compress_file frame_offset frame_roundtrip get_set_slice get_blocksize) + sframe_simple frame_backed_schunk compress_file decompress_file frame_offset frame_roundtrip get_set_slice get_blocksize) add_subdirectory(b2nd) @@ -44,8 +44,12 @@ if(BUILD_TESTS) foreach(example ${EXAMPLES}) if(example STREQUAL compress_file) add_test(NAME test_example_${example} - COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ - "${PROJECT_BINARY_DIR}/CMakeCache.txt" CMakeCache.b2frame) + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ + "${PROJECT_BINARY_DIR}/CMakeCache.txt" CMakeCache.b2frame) + elseif(example STREQUAL decompress_file) + add_test(NAME test_example_${example} + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ + CMakeCache.b2frame CMakeCache-2.txt) else() add_test(NAME test_example_${example} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) diff --git a/examples/decompress_file.c b/examples/decompress_file.c new file mode 100644 index 00000000..fd443988 --- /dev/null +++ b/examples/decompress_file.c @@ -0,0 +1,89 @@ +/* + Copyright (c) 2024 The Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + Example program demonstrating use of the Blosc filter from C code. + + To compile this program: + + $ gcc decompress_file.c -o decompress_file -lblosc2 + + Example usage for compression/decompression verification: + + $ sha512sum compress_file + 385c93c..feaf38dbec compress_file + $ ./compress_file compress_file compress_file.bl2 + Blosc version info: 2.13.2.dev ($Date:: 2023-01-25 #$) + Compression ratio: 5.1 MB -> 2.0 MB (2.5x) + Compression time: 0.07 s, 72.8 MB/s + $ ./decompress_file compress_file.bl2 compress_file.1 + Blosc version info: 2.13.2.dev ($Date:: 2023-01-25 #$) + Decompression ratio: 2.0 MB -> 5.1 MB (0.4x) + Decompression time: 0.0343 s, 148.5 MB/s + $ sha512sum compress_file.1 + 385c93c..feaf38dbec compress_file.1 + + */ + +#include +#include + +#define KB 1024. +#define MB (1024*KB) +#define GB (1024*MB) + +int main(int argc, char* argv[]) { + blosc2_init(); + static char* data; + int32_t dsize; + int64_t nbytes, cbytes; + blosc_timestamp_t last, current; + double ttotal; + + if (argc != 3) { + fprintf(stderr, "Usage: decompress_file input_file.b2frame output_file\n"); + return -1; + } + + printf("Blosc version info: %s (%s)\n", + BLOSC2_VERSION_STRING, BLOSC2_VERSION_DATE); + + /* Open an existing super-chunk that is on-disk (frame). */ + blosc2_schunk* schunk = blosc2_schunk_open(argv[1]); + + data = (char*)malloc(schunk->chunksize); + + // Decompress the file + blosc_set_timestamp(&last); + FILE* foutput = fopen(argv[2], "wb"); + if (foutput == NULL) { + printf("Output file cannot be open."); + exit(1); + } + for (int nchunk = 0; nchunk < schunk->nchunks; nchunk++) { + dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, data, schunk->chunksize); + if (dsize < 0) { + fprintf(stderr, "Decompression error. Error code: %d\n", dsize); + return dsize; + } + fwrite(data, dsize, 1, foutput); + } + fclose(foutput); + + /* Gather some info */ + nbytes = schunk->nbytes; + cbytes = schunk->cbytes; + blosc_set_timestamp(¤t); + ttotal = blosc_elapsed_secs(last, current); + printf("Decompression ratio: %.1f MB -> %.1f MB (%.1fx)\n", + (float)cbytes / MB, (float)nbytes / MB, (1. * (float)cbytes) / (float)nbytes); + printf("Decompression time: %.3g s, %.1f MB/s\n", + ttotal, (float)nbytes / (ttotal * MB)); + + /* Free resources */ + free(data); + blosc2_schunk_free(schunk); + blosc2_destroy(); + return 0; +} diff --git a/include/blosc2.h b/include/blosc2.h index a8372cdb..73559441 100644 --- a/include/blosc2.h +++ b/include/blosc2.h @@ -83,10 +83,10 @@ extern "C" { /* Version numbers */ #define BLOSC2_VERSION_MAJOR 2 /* for major interface/format changes */ #define BLOSC2_VERSION_MINOR 13 /* for minor interface/format changes */ -#define BLOSC2_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define BLOSC2_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ -#define BLOSC2_VERSION_STRING "2.13.1" /* string version. Sync with above! */ -#define BLOSC2_VERSION_DATE "$Date:: 2023-01-25 #$" /* date version */ +#define BLOSC2_VERSION_STRING "2.13.2" /* string version. Sync with above! */ +#define BLOSC2_VERSION_DATE "$Date:: 2023-02-07 #$" /* date version */ /* The maximum number of dimensions for Blosc2 NDim arrays */ @@ -251,7 +251,7 @@ enum { BLOSC_BITSHUFFLE = 2, //!< Bit-wise shuffle. #endif // BLOSC_H BLOSC_DELTA = 3, //!< Delta filter. - BLOSC_TRUNC_PREC = 4, //!< Truncate mantissa precision; positive values in cparams.filters_meta will keep bits; negative values will reduce bits. + BLOSC_TRUNC_PREC = 4, //!< Truncate mantissa precision; positive values in `filters_meta` will keep bits; negative values will zero bits. BLOSC_LAST_FILTER = 5, //!< sentinel BLOSC_LAST_REGISTERED_FILTER = BLOSC2_GLOBAL_REGISTERED_FILTERS_START + BLOSC2_GLOBAL_REGISTERED_FILTERS - 1, //!< Determine the last registered filter. It is used to check if a filter is registered or not. @@ -2505,6 +2505,11 @@ BLOSC_EXPORT void blosc2_multidim_to_unidim(const int64_t *index, int8_t ndim, c BLOSC_EXPORT int blosc2_get_slice_nchunks(blosc2_schunk* schunk, int64_t *start, int64_t *stop, int64_t **chunks_idx); +/********************************************************************* + Private functions, these are here for convenience, + and are not meant to be included in public docs +*********************************************************************/ + // Private function needed in b2nd.h for deserializing meta static inline void swap_store(void *dest, const void *pa, int size) { uint8_t *pa_ = (uint8_t *) pa; diff --git a/include/blosc2/codecs-registry.h b/include/blosc2/codecs-registry.h index fdc25d8c..4c8ea34e 100644 --- a/include/blosc2/codecs-registry.h +++ b/include/blosc2/codecs-registry.h @@ -19,11 +19,22 @@ extern "C" { enum { BLOSC_CODEC_NDLZ = 32, + //!< Simple Lempel-Ziv compressor for NDim data. Experimental, mainly for teaching purposes. BLOSC_CODEC_ZFP_FIXED_ACCURACY = 33, + //!< ZFP compressor for fixed accuracy mode. The desired accuracy is set in `compcode_meta`. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/codecs/zfp/README.md BLOSC_CODEC_ZFP_FIXED_PRECISION = 34, + //!< ZFP compressor for fixed precision. The desired precision is set in `compcode_meta`. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/codecs/zfp/README.md BLOSC_CODEC_ZFP_FIXED_RATE = 35, + //!< ZFP compressor for fixed precision. The desired rate is set in `compcode_meta`. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/codecs/zfp/README.md BLOSC_CODEC_OPENHTJ2K = 36, + //!< OpenHTJ2K compressor for JPEG 2000 HT. + //!< See https://github.com/Blosc/blosc2_openhtj2k BLOSC_CODEC_GROK = 37, + //!< Grok compressor for JPEG 2000. + //!< See https://github.com/Blosc/blosc2_grok }; void register_codecs(void); diff --git a/include/blosc2/filters-registry.h b/include/blosc2/filters-registry.h index efcd7cfd..0dd3fb8a 100644 --- a/include/blosc2/filters-registry.h +++ b/include/blosc2/filters-registry.h @@ -17,10 +17,20 @@ extern "C" { enum { BLOSC_FILTER_NDCELL = 32, + //!< Simple filter for grouping NDim cell data together. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/filters/ndcell/README.md BLOSC_FILTER_NDMEAN = 33, - BLOSC_FILTER_BYTEDELTA_BUGGY = 34, // buggy version. See #524 - BLOSC_FILTER_BYTEDELTA = 35, // fixed version - BLOSC_FILTER_INT_TRUNC = 36, // truncate int precision; positive values in cparams.filters_meta will keep bits; negative values will reduce bits. + //!< Simple filter for replacing content of a NDim cell with its mean value. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/filters/ndmean/README.md + BLOSC_FILTER_BYTEDELTA_BUGGY = 34, + // buggy version. See #524 + BLOSC_FILTER_BYTEDELTA = 35, + //!< Byteshuffle + delta. Sometimes this can represent an advantage over + //!< @ref BLOSC_SHUFFLE or @ref BLOSC_BITSHUFFLE. + //!< See https://www.blosc.org/posts/bytedelta-enhance-compression-toolset/ + BLOSC_FILTER_INT_TRUNC = 36, + //!< Truncate int precision; positive values in `filter_meta` will keep bits; negative values will zero bits. + //!< This is similar to @ref BLOSC_TRUNC_PREC, but for integers instead of floating point data. }; void register_filters(void); diff --git a/plugins/filters/bytedelta/bytedelta.c b/plugins/filters/bytedelta/bytedelta.c index 87d4ed16..ba78120c 100644 --- a/plugins/filters/bytedelta/bytedelta.c +++ b/plugins/filters/bytedelta/bytedelta.c @@ -18,7 +18,15 @@ #include #include -#if defined __i386__ || defined _M_IX86 || defined __x86_64__ || defined _M_X64 +/* Define the __SSSE3__ symbol if compiling with Visual C++ and + targeting the minimum architecture level. +*/ +#if !defined(__SSSE3__) && defined(_MSC_VER) && \ + (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2)) + #define __SSSE3__ +#endif + +#if defined(__SSSE3__) // SSSE3 code path for x64/x64 #define CPU_HAS_SIMD 1 #include From 9d706752f540b31475e2f5339931daf2fa71f959 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 7 Feb 2024 16:26:46 +0100 Subject: [PATCH 10/13] update doc for blosc2 v2.13.2 update --- doc/information.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/information.rst b/doc/information.rst index 4218efb6..e8825195 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -76,7 +76,7 @@ HDF5 compression filters and compression libraries sources were obtained from: * `hdf5-blosc plugin `_ (v1.0.0) using `c-blosc `_ (v1.21.5), LZ4, Snappy, ZLib and ZStd. * hdf5-blosc2 plugin (from `PyTables `_ v3.9.2) - using `c-blosc2 `_ (v2.13.1), LZ4, ZLib and ZStd. + using `c-blosc2 `_ (v2.13.2), LZ4, ZLib and ZStd. * `FCIDECOMP plugin `_ (v1.0.2) using `CharLS `_ (1.x branch, commit `25160a4 `_). @@ -93,9 +93,9 @@ HDF5 compression filters and compression libraries sources were obtained from: Sources of compression libraries shared accross multiple filters were obtained from: -* `LZ4 v1.9.4 `_ +* `LZ4 v1.9.4 `_ * `Snappy v1.1.10 `_ -* `ZStd v1.5.5 `_ +* `ZStd v1.5.5 `_ * `ZLib v1.2.13 `_ When compiled with Intel IPP, the LZ4 compression library is replaced with `LZ4 v1.9.3 `_ patched with a patch from Intel IPP 2021.7.0. From 810ff28666d1f60634d2313e691573de4fe0842b Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 7 Feb 2024 16:29:10 +0100 Subject: [PATCH 11/13] add links to the list of blosc2 filters and codecs --- doc/usage.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/usage.rst b/doc/usage.rst index 6a7d1411..8adae727 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -21,6 +21,7 @@ Hence, HDF5 compressed datasets can be read as any other dataset (see `h5py docu .. note:: HDF5 datasets compressed with `Blosc2`_ can require additional plugins to enable decompression, such as `blosc2-grok `_ or `blosc2-openhtj2k `_. + See list of Blosc2 `filters `_ and `codecs `_. Write compressed datasets +++++++++++++++++++++++++ From 13bf15519a81560b9a771554d4e6ce0900096979 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 8 Feb 2024 10:15:17 +0100 Subject: [PATCH 12/13] remove -ffast-math compilation flag --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9ab8710a..0e6110a6 100644 --- a/setup.py +++ b/setup.py @@ -942,8 +942,8 @@ def get_blosc2_plugin(): include_dirs += get_zstd_clib('include_dirs') define_macros.append(('HAVE_ZSTD', 1)) - extra_compile_args += ['-O3', '-ffast-math', '-std=gnu99'] - extra_compile_args += ['/Ox', '/fp:fast'] + extra_compile_args += ['-O3', '-std=gnu99'] + extra_compile_args += ['/Ox'] extra_compile_args += ['-pthread'] extra_link_args += ['-pthread'] From 73b9478ce9f8bef183bfd147a4202f511f031590 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 8 Feb 2024 11:46:55 +0100 Subject: [PATCH 13/13] Change the way to install hdf5plugin wheel in the test --- .github/workflows/release.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5f371032..9acc0fa8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,10 +119,8 @@ jobs: include: - python-version: '3.7' OLDEST_DEPENDENCIES: 'h5py==2.8.0' - EXTRAS: '' - python-version: '3.12' OLDEST_DEPENDENCIES: 'h5py==3.10.0' - EXTRAS: '[test]' steps: - uses: actions/checkout@v4 @@ -135,10 +133,11 @@ jobs: pattern: cibw-wheels-* path: dist merge-multiple: true - - name: Install h5py and hdf5plugin + - name: Install hdf5plugin + # First select the right wheel from dist/ with pip download, then install it run: | - pip install h5py - pip install --no-index --no-cache --find-links=./dist "hdf5plugin${{ matrix.EXTRAS }}" --only-binary hdf5plugin + pip download --no-index --no-cache --no-deps --find-links=./dist --only-binary :all: hdf5plugin + pip install "$(ls ./hdf5plugin-*.whl)[test]" --only-binary blosc2 || pip install "$(ls ./hdf5plugin-*.whl)" - name: Run test with latest h5py run: python test/test.py - name: Run test with oldest h5py