Merge pull request #106 from silx-kit/zstd

Add Zstd to the supported plugin list

README.rst

@@ -1,7 +1,7 @@
 hdf5plugin
 ==========
 
-This module provides HDF5 compression filters (namely: blosc, bitshuffle, lz4, FCIDECOMP, ZFP) and registers them to the HDF5 library used by `h5py <https://www.h5py.org>`_.
+This module provides HDF5 compression filters (namely: blosc, bitshuffle, lz4, FCIDECOMP, ZFP, zstd) and registers them to the HDF5 library used by `h5py <https://www.h5py.org>`_.
 
 * Supported operating systems: Linux, Windows, macOS.
 * Supported versions of Python: >= 3.4
@@ -17,6 +17,7 @@ The HDF5 plugin sources were obtained from:
 * hdf5-blosc plugin (v1.0.0), c-blosc (v1.20.1) and snappy (v1.1.1): https://github.com/Blosc/hdf5-blosc, https://github.com/Blosc/c-blosc and https://github.com/Blosc/c-blosc/tree/v1.17.0/internal-complibs/snappy-1.1.1
 * FCIDECOMP plugin (v1.0.2) and CharLS (branch 1.x-master SHA1 ID:25160a42fb62e71e4b0ce081f5cb3f8bb73938b5): ftp://ftp.eumetsat.int/pub/OPS/out/test-data/Test-data-for-External-Users/MTG_FCI_Test-Data/FCI_Decompression_Software_V1.0.2/ and https://github.com/team-charls/charls.git 
 * HDF5-ZFP plugin (v1.0.1) and zfp (v0.5.5): https://github.com/LLNL/H5Z-ZFP and https://github.com/LLNL/zfp
+* HDF5Plugin-Zstandard (commit d5afdb5) and zstd (v1.4.5): https://github.com/aparamon/HDF5Plugin-Zstandard and https://github.com/Blosc/c-blosc/tree/v1.20.1/internal-complibs/zstd-1.4.5
 
 Installation
 ------------
@@ -76,6 +77,7 @@ Sample code:
   - ``FCIDECOMP_ID``
   - ``LZ4_ID``
   - ``ZFP_ID``
+  - ``ZSTD_ID``
 
 * ``FILTERS``: A dictionary mapping provided filters to their ID
 * ``PLUGINS_PATH``: The directory where the provided filters library are stored.
@@ -225,6 +227,23 @@ The following compression modes are supported:
         f.create_dataset('zfp_expert', data=numpy.random.random(100),
             **hdf5plugin.Zfp(minbits=1, maxbits=16657, maxprec=64, minexp=-1074))
 
+Zstd()
+******
+
+This class returns the compression options to feed into ``h5py.Group.create_dataset`` for using the Zstd filter:
+
+It can be passed as keyword arguments.
+
+Sample code:
+
+.. code-block:: python
+
+        f = h5py.File('test.h5', 'w')
+        f.create_dataset('zstd', data=numpy.arange(100),
+            **hdf5plugin.Zstd())
+        f.close()
+
+
 Dependencies
 ------------
 
@@ -265,5 +284,6 @@ Please read the different licenses:
 * lz4: See `src/LZ4/COPYING  <https://github.com/silx-kit/hdf5plugin/blob/master/src/LZ4/COPYING>`_
 * FCIDECOMP: See `src/fcidecomp/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/fcidecomp/LICENSE.txt>`_ and `src/charls/src/License.txt  <https://github.com/silx-kit/hdf5plugin/blob/master/src/charls/License.txt>`_
 * zfp: See `src/H5Z-ZFP/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/H5Z-ZFP/LICENSE>`_ and `src/zfp/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/zfp/LICENSE>`_
+* zstd: See `src/HDF5Plugin-Zstandard/LICENSE`
 
 The HDF5 v1.10.5 headers (and Windows .lib file) used to build the filters are stored for convenience in the repository. The license is available here: `src/hdf5/COPYING <https://github.com/silx-kit/hdf5plugin/blob/master/src/hdf5/COPYING>`_.

hdf5plugin/__init__.py

@@ -27,7 +27,7 @@
 
 __authors__ = ["V.A. Sole", "H. Payno", "T. Vincent"]
 __license__ = "MIT"
-__date__ = "12/05/2020"
+__date__ = "15/05/2020"
 
 import ctypes as _ctypes
 from glob import glob as _glob
@@ -70,13 +70,17 @@
 ZFP_ID = 32013
 """ZFP filter ID"""
 
+ZSTD_ID = 32015
+"""Zstandard filter ID"""
+
 FCIDECOMP_ID = 32018
 """FCIDECOMP filter ID"""
 
 FILTERS = {'blosc': BLOSC_ID,
            'bshuf': BSHUF_ID,
            'lz4': LZ4_ID,
            'zfp': ZFP_ID,
+           'zstd': ZSTD_ID,
            'fcidecomp': FCIDECOMP_ID,
            }
 """Mapping of filter name to HDF5 filter ID for available filters"""
@@ -268,6 +272,12 @@ def __init__(self,
         _logger.info("filter options = %s" % (self.filter_options,))
 
 
+class Zstd(_FilterRefClass):
+    """h5py.Group.create_dataset's compression and compression_opts arguments for using FciDecomp filter.
+    """
+    filter_id = ZSTD_ID
+
+
 class FciDecomp(_FilterRefClass):
     """h5py.Group.create_dataset's compression and compression_opts arguments for using FciDecomp filter.
     """

hdf5plugin/test.py

@@ -59,6 +59,7 @@ def _test(self, filter_name, dtype=numpy.int32, lossless=True, **options):
                 "lz4": hdf5plugin.LZ4,
                 "fcidecomp": hdf5plugin.FciDecomp,
                 "zfp": hdf5plugin.Zfp,
+                "zstd": hdf5plugin.Zstd,
                 }[filter_name](**options)
 
         # Write
@@ -148,16 +149,21 @@ def testZfp(self):
 
         self._test('zfp', dtype=numpy.int32, reversible=True)
 
+    def testZstd(self):
+        """Write/read test with Zstd filter plugin"""
+        self._test('zstd')
+
+
 def suite():
     test_suite = unittest.TestSuite()
     for cls in (TestHDF5PluginRW,):
         test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(cls))
     return test_suite
 
 
-def run_tests():
+def run_tests(*args, **kwargs):
     """Run test complete test_suite"""
-    runner = unittest.TextTestRunner()
+    runner = unittest.TextTestRunner(*args, **kwargs)
     success = runner.run(suite()).wasSuccessful()
     print("Test suite " + ("succeeded" if success else "failed"))
     return success

setup.py

@@ -24,7 +24,7 @@
 # ###########################################################################*/
 __authors__ = ["V.A. Sole", "T. Vincent"]
 __license__ = "MIT"
-__date__ = "12/05/2020"
+__date__ = "15/12/2020"
 
 
 from glob import glob
@@ -450,6 +450,22 @@ def prefix(directory, files):
     cpp11=cpp11_kwargs,
     )
 
+# HDF5Plugin-Zstandard
+zstandard_dir = os.path.join("src", "HDF5Plugin-Zstandard")
+zstandard_include_dirs = glob(blosc_dir + 'internal-complibs/zstd*')
+zstandard_include_dirs += glob(blosc_dir + 'internal-complibs/zstd*/common')
+zstandard_sources = [os.path.join(zstandard_dir, 'zstd_h5plugin.c')]
+zstandard_sources += glob(blosc_dir +'internal-complibs/zstd*/*/*.c')
+zstandard_depends = [os.path.join(zstandard_dir, 'zstd_h5plugin.h')]
+zstandard_depends += glob(blosc_dir +'internal-complibs/zstd*/*/*.h')
+zstandard_plugin = HDF5PluginExtension(
+    "hdf5plugin.plugins.libh5zstd",
+    sources=zstandard_sources,
+    depends=zstandard_depends,
+    include_dirs=zstandard_include_dirs,
+    define_macros=define_macros,
+    )
+
 
 # lz4 plugin
 # Source from https://github.com/nexusformat/HDF5-External-Filter-Plugins
@@ -546,8 +562,10 @@ def prefix(directory, files):
               blosc_plugin,
               fcidecomp_plugin,
               h5zfp_plugin,
+              zstandard_plugin,
               ]
 
+
 # setup
 
 # ########## #

src/HDF5Plugin-Zstandard/.gitignore

@@ -0,0 +1,33 @@
+# Object files
+*.o
+*.ko
+*.obj
+*.elf
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Libraries
+*.lib
+*.a
+*.la
+*.lo
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+
+# Executables
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+# Debug files
+*.dSYM/
+*.su

src/HDF5Plugin-Zstandard/CMakeLists.txt

@@ -0,0 +1,31 @@
+cmake_minimum_required(VERSION 2.8.10)
+project(zstd_hdf5)
+
+# options
+set(PLUGIN_INSTALL_PATH "/usr/local/hdf5/lib/plugin" CACHE PATH
+      "Where to install the dynamic HDF5-plugin")
+
+# sources
+set(SOURCES zstd_h5plugin.c)
+set(PLUGIN_SOURCES zstd_h5plugin.c)
+
+# dependencies
+if(MSVC)
+    # FindHDF5.cmake does not find Windows installations. Try to
+    # use an environment variable instead until the official "find"
+    # file can be updated for Windows.
+    #
+    # Note that you have to set this environment variable by hand.
+    file(TO_CMAKE_PATH "$ENV{HDF5_DIR}" HDF5_HINT)
+    set(HDF5_DIR ${HDF5_HINT} CACHE STRING "Path to HDF5 CMake config directory.")
+    find_package(HDF5 REQUIRED HINTS ${HDF5_DIR})
+else(MSVC)
+    find_package(HDF5 REQUIRED)
+endif(MSVC)
+include_directories(${HDF5_INCLUDE_DIRS})
+
+# HDF5 plugin as shared library
+add_library(zstd_h5_plugin_shared SHARED ${PLUGIN_SOURCES})
+set_target_properties(zstd_h5_plugin_shared PROPERTIES OUTPUT_NAME H5Zzstd)
+target_link_libraries(zstd_h5_plugin_shared zstd ${HDF5_LIBRARIES})
+install(TARGETS zstd_h5_plugin_shared DESTINATION ${PLUGIN_INSTALL_PATH} COMPONENT HDF5_FILTER_DEV)