Skip to content

Commit

Permalink
Merge pull request #106 from silx-kit/zstd
Browse files Browse the repository at this point in the history
Add Zstd to the supported plugin list
  • Loading branch information
vasole authored Dec 22, 2020
2 parents 1a1573e + 5e3eea5 commit ff381f0
Show file tree
Hide file tree
Showing 10 changed files with 463 additions and 5 deletions.
22 changes: 21 additions & 1 deletion README.rst
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
hdf5plugin
==========

This module provides HDF5 compression filters (namely: blosc, bitshuffle, lz4, FCIDECOMP, ZFP) and registers them to the HDF5 library used by `h5py <https://www.h5py.org>`_.
This module provides HDF5 compression filters (namely: blosc, bitshuffle, lz4, FCIDECOMP, ZFP, zstd) and registers them to the HDF5 library used by `h5py <https://www.h5py.org>`_.

* Supported operating systems: Linux, Windows, macOS.
* Supported versions of Python: >= 3.4
Expand All @@ -17,6 +17,7 @@ The HDF5 plugin sources were obtained from:
* hdf5-blosc plugin (v1.0.0), c-blosc (v1.20.1) and snappy (v1.1.1): https://github.com/Blosc/hdf5-blosc, https://github.com/Blosc/c-blosc and https://github.com/Blosc/c-blosc/tree/v1.17.0/internal-complibs/snappy-1.1.1
* FCIDECOMP plugin (v1.0.2) and CharLS (branch 1.x-master SHA1 ID:25160a42fb62e71e4b0ce081f5cb3f8bb73938b5): ftp://ftp.eumetsat.int/pub/OPS/out/test-data/Test-data-for-External-Users/MTG_FCI_Test-Data/FCI_Decompression_Software_V1.0.2/ and https://github.com/team-charls/charls.git
* HDF5-ZFP plugin (v1.0.1) and zfp (v0.5.5): https://github.com/LLNL/H5Z-ZFP and https://github.com/LLNL/zfp
* HDF5Plugin-Zstandard (commit d5afdb5) and zstd (v1.4.5): https://github.com/aparamon/HDF5Plugin-Zstandard and https://github.com/Blosc/c-blosc/tree/v1.20.1/internal-complibs/zstd-1.4.5

Installation
------------
Expand Down Expand Up @@ -76,6 +77,7 @@ Sample code:
- ``FCIDECOMP_ID``
- ``LZ4_ID``
- ``ZFP_ID``
- ``ZSTD_ID``

* ``FILTERS``: A dictionary mapping provided filters to their ID
* ``PLUGINS_PATH``: The directory where the provided filters library are stored.
Expand Down Expand Up @@ -225,6 +227,23 @@ The following compression modes are supported:
f.create_dataset('zfp_expert', data=numpy.random.random(100),
**hdf5plugin.Zfp(minbits=1, maxbits=16657, maxprec=64, minexp=-1074))
Zstd()
******

This class returns the compression options to feed into ``h5py.Group.create_dataset`` for using the Zstd filter:

It can be passed as keyword arguments.

Sample code:

.. code-block:: python
f = h5py.File('test.h5', 'w')
f.create_dataset('zstd', data=numpy.arange(100),
**hdf5plugin.Zstd())
f.close()
Dependencies
------------

Expand Down Expand Up @@ -265,5 +284,6 @@ Please read the different licenses:
* lz4: See `src/LZ4/COPYING <https://github.com/silx-kit/hdf5plugin/blob/master/src/LZ4/COPYING>`_
* FCIDECOMP: See `src/fcidecomp/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/fcidecomp/LICENSE.txt>`_ and `src/charls/src/License.txt <https://github.com/silx-kit/hdf5plugin/blob/master/src/charls/License.txt>`_
* zfp: See `src/H5Z-ZFP/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/H5Z-ZFP/LICENSE>`_ and `src/zfp/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/zfp/LICENSE>`_
* zstd: See `src/HDF5Plugin-Zstandard/LICENSE`

The HDF5 v1.10.5 headers (and Windows .lib file) used to build the filters are stored for convenience in the repository. The license is available here: `src/hdf5/COPYING <https://github.com/silx-kit/hdf5plugin/blob/master/src/hdf5/COPYING>`_.
12 changes: 11 additions & 1 deletion hdf5plugin/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

__authors__ = ["V.A. Sole", "H. Payno", "T. Vincent"]
__license__ = "MIT"
__date__ = "12/05/2020"
__date__ = "15/05/2020"

import ctypes as _ctypes
from glob import glob as _glob
Expand Down Expand Up @@ -70,13 +70,17 @@
ZFP_ID = 32013
"""ZFP filter ID"""

ZSTD_ID = 32015
"""Zstandard filter ID"""

FCIDECOMP_ID = 32018
"""FCIDECOMP filter ID"""

FILTERS = {'blosc': BLOSC_ID,
'bshuf': BSHUF_ID,
'lz4': LZ4_ID,
'zfp': ZFP_ID,
'zstd': ZSTD_ID,
'fcidecomp': FCIDECOMP_ID,
}
"""Mapping of filter name to HDF5 filter ID for available filters"""
Expand Down Expand Up @@ -268,6 +272,12 @@ def __init__(self,
_logger.info("filter options = %s" % (self.filter_options,))


class Zstd(_FilterRefClass):
"""h5py.Group.create_dataset's compression and compression_opts arguments for using FciDecomp filter.
"""
filter_id = ZSTD_ID


class FciDecomp(_FilterRefClass):
"""h5py.Group.create_dataset's compression and compression_opts arguments for using FciDecomp filter.
"""
Expand Down
10 changes: 8 additions & 2 deletions hdf5plugin/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def _test(self, filter_name, dtype=numpy.int32, lossless=True, **options):
"lz4": hdf5plugin.LZ4,
"fcidecomp": hdf5plugin.FciDecomp,
"zfp": hdf5plugin.Zfp,
"zstd": hdf5plugin.Zstd,
}[filter_name](**options)

# Write
Expand Down Expand Up @@ -148,16 +149,21 @@ def testZfp(self):

self._test('zfp', dtype=numpy.int32, reversible=True)

def testZstd(self):
"""Write/read test with Zstd filter plugin"""
self._test('zstd')


def suite():
test_suite = unittest.TestSuite()
for cls in (TestHDF5PluginRW,):
test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(cls))
return test_suite


def run_tests():
def run_tests(*args, **kwargs):
"""Run test complete test_suite"""
runner = unittest.TextTestRunner()
runner = unittest.TextTestRunner(*args, **kwargs)
success = runner.run(suite()).wasSuccessful()
print("Test suite " + ("succeeded" if success else "failed"))
return success
Expand Down
20 changes: 19 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
# ###########################################################################*/
__authors__ = ["V.A. Sole", "T. Vincent"]
__license__ = "MIT"
__date__ = "12/05/2020"
__date__ = "15/12/2020"


from glob import glob
Expand Down Expand Up @@ -450,6 +450,22 @@ def prefix(directory, files):
cpp11=cpp11_kwargs,
)

# HDF5Plugin-Zstandard
zstandard_dir = os.path.join("src", "HDF5Plugin-Zstandard")
zstandard_include_dirs = glob(blosc_dir + 'internal-complibs/zstd*')
zstandard_include_dirs += glob(blosc_dir + 'internal-complibs/zstd*/common')
zstandard_sources = [os.path.join(zstandard_dir, 'zstd_h5plugin.c')]
zstandard_sources += glob(blosc_dir +'internal-complibs/zstd*/*/*.c')
zstandard_depends = [os.path.join(zstandard_dir, 'zstd_h5plugin.h')]
zstandard_depends += glob(blosc_dir +'internal-complibs/zstd*/*/*.h')
zstandard_plugin = HDF5PluginExtension(
"hdf5plugin.plugins.libh5zstd",
sources=zstandard_sources,
depends=zstandard_depends,
include_dirs=zstandard_include_dirs,
define_macros=define_macros,
)


# lz4 plugin
# Source from https://github.com/nexusformat/HDF5-External-Filter-Plugins
Expand Down Expand Up @@ -546,8 +562,10 @@ def prefix(directory, files):
blosc_plugin,
fcidecomp_plugin,
h5zfp_plugin,
zstandard_plugin,
]


# setup

# ########## #
Expand Down
33 changes: 33 additions & 0 deletions src/HDF5Plugin-Zstandard/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Object files
*.o
*.ko
*.obj
*.elf

# Precompiled Headers
*.gch
*.pch

# Libraries
*.lib
*.a
*.la
*.lo

# Shared objects (inc. Windows DLLs)
*.dll
*.so
*.so.*
*.dylib

# Executables
*.exe
*.out
*.app
*.i*86
*.x86_64
*.hex

# Debug files
*.dSYM/
*.su
31 changes: 31 additions & 0 deletions src/HDF5Plugin-Zstandard/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
cmake_minimum_required(VERSION 2.8.10)
project(zstd_hdf5)

# options
set(PLUGIN_INSTALL_PATH "/usr/local/hdf5/lib/plugin" CACHE PATH
"Where to install the dynamic HDF5-plugin")

# sources
set(SOURCES zstd_h5plugin.c)
set(PLUGIN_SOURCES zstd_h5plugin.c)

# dependencies
if(MSVC)
# FindHDF5.cmake does not find Windows installations. Try to
# use an environment variable instead until the official "find"
# file can be updated for Windows.
#
# Note that you have to set this environment variable by hand.
file(TO_CMAKE_PATH "$ENV{HDF5_DIR}" HDF5_HINT)
set(HDF5_DIR ${HDF5_HINT} CACHE STRING "Path to HDF5 CMake config directory.")
find_package(HDF5 REQUIRED HINTS ${HDF5_DIR})
else(MSVC)
find_package(HDF5 REQUIRED)
endif(MSVC)
include_directories(${HDF5_INCLUDE_DIRS})

# HDF5 plugin as shared library
add_library(zstd_h5_plugin_shared SHARED ${PLUGIN_SOURCES})
set_target_properties(zstd_h5_plugin_shared PROPERTIES OUTPUT_NAME H5Zzstd)
target_link_libraries(zstd_h5_plugin_shared zstd ${HDF5_LIBRARIES})
install(TARGETS zstd_h5_plugin_shared DESTINATION ${PLUGIN_INSTALL_PATH} COMPONENT HDF5_FILTER_DEV)
Loading

0 comments on commit ff381f0

Please sign in to comment.