Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Zstd to the supported plugin list #106

Merged
merged 6 commits into from
Dec 22, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion README.rst
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
hdf5plugin
==========

This module provides HDF5 compression filters (namely: blosc, bitshuffle, lz4, FCIDECOMP, ZFP) and registers them to the HDF5 library used by `h5py <https://www.h5py.org>`_.
This module provides HDF5 compression filters (namely: blosc, bitshuffle, lz4, FCIDECOMP, ZFP, zstd) and registers them to the HDF5 library used by `h5py <https://www.h5py.org>`_.

* Supported operating systems: Linux, Windows, macOS.
* Supported versions of Python: >= 3.4
Expand All @@ -17,6 +17,7 @@ The HDF5 plugin sources were obtained from:
* hdf5-blosc plugin (v1.0.0) and c-blosc (v1.17.0): https://github.com/Blosc/hdf5-blosc and https://github.com/Blosc/c-blosc
* FCIDECOMP plugin (v1.0.2) and CharLS (branch 1.x-master SHA1 ID:25160a42fb62e71e4b0ce081f5cb3f8bb73938b5): ftp://ftp.eumetsat.int/pub/OPS/out/test-data/Test-data-for-External-Users/MTG_FCI_Test-Data/FCI_Decompression_Software_V1.0.2/ and https://github.com/team-charls/charls.git
* HDF5-ZFP plugin (v1.0.1) and zfp (v0.5.5): https://github.com/LLNL/H5Z-ZFP and https://github.com/LLNL/zfp
* HDF5Plugin-Zstandard (commit d5afdb5) and zstd (v1.4.5): https://github.com/aparamon/HDF5Plugin-Zstandard and https://github.com/Blosc/c-blosc/tree/v1.20.1/internal-complibs/zstd-1.4.5

Installation
------------
Expand Down Expand Up @@ -76,6 +77,7 @@ Sample code:
- ``FCIDECOMP_ID``
- ``LZ4_ID``
- ``ZFP_ID``
- ``ZSTD_ID``

* ``FILTERS``: A dictionary mapping provided filters to their ID
* ``PLUGINS_PATH``: The directory where the provided filters library are stored.
Expand Down Expand Up @@ -225,6 +227,23 @@ The following compression modes are supported:
f.create_dataset('zfp_expert', data=numpy.random.random(100),
**hdf5plugin.Zfp(minbits=1, maxbits=16657, maxprec=64, minexp=-1074))

Zstd()
******

This class returns the compression options to feed into ``h5py.Group.create_dataset`` for using the Zstd filter:

It can be passed as keyword arguments.

Sample code:

.. code-block:: python

f = h5py.File('test.h5', 'w')
f.create_dataset('zstd', data=numpy.arange(100),
**hdf5plugin.Zstd())
f.close()


Dependencies
------------

Expand Down Expand Up @@ -265,5 +284,6 @@ Please read the different licenses:
* lz4: See `src/LZ4/COPYING <https://github.com/silx-kit/hdf5plugin/blob/master/src/LZ4/COPYING>`_
* FCIDECOMP: See `src/fcidecomp/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/fcidecomp/LICENSE.txt>`_ and `src/charls/src/License.txt <https://github.com/silx-kit/hdf5plugin/blob/master/src/charls/License.txt>`_
* zfp: See `src/H5Z-ZFP/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/H5Z-ZFP/LICENSE>`_ and `src/zfp/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/zfp/LICENSE>`_
* zstd: See `src/HDF5Plugin-Zstandard/LICENSE`

The HDF5 v1.10.5 headers (and Windows .lib file) used to build the filters are stored for convenience in the repository. The license is available here: `src/hdf5/COPYING <https://github.com/silx-kit/hdf5plugin/blob/master/src/hdf5/COPYING>`_.
12 changes: 11 additions & 1 deletion hdf5plugin/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

__authors__ = ["V.A. Sole", "H. Payno", "T. Vincent"]
__license__ = "MIT"
__date__ = "12/05/2020"
__date__ = "15/05/2020"

import ctypes as _ctypes
from glob import glob as _glob
Expand Down Expand Up @@ -70,13 +70,17 @@
ZFP_ID = 32013
"""ZFP filter ID"""

ZSTD_ID = 32015
"""Zstandard filter ID"""

FCIDECOMP_ID = 32018
"""FCIDECOMP filter ID"""

FILTERS = {'blosc': BLOSC_ID,
'bshuf': BSHUF_ID,
'lz4': LZ4_ID,
'zfp': ZFP_ID,
'zstd': ZSTD_ID,
'fcidecomp': FCIDECOMP_ID,
}
"""Mapping of filter name to HDF5 filter ID for available filters"""
Expand Down Expand Up @@ -268,6 +272,12 @@ def __init__(self,
_logger.info("filter options = %s" % (self.filter_options,))


class Zstd(_FilterRefClass):
"""h5py.Group.create_dataset's compression and compression_opts arguments for using FciDecomp filter.
"""
filter_id = ZSTD_ID


class FciDecomp(_FilterRefClass):
"""h5py.Group.create_dataset's compression and compression_opts arguments for using FciDecomp filter.
"""
Expand Down
10 changes: 8 additions & 2 deletions hdf5plugin/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def _test(self, filter_name, dtype=numpy.int32, lossless=True, **options):
"lz4": hdf5plugin.LZ4,
"fcidecomp": hdf5plugin.FciDecomp,
"zfp": hdf5plugin.Zfp,
"zstd": hdf5plugin.Zstd,
}[filter_name](**options)

# Write
Expand Down Expand Up @@ -148,16 +149,21 @@ def testZfp(self):

self._test('zfp', dtype=numpy.int32, reversible=True)

def testZstd(self):
"""Write/read test with Zstd filter plugin"""
self._test('zstd')


def suite():
test_suite = unittest.TestSuite()
for cls in (TestHDF5PluginRW,):
test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(cls))
return test_suite


def run_tests():
def run_tests(*args, **kwargs):
"""Run test complete test_suite"""
runner = unittest.TextTestRunner()
runner = unittest.TextTestRunner(*args, **kwargs)
success = runner.run(suite()).wasSuccessful()
print("Test suite " + ("succeeded" if success else "failed"))
return success
Expand Down
20 changes: 19 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
# ###########################################################################*/
__authors__ = ["V.A. Sole", "T. Vincent"]
__license__ = "MIT"
__date__ = "12/05/2020"
__date__ = "15/12/2020"


from glob import glob
Expand Down Expand Up @@ -449,6 +449,22 @@ def prefix(directory, files):
cpp11=cpp11_kwargs,
)

# HDF5Plugin-Zstandard
zstandard_dir = os.path.join("src", "HDF5Plugin-Zstandard")
zstandard_include_dirs = glob(blosc_dir + 'internal-complibs/zstd*')
zstandard_include_dirs += glob(blosc_dir + 'internal-complibs/zstd*/common')
zstandard_sources = [os.path.join(zstandard_dir, 'zstd_h5plugin.c')]
zstandard_sources += glob(blosc_dir +'internal-complibs/zstd*/*/*.c')
zstandard_depends = [os.path.join(zstandard_dir, 'zstd_h5plugin.h')]
zstandard_depends += glob(blosc_dir +'internal-complibs/zstd*/*/*.h')
zstandard_plugin = HDF5PluginExtension(
"hdf5plugin.plugins.libh5zstd",
sources=zstandard_sources,
depends=zstandard_depends,
include_dirs=zstandard_include_dirs,
define_macros=define_macros,
)


# lz4 plugin
# Source from https://github.com/nexusformat/HDF5-External-Filter-Plugins
Expand Down Expand Up @@ -545,8 +561,10 @@ def prefix(directory, files):
blosc_plugin,
fcidecomp_plugin,
h5zfp_plugin,
zstandard_plugin,
]


# setup

# ########## #
Expand Down
33 changes: 33 additions & 0 deletions src/HDF5Plugin-Zstandard/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Object files
*.o
*.ko
*.obj
*.elf

# Precompiled Headers
*.gch
*.pch

# Libraries
*.lib
*.a
*.la
*.lo

# Shared objects (inc. Windows DLLs)
*.dll
*.so
*.so.*
*.dylib

# Executables
*.exe
*.out
*.app
*.i*86
*.x86_64
*.hex

# Debug files
*.dSYM/
*.su
31 changes: 31 additions & 0 deletions src/HDF5Plugin-Zstandard/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
cmake_minimum_required(VERSION 2.8.10)
project(zstd_hdf5)

# options
set(PLUGIN_INSTALL_PATH "/usr/local/hdf5/lib/plugin" CACHE PATH
"Where to install the dynamic HDF5-plugin")

# sources
set(SOURCES zstd_h5plugin.c)
set(PLUGIN_SOURCES zstd_h5plugin.c)

# dependencies
if(MSVC)
# FindHDF5.cmake does not find Windows installations. Try to
# use an environment variable instead until the official "find"
# file can be updated for Windows.
#
# Note that you have to set this environment variable by hand.
file(TO_CMAKE_PATH "$ENV{HDF5_DIR}" HDF5_HINT)
set(HDF5_DIR ${HDF5_HINT} CACHE STRING "Path to HDF5 CMake config directory.")
find_package(HDF5 REQUIRED HINTS ${HDF5_DIR})
else(MSVC)
find_package(HDF5 REQUIRED)
endif(MSVC)
include_directories(${HDF5_INCLUDE_DIRS})

# HDF5 plugin as shared library
add_library(zstd_h5_plugin_shared SHARED ${PLUGIN_SOURCES})
set_target_properties(zstd_h5_plugin_shared PROPERTIES OUTPUT_NAME H5Zzstd)
target_link_libraries(zstd_h5_plugin_shared zstd ${HDF5_LIBRARIES})
install(TARGETS zstd_h5_plugin_shared DESTINATION ${PLUGIN_INSTALL_PATH} COMPONENT HDF5_FILTER_DEV)
Loading