From 7fe328128d7f8695fadafb3e7853f4ae350b522e Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 22 Nov 2022 15:17:43 +0100 Subject: [PATCH 1/8] refactor register and expose register_filter function --- src/hdf5plugin/__init__.py | 2 +- src/hdf5plugin/_utils.py | 157 ++++++++++++++++++++++++------------- 2 files changed, 104 insertions(+), 55 deletions(-) diff --git a/src/hdf5plugin/__init__.py b/src/hdf5plugin/__init__.py index d7328562..eb1e9c27 100644 --- a/src/hdf5plugin/__init__.py +++ b/src/hdf5plugin/__init__.py @@ -38,7 +38,7 @@ from ._filters import ZSTD_ID, Zstd # noqa from ._filters import SZ_ID, SZ # noqa -from ._utils import get_config, PLUGIN_PATH # noqa +from ._utils import get_config, PLUGIN_PATH, register_filter # noqa # Backward compatibility from ._config import build_config as config # noqa diff --git a/src/hdf5plugin/_utils.py b/src/hdf5plugin/_utils.py index 1e7abdeb..5075648f 100644 --- a/src/hdf5plugin/_utils.py +++ b/src/hdf5plugin/_utils.py @@ -44,71 +44,120 @@ """Directory where the provided HDF5 filter plugins are stored.""" -def _init_filters(): - """Initialise and register HDF5 filters with h5py +def is_filter_available(name): + """Returns whether filter is already registered or not. - Generator of tuples: (filename, library handle) + :param str name: Name of the filter (See `hdf5plugin.FILTERS`) + :return: True if filter is registered, False if not and + None if it cannot be checked (libhdf5 not supporting it) + :rtype: Union[bool,None] """ + filter_id = FILTERS[name] + hdf5_version = h5py.h5.get_libversion() + if hdf5_version < (1, 8, 20) or (1, 10) <= hdf5_version < (1, 10, 2): + return None # h5z.filter_avail not available + return h5py.h5z.filter_avail(filter_id) > 0 - for name, filter_id in FILTERS.items(): - # Skip filters that were not embedded - if name not in build_config.embedded_filters: - logger.debug("%s filter not available in this build of hdf5plugin.", name) - continue - # Check if filter is already loaded (not on buggy HDF5 versions) - if (1, 8, 20) <= hdf5_version < (1, 10) or hdf5_version >= (1, 10, 2): - if h5py.h5z.filter_avail(filter_id): - logger.info("%s filter already loaded, skip it.", name) - yield name, ("unknown", None) - continue - - # Load DLL - filename = glob.glob(os.path.join( - PLUGIN_PATH, 'libh5' + name + '*' + build_config.filter_file_extension)) - if len(filename): - filename = filename[0] - else: - logger.error("Cannot initialize filter %s: File not found", name) - continue - try: - lib = ctypes.CDLL(filename) - except OSError: - logger.error("Failed to load filter %s: %s", name, filename) - logger.error(traceback.format_exc()) - continue +REGISTERED_FILTERS = {} +"""Store hdf5plugin registered filters as a mapping: name: (filename, ctypes.CDLL)""" - if sys.platform.startswith('win'): - # Use register_filter function to register filter - lib.register_filter.restype = ctypes.c_int - retval = lib.register_filter() - else: - # Use init_filter function to initialize DLL and register filter - lib.init_filter.argtypes = [ctypes.c_char_p] - lib.init_filter.restype = ctypes.c_int - retval = lib.init_filter( - bytes(h5py.h5z.__file__, encoding='utf-8')) - - if retval < 0: - logger.error("Cannot initialize filter %s: %d", name, retval) - continue - logger.debug("Registered filter: %s (%s)", name, filename) - yield name, (filename, lib) +def register_filter(name): + """Register a filter given its name + Unregister the previously registered filter if any. -_filters = dict(_init_filters()) # Store loaded filters + :param str name: Name of the filter (See `hdf5plugin.FILTERS`) + :return: Library filename if filter was successfully registered, None otherwise + :rtype: Union[str,None] + """ + if name not in FILTERS: + raise ValueError("Unknown filter name: %s" % name) + + if name not in build_config.embedded_filters: + logger.debug("%s filter not available in this build of hdf5plugin.", name) + return False + + # Unregister existing filter + filter_id = FILTERS[name] + is_avail = is_filter_available(name) + if is_avail is True: + if not h5py.h5z.unregister_filter(filter_id): + logger.error("Failed to unregister filter %s (%d)" % (name, filter_id)) + return False + elif is_avail is None: # Cannot probe filter availability + try: + h5py.h5z.unregister_filter(filter_id) + except RuntimeError: + logger.debug("Filter %s (%d) not unregistered" % (name, filter_id)) + logger.debug(traceback.format_exc()) + REGISTERED_FILTERS.pop(name, None) + + # Load DLL + filename = glob.glob(os.path.join( + PLUGIN_PATH, 'libh5' + name + '*' + build_config.filter_file_extension)) + if len(filename): + filename = filename[0] + else: + logger.error("Cannot initialize filter %s: File not found", name) + return False + try: + lib = ctypes.CDLL(filename) + except OSError: + logger.error("Failed to load filter %s: %s", name, filename) + logger.error(traceback.format_exc()) + return False + + if sys.platform.startswith('win'): + # Use register_filter function to register filter + lib.register_filter.restype = ctypes.c_int + retval = lib.register_filter() + else: + # Use init_filter function to initialize DLL and register filter + lib.init_filter.argtypes = [ctypes.c_char_p] + lib.init_filter.restype = ctypes.c_int + retval = lib.init_filter( + bytes(h5py.h5z.__file__, encoding='utf-8')) + + if retval < 0: + logger.error("Cannot initialize filter %s: %d", name, retval) + return False + + logger.debug("Registered filter: %s (%s)", name, filename) + REGISTERED_FILTERS[name] = filename, lib + return True + + +HDF5PluginConfig = namedtuple( + 'HDF5PluginConfig', + ('build_config', 'registered_filters'), +) def get_config(): """Provides information about build configuration and filters registered by hdf5plugin. """ - HDF5PluginConfig = namedtuple( - 'HDF5PluginConfig', - ('build_config', 'registered_filters'), - ) - return HDF5PluginConfig( - build_config=build_config, - registered_filters=dict((name, filename) for name, (filename, lib) in _filters.items()), - ) + registered_filters = {} + for name in FILTERS: + info = REGISTERED_FILTERS.get(name) + if info is not None: # Registered by hdf5plugin + if is_filter_available(name) in (True, None): + registered_filters[name] = info[0] + elif is_filter_available(name) is True: # Registered elsewhere + registered_filters[name] = "unknown" + + return HDF5PluginConfig(build_config, registered_filters) + + +def init_filters(): + """Initialise and register HDF5 filters""" + for name in FILTERS: + if is_filter_available(name) is True: + logger.info("%s filter already loaded, skip it.", name) + continue + register_filter(name) + + +init_filters() From 6538d9d1f5bc1d9a7c5ab2e1987c69c0d4aab669 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 22 Nov 2022 15:51:08 +0100 Subject: [PATCH 2/8] Update doc --- doc/usage.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/usage.rst b/doc/usage.rst index c39a85de..ac20e998 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -112,6 +112,19 @@ Functions: .. autofunction:: get_config +Manage registered filters ++++++++++++++++++++++++++ + +When imported, `hdf5plugin` initialises and registers the filters it embeds if there is no already registered filters for the corresponding filter IDs. + +`h5py`_ gives access to HDF5 functions handling registered filters in `h5py.h5z`_. +This module allows checking the filter availability and registering/unregistering filters. + +`hdf5plugin` provides an extra `register_filter` function to register the filters it provides, e.g., to override an already loaded filters. +Registering with this function is required to perform additional initialisation and enable writing compressed data with the given filter. + +.. autofunction:: register_filter + Use HDF5 filters in other applications ++++++++++++++++++++++++++++++++++++++ @@ -128,4 +141,5 @@ should allow MatLab or IDL users to read data compressed using the supported plu Setting the ``HDF5_PLUGIN_PATH`` environment variable allows already existing programs or Python code to read compressed data without any modification. .. _h5py: https://www.h5py.org +.. _h5py.h5z: https://github.com/h5py/h5py/blob/master/h5py/h5z.pyx .. _h5py.Group.create_dataset: https://docs.h5py.org/en/stable/high/group.html#h5py.Group.create_dataset From 6431616ddafc2254b533f5fe624d65926aaf1352 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 22 Nov 2022 17:44:24 +0100 Subject: [PATCH 3/8] Add test for register_filter --- src/hdf5plugin/test.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/hdf5plugin/test.py b/src/hdf5plugin/test.py index 667ab905..04c57cfa 100644 --- a/src/hdf5plugin/test.py +++ b/src/hdf5plugin/test.py @@ -38,8 +38,8 @@ def should_test(filter_name): return filter_name in hdf5plugin.config.embedded_filters or h5py.h5z.filter_avail(filter_id) -class TestHDF5PluginRW(unittest.TestCase): - """Test write/read a HDF5 file with the plugins""" +class BaseTestHDF5PluginRW(unittest.TestCase): + """Base class for testing write/read HDF5 dataset with the plugins""" @classmethod def setUpClass(cls): @@ -107,6 +107,10 @@ def _test(self, os.remove(filename) return filters[0] + +class TestHDF5PluginRW(BaseTestHDF5PluginRW): + """Test write/read a HDF5 file with the plugins""" + @unittest.skipUnless(should_test("bshuf"), "Bitshuffle filter not available") def testDepreactedBitshuffle(self): """Write/read test with bitshuffle filter plugin""" @@ -265,9 +269,28 @@ def testVersion(self): self.assertIsInstance(version_info.serial, int) +class TestRegisterFilter(BaseTestHDF5PluginRW): + """Test usage of the register_filter function""" + + @unittest.skipUnless(hdf5plugin.config.embedded_filters, "No embedded filters") + def test(self): + """Re-register all embedded filters""" + for name in hdf5plugin.config.embedded_filters: + with self.subTest(name=name): + status = hdf5plugin.register_filter(name) + self.assertTrue(status) + + if name == 'fcidecomp': + self._test('fcidecomp', dtype=numpy.uint8) + elif name in ('sz', 'zfp'): + self._test(name, dtype=numpy.float32, lossless=False) + else: + self._test(name) + + def suite(): test_suite = unittest.TestSuite() - for cls in (TestHDF5PluginRW, TestPackage): + for cls in (TestHDF5PluginRW, TestPackage, TestRegisterFilter): test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(cls)) return test_suite From a95ade86bbf874bb710d9bbf9f880ba8d1baf7d4 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 22 Nov 2022 18:25:08 +0100 Subject: [PATCH 4/8] Add register function and make it the public API --- doc/usage.rst | 4 ++-- src/hdf5plugin/__init__.py | 2 +- src/hdf5plugin/_utils.py | 36 ++++++++++++++++++++++++++---------- src/hdf5plugin/test.py | 30 ++++++++++++++++++++---------- 4 files changed, 49 insertions(+), 23 deletions(-) diff --git a/doc/usage.rst b/doc/usage.rst index ac20e998..482975cc 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -120,10 +120,10 @@ When imported, `hdf5plugin` initialises and registers the filters it embeds if t `h5py`_ gives access to HDF5 functions handling registered filters in `h5py.h5z`_. This module allows checking the filter availability and registering/unregistering filters. -`hdf5plugin` provides an extra `register_filter` function to register the filters it provides, e.g., to override an already loaded filters. +`hdf5plugin` provides an extra `register` function to register the filters it provides, e.g., to override an already loaded filters. Registering with this function is required to perform additional initialisation and enable writing compressed data with the given filter. -.. autofunction:: register_filter +.. autofunction:: register Use HDF5 filters in other applications ++++++++++++++++++++++++++++++++++++++ diff --git a/src/hdf5plugin/__init__.py b/src/hdf5plugin/__init__.py index eb1e9c27..654b39e3 100644 --- a/src/hdf5plugin/__init__.py +++ b/src/hdf5plugin/__init__.py @@ -38,7 +38,7 @@ from ._filters import ZSTD_ID, Zstd # noqa from ._filters import SZ_ID, SZ # noqa -from ._utils import get_config, PLUGIN_PATH, register_filter # noqa +from ._utils import get_config, PLUGIN_PATH, register # noqa # Backward compatibility from ._config import build_config as config # noqa diff --git a/src/hdf5plugin/_utils.py b/src/hdf5plugin/_utils.py index 5075648f..0723f085 100644 --- a/src/hdf5plugin/_utils.py +++ b/src/hdf5plugin/_utils.py @@ -70,8 +70,8 @@ def register_filter(name): Unregister the previously registered filter if any. :param str name: Name of the filter (See `hdf5plugin.FILTERS`) - :return: Library filename if filter was successfully registered, None otherwise - :rtype: Union[str,None] + :return: True if successfully registered, False otherwise + :rtype: bool """ if name not in FILTERS: raise ValueError("Unknown filter name: %s" % name) @@ -83,6 +83,7 @@ def register_filter(name): # Unregister existing filter filter_id = FILTERS[name] is_avail = is_filter_available(name) + # TODO h5py>=2.10 if is_avail is True: if not h5py.h5z.unregister_filter(filter_id): logger.error("Failed to unregister filter %s (%d)" % (name, filter_id)) @@ -151,13 +152,28 @@ def get_config(): return HDF5PluginConfig(build_config, registered_filters) -def init_filters(): - """Initialise and register HDF5 filters""" - for name in FILTERS: - if is_filter_available(name) is True: - logger.info("%s filter already loaded, skip it.", name) - continue - register_filter(name) +def register(filters=tuple(FILTERS.keys()), force=True): + """Initialise and register `hdf5plugin` embedded filters given their names. + + Unregister corresponding previously registered filters if any. + :param Union[str.Tuple[str]] filters: + Filter name or sequence of filter names (See `hdf5plugin.FILTERS`). + :param bool force: + True to register the filter even if a corresponding one if already available. + False to skip already available filters. + :return: True if all filters were registered successfully, False otherwise. + :rtype: bool + """ + if isinstance(filters, str): + names = (filters,) + + status = True + for filter_name in filters: + if not force and is_filter_available(filter_name) is True: + logger.info("%s filter already loaded, skip it.", filter_name) + continue + status = status and register_filter(filter_name) + return status -init_filters() +register(force=False) diff --git a/src/hdf5plugin/test.py b/src/hdf5plugin/test.py index 04c57cfa..5ab34f17 100644 --- a/src/hdf5plugin/test.py +++ b/src/hdf5plugin/test.py @@ -270,22 +270,32 @@ def testVersion(self): class TestRegisterFilter(BaseTestHDF5PluginRW): - """Test usage of the register_filter function""" + """Test usage of the register function""" + + def _simple_test(self, filter_name): + if filter_name == 'fcidecomp': + self._test('fcidecomp', dtype=numpy.uint8) + elif filter_name in ('sz', 'zfp'): + self._test(filter_name, dtype=numpy.float32, lossless=False) + else: + self._test(filter_name) @unittest.skipUnless(hdf5plugin.config.embedded_filters, "No embedded filters") - def test(self): - """Re-register all embedded filters""" + def test_register_single_filter(self): + """Re-register embedded filters one at a time""" for name in hdf5plugin.config.embedded_filters: with self.subTest(name=name): - status = hdf5plugin.register_filter(name) + status = hdf5plugin.register(name, force=True) self.assertTrue(status) + self._simple_test(name) - if name == 'fcidecomp': - self._test('fcidecomp', dtype=numpy.uint8) - elif name in ('sz', 'zfp'): - self._test(name, dtype=numpy.float32, lossless=False) - else: - self._test(name) + @unittest.skipUnless(hdf5plugin.config.embedded_filters, "No embedded filters") + def test_register_all_filters(self): + """Re-register embedded filters all at once""" + status = hdf5plugin.register() + for name in hdf5plugin.config.embedded_filters: + with self.subTest(name=name): + self._simple_test(name) def suite(): From 1f31c4e54c745358c51faae5fb43453d6b7cdc69 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 23 Nov 2022 11:24:45 +0100 Subject: [PATCH 5/8] rename REGISTERED_FITLERS to registered_filters (it's not a constant) --- src/hdf5plugin/_utils.py | 18 +++++++++--------- src/hdf5plugin/test.py | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/hdf5plugin/_utils.py b/src/hdf5plugin/_utils.py index 0723f085..7f3d377c 100644 --- a/src/hdf5plugin/_utils.py +++ b/src/hdf5plugin/_utils.py @@ -60,7 +60,7 @@ def is_filter_available(name): return h5py.h5z.filter_avail(filter_id) > 0 -REGISTERED_FILTERS = {} +registered_filters = {} """Store hdf5plugin registered filters as a mapping: name: (filename, ctypes.CDLL)""" @@ -94,7 +94,7 @@ def register_filter(name): except RuntimeError: logger.debug("Filter %s (%d) not unregistered" % (name, filter_id)) logger.debug(traceback.format_exc()) - REGISTERED_FILTERS.pop(name, None) + registered_filters.pop(name, None) # Load DLL filename = glob.glob(os.path.join( @@ -127,7 +127,7 @@ def register_filter(name): return False logger.debug("Registered filter: %s (%s)", name, filename) - REGISTERED_FILTERS[name] = filename, lib + registered_filters[name] = filename, lib return True @@ -140,16 +140,16 @@ def register_filter(name): def get_config(): """Provides information about build configuration and filters registered by hdf5plugin. """ - registered_filters = {} + filters = {} for name in FILTERS: - info = REGISTERED_FILTERS.get(name) + info = registered_filters.get(name) if info is not None: # Registered by hdf5plugin if is_filter_available(name) in (True, None): - registered_filters[name] = info[0] + filters[name] = info[0] elif is_filter_available(name) is True: # Registered elsewhere - registered_filters[name] = "unknown" + filters[name] = "unknown" - return HDF5PluginConfig(build_config, registered_filters) + return HDF5PluginConfig(build_config, filters) def register(filters=tuple(FILTERS.keys()), force=True): @@ -166,7 +166,7 @@ def register(filters=tuple(FILTERS.keys()), force=True): :rtype: bool """ if isinstance(filters, str): - names = (filters,) + filters = (filters,) status = True for filter_name in filters: diff --git a/src/hdf5plugin/test.py b/src/hdf5plugin/test.py index 5ab34f17..e8b14826 100644 --- a/src/hdf5plugin/test.py +++ b/src/hdf5plugin/test.py @@ -283,19 +283,19 @@ def _simple_test(self, filter_name): @unittest.skipUnless(hdf5plugin.config.embedded_filters, "No embedded filters") def test_register_single_filter(self): """Re-register embedded filters one at a time""" - for name in hdf5plugin.config.embedded_filters: - with self.subTest(name=name): - status = hdf5plugin.register(name, force=True) + for filter_name in hdf5plugin.config.embedded_filters: + with self.subTest(name=filter_name): + status = hdf5plugin.register(filter_name, force=True) self.assertTrue(status) - self._simple_test(name) + self._simple_test(filter_name) @unittest.skipUnless(hdf5plugin.config.embedded_filters, "No embedded filters") def test_register_all_filters(self): """Re-register embedded filters all at once""" status = hdf5plugin.register() - for name in hdf5plugin.config.embedded_filters: - with self.subTest(name=name): - self._simple_test(name) + for filter_name in hdf5plugin.config.embedded_filters: + with self.subTest(name=filter_name): + self._simple_test(filter_name) def suite(): From f63ed2e174abdecee538822e15b4b73db3ebb281 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 23 Nov 2022 11:54:36 +0100 Subject: [PATCH 6/8] fix doc --- doc/usage.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/usage.rst b/doc/usage.rst index 482975cc..92ae6b90 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -102,11 +102,13 @@ Get information about hdf5plugin Available constants: -.. autodata:: FILTERS - :annotation: +.. py:data:: FILTERS -.. autodata:: PLUGIN_PATH - :annotation: + Mapping of provided filter's name to their HDF5 filter ID. + +.. py:data:: PLUGIN_PATH + + Directory where the provided HDF5 filter plugins are stored. Functions: From 2fe2bf3ddc28874f721d7ab798dc5df377e1bcf2 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 23 Nov 2022 14:08:49 +0100 Subject: [PATCH 7/8] special case of h5py<2.10 --- src/hdf5plugin/_utils.py | 7 +++++-- src/hdf5plugin/test.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/hdf5plugin/_utils.py b/src/hdf5plugin/_utils.py index 7f3d377c..b7bea1d3 100644 --- a/src/hdf5plugin/_utils.py +++ b/src/hdf5plugin/_utils.py @@ -83,12 +83,15 @@ def register_filter(name): # Unregister existing filter filter_id = FILTERS[name] is_avail = is_filter_available(name) - # TODO h5py>=2.10 + if h5py.version.version_tuple < (2, 10) and is_avail in (True, None): + logger.error( + "h5py.h5z.unregister_filter is not available in this version of h5py.") + return False if is_avail is True: if not h5py.h5z.unregister_filter(filter_id): logger.error("Failed to unregister filter %s (%d)" % (name, filter_id)) return False - elif is_avail is None: # Cannot probe filter availability + if is_avail is None: # Cannot probe filter availability try: h5py.h5z.unregister_filter(filter_id) except RuntimeError: diff --git a/src/hdf5plugin/test.py b/src/hdf5plugin/test.py index e8b14826..df61300e 100644 --- a/src/hdf5plugin/test.py +++ b/src/hdf5plugin/test.py @@ -280,6 +280,7 @@ def _simple_test(self, filter_name): else: self._test(filter_name) + @unittest.skipIf(h5py.version.version_tuple < (2, 10), "h5py<2.10: unregister_filer not available") @unittest.skipUnless(hdf5plugin.config.embedded_filters, "No embedded filters") def test_register_single_filter(self): """Re-register embedded filters one at a time""" @@ -289,6 +290,7 @@ def test_register_single_filter(self): self.assertTrue(status) self._simple_test(filter_name) + @unittest.skipIf(h5py.version.version_tuple < (2, 10), "h5py<2.10: unregister_filer not available") @unittest.skipUnless(hdf5plugin.config.embedded_filters, "No embedded filters") def test_register_all_filters(self): """Re-register embedded filters all at once""" From 55d330375837aa0735a23afa9b9b0bd68478afd6 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 23 Nov 2022 16:55:22 +0100 Subject: [PATCH 8/8] Fix docstring --- src/hdf5plugin/_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hdf5plugin/_utils.py b/src/hdf5plugin/_utils.py index b7bea1d3..02aae610 100644 --- a/src/hdf5plugin/_utils.py +++ b/src/hdf5plugin/_utils.py @@ -158,8 +158,6 @@ def get_config(): def register(filters=tuple(FILTERS.keys()), force=True): """Initialise and register `hdf5plugin` embedded filters given their names. - Unregister corresponding previously registered filters if any. - :param Union[str.Tuple[str]] filters: Filter name or sequence of filter names (See `hdf5plugin.FILTERS`). :param bool force: