diff --git a/doc/usage.rst b/doc/usage.rst index c39a85de..92ae6b90 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -102,16 +102,31 @@ Get information about hdf5plugin Available constants: -.. autodata:: FILTERS - :annotation: +.. py:data:: FILTERS -.. autodata:: PLUGIN_PATH - :annotation: + Mapping of provided filter's name to their HDF5 filter ID. + +.. py:data:: PLUGIN_PATH + + Directory where the provided HDF5 filter plugins are stored. Functions: .. autofunction:: get_config +Manage registered filters ++++++++++++++++++++++++++ + +When imported, `hdf5plugin` initialises and registers the filters it embeds if there is no already registered filters for the corresponding filter IDs. + +`h5py`_ gives access to HDF5 functions handling registered filters in `h5py.h5z`_. +This module allows checking the filter availability and registering/unregistering filters. + +`hdf5plugin` provides an extra `register` function to register the filters it provides, e.g., to override an already loaded filters. +Registering with this function is required to perform additional initialisation and enable writing compressed data with the given filter. + +.. autofunction:: register + Use HDF5 filters in other applications ++++++++++++++++++++++++++++++++++++++ @@ -128,4 +143,5 @@ should allow MatLab or IDL users to read data compressed using the supported plu Setting the ``HDF5_PLUGIN_PATH`` environment variable allows already existing programs or Python code to read compressed data without any modification. .. _h5py: https://www.h5py.org +.. _h5py.h5z: https://github.com/h5py/h5py/blob/master/h5py/h5z.pyx .. _h5py.Group.create_dataset: https://docs.h5py.org/en/stable/high/group.html#h5py.Group.create_dataset diff --git a/src/hdf5plugin/__init__.py b/src/hdf5plugin/__init__.py index d7328562..654b39e3 100644 --- a/src/hdf5plugin/__init__.py +++ b/src/hdf5plugin/__init__.py @@ -38,7 +38,7 @@ from ._filters import ZSTD_ID, Zstd # noqa from ._filters import SZ_ID, SZ # noqa -from ._utils import get_config, PLUGIN_PATH # noqa +from ._utils import get_config, PLUGIN_PATH, register # noqa # Backward compatibility from ._config import build_config as config # noqa diff --git a/src/hdf5plugin/_utils.py b/src/hdf5plugin/_utils.py index 1e7abdeb..02aae610 100644 --- a/src/hdf5plugin/_utils.py +++ b/src/hdf5plugin/_utils.py @@ -44,71 +44,137 @@ """Directory where the provided HDF5 filter plugins are stored.""" -def _init_filters(): - """Initialise and register HDF5 filters with h5py +def is_filter_available(name): + """Returns whether filter is already registered or not. - Generator of tuples: (filename, library handle) + :param str name: Name of the filter (See `hdf5plugin.FILTERS`) + :return: True if filter is registered, False if not and + None if it cannot be checked (libhdf5 not supporting it) + :rtype: Union[bool,None] """ + filter_id = FILTERS[name] + hdf5_version = h5py.h5.get_libversion() + if hdf5_version < (1, 8, 20) or (1, 10) <= hdf5_version < (1, 10, 2): + return None # h5z.filter_avail not available + return h5py.h5z.filter_avail(filter_id) > 0 - for name, filter_id in FILTERS.items(): - # Skip filters that were not embedded - if name not in build_config.embedded_filters: - logger.debug("%s filter not available in this build of hdf5plugin.", name) - continue - # Check if filter is already loaded (not on buggy HDF5 versions) - if (1, 8, 20) <= hdf5_version < (1, 10) or hdf5_version >= (1, 10, 2): - if h5py.h5z.filter_avail(filter_id): - logger.info("%s filter already loaded, skip it.", name) - yield name, ("unknown", None) - continue - - # Load DLL - filename = glob.glob(os.path.join( - PLUGIN_PATH, 'libh5' + name + '*' + build_config.filter_file_extension)) - if len(filename): - filename = filename[0] - else: - logger.error("Cannot initialize filter %s: File not found", name) - continue - try: - lib = ctypes.CDLL(filename) - except OSError: - logger.error("Failed to load filter %s: %s", name, filename) - logger.error(traceback.format_exc()) - continue +registered_filters = {} +"""Store hdf5plugin registered filters as a mapping: name: (filename, ctypes.CDLL)""" - if sys.platform.startswith('win'): - # Use register_filter function to register filter - lib.register_filter.restype = ctypes.c_int - retval = lib.register_filter() - else: - # Use init_filter function to initialize DLL and register filter - lib.init_filter.argtypes = [ctypes.c_char_p] - lib.init_filter.restype = ctypes.c_int - retval = lib.init_filter( - bytes(h5py.h5z.__file__, encoding='utf-8')) - - if retval < 0: - logger.error("Cannot initialize filter %s: %d", name, retval) - continue - logger.debug("Registered filter: %s (%s)", name, filename) - yield name, (filename, lib) +def register_filter(name): + """Register a filter given its name + Unregister the previously registered filter if any. -_filters = dict(_init_filters()) # Store loaded filters + :param str name: Name of the filter (See `hdf5plugin.FILTERS`) + :return: True if successfully registered, False otherwise + :rtype: bool + """ + if name not in FILTERS: + raise ValueError("Unknown filter name: %s" % name) + + if name not in build_config.embedded_filters: + logger.debug("%s filter not available in this build of hdf5plugin.", name) + return False + + # Unregister existing filter + filter_id = FILTERS[name] + is_avail = is_filter_available(name) + if h5py.version.version_tuple < (2, 10) and is_avail in (True, None): + logger.error( + "h5py.h5z.unregister_filter is not available in this version of h5py.") + return False + if is_avail is True: + if not h5py.h5z.unregister_filter(filter_id): + logger.error("Failed to unregister filter %s (%d)" % (name, filter_id)) + return False + if is_avail is None: # Cannot probe filter availability + try: + h5py.h5z.unregister_filter(filter_id) + except RuntimeError: + logger.debug("Filter %s (%d) not unregistered" % (name, filter_id)) + logger.debug(traceback.format_exc()) + registered_filters.pop(name, None) + + # Load DLL + filename = glob.glob(os.path.join( + PLUGIN_PATH, 'libh5' + name + '*' + build_config.filter_file_extension)) + if len(filename): + filename = filename[0] + else: + logger.error("Cannot initialize filter %s: File not found", name) + return False + try: + lib = ctypes.CDLL(filename) + except OSError: + logger.error("Failed to load filter %s: %s", name, filename) + logger.error(traceback.format_exc()) + return False + + if sys.platform.startswith('win'): + # Use register_filter function to register filter + lib.register_filter.restype = ctypes.c_int + retval = lib.register_filter() + else: + # Use init_filter function to initialize DLL and register filter + lib.init_filter.argtypes = [ctypes.c_char_p] + lib.init_filter.restype = ctypes.c_int + retval = lib.init_filter( + bytes(h5py.h5z.__file__, encoding='utf-8')) + + if retval < 0: + logger.error("Cannot initialize filter %s: %d", name, retval) + return False + + logger.debug("Registered filter: %s (%s)", name, filename) + registered_filters[name] = filename, lib + return True + + +HDF5PluginConfig = namedtuple( + 'HDF5PluginConfig', + ('build_config', 'registered_filters'), +) def get_config(): """Provides information about build configuration and filters registered by hdf5plugin. """ - HDF5PluginConfig = namedtuple( - 'HDF5PluginConfig', - ('build_config', 'registered_filters'), - ) - return HDF5PluginConfig( - build_config=build_config, - registered_filters=dict((name, filename) for name, (filename, lib) in _filters.items()), - ) + filters = {} + for name in FILTERS: + info = registered_filters.get(name) + if info is not None: # Registered by hdf5plugin + if is_filter_available(name) in (True, None): + filters[name] = info[0] + elif is_filter_available(name) is True: # Registered elsewhere + filters[name] = "unknown" + + return HDF5PluginConfig(build_config, filters) + + +def register(filters=tuple(FILTERS.keys()), force=True): + """Initialise and register `hdf5plugin` embedded filters given their names. + + :param Union[str.Tuple[str]] filters: + Filter name or sequence of filter names (See `hdf5plugin.FILTERS`). + :param bool force: + True to register the filter even if a corresponding one if already available. + False to skip already available filters. + :return: True if all filters were registered successfully, False otherwise. + :rtype: bool + """ + if isinstance(filters, str): + filters = (filters,) + + status = True + for filter_name in filters: + if not force and is_filter_available(filter_name) is True: + logger.info("%s filter already loaded, skip it.", filter_name) + continue + status = status and register_filter(filter_name) + return status + +register(force=False) diff --git a/src/hdf5plugin/test.py b/src/hdf5plugin/test.py index 667ab905..df61300e 100644 --- a/src/hdf5plugin/test.py +++ b/src/hdf5plugin/test.py @@ -38,8 +38,8 @@ def should_test(filter_name): return filter_name in hdf5plugin.config.embedded_filters or h5py.h5z.filter_avail(filter_id) -class TestHDF5PluginRW(unittest.TestCase): - """Test write/read a HDF5 file with the plugins""" +class BaseTestHDF5PluginRW(unittest.TestCase): + """Base class for testing write/read HDF5 dataset with the plugins""" @classmethod def setUpClass(cls): @@ -107,6 +107,10 @@ def _test(self, os.remove(filename) return filters[0] + +class TestHDF5PluginRW(BaseTestHDF5PluginRW): + """Test write/read a HDF5 file with the plugins""" + @unittest.skipUnless(should_test("bshuf"), "Bitshuffle filter not available") def testDepreactedBitshuffle(self): """Write/read test with bitshuffle filter plugin""" @@ -265,9 +269,40 @@ def testVersion(self): self.assertIsInstance(version_info.serial, int) +class TestRegisterFilter(BaseTestHDF5PluginRW): + """Test usage of the register function""" + + def _simple_test(self, filter_name): + if filter_name == 'fcidecomp': + self._test('fcidecomp', dtype=numpy.uint8) + elif filter_name in ('sz', 'zfp'): + self._test(filter_name, dtype=numpy.float32, lossless=False) + else: + self._test(filter_name) + + @unittest.skipIf(h5py.version.version_tuple < (2, 10), "h5py<2.10: unregister_filer not available") + @unittest.skipUnless(hdf5plugin.config.embedded_filters, "No embedded filters") + def test_register_single_filter(self): + """Re-register embedded filters one at a time""" + for filter_name in hdf5plugin.config.embedded_filters: + with self.subTest(name=filter_name): + status = hdf5plugin.register(filter_name, force=True) + self.assertTrue(status) + self._simple_test(filter_name) + + @unittest.skipIf(h5py.version.version_tuple < (2, 10), "h5py<2.10: unregister_filer not available") + @unittest.skipUnless(hdf5plugin.config.embedded_filters, "No embedded filters") + def test_register_all_filters(self): + """Re-register embedded filters all at once""" + status = hdf5plugin.register() + for filter_name in hdf5plugin.config.embedded_filters: + with self.subTest(name=filter_name): + self._simple_test(filter_name) + + def suite(): test_suite = unittest.TestSuite() - for cls in (TestHDF5PluginRW, TestPackage): + for cls in (TestHDF5PluginRW, TestPackage, TestRegisterFilter): test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(cls)) return test_suite