Merge pull request #18 from t20100/dlopen

Rework hdf5plugin to add compression and blosc filter support

.travis.yml

@@ -7,19 +7,27 @@ matrix:
     include:
         - python: 2.7
           os: linux
-          env: BUILD_COMMAND=sdist
+          env:
+              BUILD_COMMAND=bdist_wheel
+              H5PY_OLDER_VERSION=2.6.0
 
-        - python: 3.4
+        - python: 3.6
           os: linux
-          env: BUILD_COMMAND=sdist
+          env:
+              BUILD_COMMAND=sdist
+              H5PY_OLDER_VERSION=2.6.0
 
-        - python: 3.5
+        - python: 3.7
           os: linux
-          env: BUILD_COMMAND=sdist
+          env:
+              BUILD_COMMAND=bdist_wheel
+              H5PY_OLDER_VERSION=2.8.0
 
         - language: generic
           os: osx
-          env: BUILD_COMMAND=bdist_wheel
+          env:
+              BUILD_COMMAND=bdist_wheel
+              H5PY_OLDER_VERSION=2.6.0
 
 cache:
     apt: true
@@ -35,7 +43,7 @@ install:
 
     # Install build dependencies
     - pip install --upgrade wheel
-    #- pip install --trusted-host www.silx.org --find-links http://www.silx.org/pub/wheelhouse/ h5py --no-index
+    - pip install h5py
 
     # Print Python info
     - python ./ci/info_platform.py
@@ -57,7 +65,7 @@ script:
     - python -m pip install --upgrade pip
 
     # Install h5py for tests
-    - pip install --pre --find-links dist/ --trusted-host www.silx.org --find-links http://www.silx.org/pub/wheelhouse/ h5py
+    - pip install --pre h5py
 
     # Install built package
     # Make sure it does not come from cache or pypi
@@ -69,7 +77,12 @@ script:
     - python ci/info_platform.py
     - pip list
 
-    # Run the tests
+    # Run the tests with latest h5py
+    - python test/test.py
+
+    # Run the tests with oldest available wheel of h5py
+    - pip install --upgrade h5py==$H5PY_OLDER_VERSION
+    - pip list
     - python test/test.py
 
     # Mac OS X specific cleanup

LICENSE

@@ -0,0 +1,31 @@
+The hdf5plugin code itself is licensed under the MIT license (See below).
+
+The HDF5 filter plugins bundled in hdf5plugin have different copyright and licenses:
+
+* bitshuffle: See src/bitshuffle/LICENSE
+* blosc: See src/hdf5-blosc/LICENSES/ and src/c-blosc/LICENSES/
+* lz4: See src/HDF5-External-Filter-Plugins/LZ4/COPYING
+
+
+hdf5plugin copyright and license:
+
+Copyright (c) European Synchrotron Radiation Facility (ESRF)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+

MANIFEST.in

@@ -1,9 +1,10 @@
-include README.rst
-include MANIFEST.in
-include setup.py
-include setup.cfg
-include .travis.yml
-include appveyor.yml
-recursive-include ci *.py *.sh
-recursive-include hdf5plugin *.py *.dll *.so *.dylib
-recursive-include test *.py *.h5
+include README.rst
+include MANIFEST.in
+include setup.py
+include setup.cfg
+include .travis.yml
+include appveyor.yml
+recursive-include ci *.py *.sh
+recursive-include hdf5plugin *.py
+recursive-include test *.py *.h5
+recursive-include src *

README.rst

@@ -1,19 +1,15 @@
 hdf5plugin
 ==========
 
-This module provides HDF5 plugins and sets the HDF5_PLUGIN_PATH to the appropriate value according to the version of python used.
+This module provides HDF5 compression filters (namely: blosc, bitshuffle and lz4) and registers them to the HDF5 library used by `h5py <https://www.h5py.org>`_.
 
-The module is mainly intended for use under windows but shared libraries for MacOS and manylinux are also provided.
+Supported platforms are: Linux, Windows, macOS.
 
-The LZ4 plugin sources were obtained from https://github.com/nexusformat/HDF5-External-Filter-Plugins
-
-The bitshuffle plugin sources were obtained from https://github.com/kiyo-masui/bitshuffle
-
-Known issues
-------------
-
-Under MacOS and linux, the bitshuffle plugin is limited to decompression (i.e. it can be used to read bitshuffle compressed data).
+The HDF5 plugin sources were obtained from:
 
+* LZ4 plugin (v0.1.0): https://github.com/nexusformat/HDF5-External-Filter-Plugins
+* bitshuffle plugin (0.3.5): https://github.com/kiyo-masui/bitshuffle
+* hdf5-blosc plugin (v1.0.0) and c-blosc (v1.17.0): https://github.com/Blosc/hdf5-blosc and https://github.com/Blosc/c-blosc
 
 Installation
 ------------
@@ -26,22 +22,58 @@ To install locally, run::
 
      pip install hdf5plugin --user
 
-Dependencies
-------------
-
-No additional dependencies.
-
 Documentation
 -------------
 
-To use it, just ''import hdf5plugin'' **before** importing h5py or PyTables.
+To use it, just use ``import hdf5plugin`` and supported compression plugins are available from `h5py <https://www.h5py.org>`_.
+
+Sample code:
+
+.. code-block:: python
+
+  import numpy
+  import h5py
+  import hdf5plugin
+
+  # Compression
+  f = h5py.File('test.h5', 'w')
+  f.create_dataset('data', data=numpy.arange(100), compression=hdf5plugin.LZ4)
+  f.close()
+
+  # Decompression
+  f = h5py.File('test.h5', 'r')
+  data = f['data'][()]
+  f.close()
+
+``hdf5plugin`` provides:
+
+* The HDF5 filter ID of embedded plugins:
+
+  - ``hdf5plugin.BLOSC``
+  - ``hdf5plugin.BSHUF``
+  - ``hdf5plugin.LZ4``
+
+* ``hdf5plugin.FILTERS``: A dictionary mapping provided filters to their ID
+* ``hdf5plugin.PLUGINS_PATH``: The directory where the provided filters library are stored.
+
+
+Dependencies
+------------
+
+* `h5py <https://www.h5py.org>`_
 
 License
 -------
 
-This code is licensed under the MIT license. Use it at your own risk.
+The source code of *hdf5plugin* itself is licensed under the MIT license.
+Use it at your own risk.
+See `LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/LICENSE>`_
 
-The source code of the libraries is licensed under MIT (bitshuffle) and BSD like licenses (LZ4).
+The source code of the embedded HDF5 filter plugin libraries is licensed under different open-source licenses.
+Please read the different licenses:
 
-Please read the LICENSE_H5bshuf and LICENSE_H5Zlz4 respectively for details.
+* hdf5plugin itself: * bitshuffle: See `src/bitshuffle/LICENSE <https://github.com/silx-kit/hdf5plugin/blob/master/src/bitshuffle/LICENSE>`_
+* blosc: See `src/hdf5-blosc/LICENSES/ <https://github.com/silx-kit/hdf5plugin/blob/master/src/hdf5-blosc/LICENSES/>`_ and `src/c-blosc/LICENSES/ <https://github.com/silx-kit/hdf5plugin/blob/master/src/c-blosc/LICENSES/>`_
+* lz4: See `src/LZ4/COPYING  <https://github.com/silx-kit/hdf5plugin/blob/master/src/LZ4/COPYING>`_
 
+The HDF5 v1.10.5 headers (and Windows .lib file) used to build the filters are stored for convenience in the repository. The license is available here: `src/hdf5/COPYING <https://github.com/silx-kit/hdf5plugin/blob/master/src/hdf5/COPYING>`_.

appveyor.yml

@@ -14,11 +14,13 @@ environment:
         VENV_TEST_DIR: "venv_test"
 
     matrix:
-        # Python 3.5
-        - PYTHON_DIR: "C:\\Python35-x64"
+        # Python 3.7
+        - PYTHON_DIR: "C:\\Python37-x64"
+          H5PY_OLDER_VERSION: "2.8.0"
 
         # Python 2.7
         - PYTHON_DIR: "C:\\Python27-x64"
+          H5PY_OLDER_VERSION: "2.7.0"
 
 install:
     # Add Python to PATH
@@ -37,9 +39,9 @@ build_script:
     - "virtualenv --clear %VENV_BUILD_DIR%"
     - "%VENV_BUILD_DIR%\\Scripts\\activate.bat"
 
-    # Install build dependencies (should I try to install cython?)
-    - "pip install --upgrade wheel"
-    - "pip install --trusted-host www.silx.org --find-links http://www.silx.org/pub/wheelhouse/ --upgrade h5py"
+    # Install build dependencies
+    - "pip install --upgrade setuptools wheel"
+    - "pip install h5py"
 
     # Print Python info
     - "python ci\\info_platform.py"
@@ -59,10 +61,10 @@ test_script:
     - "%VENV_TEST_DIR%\\Scripts\\activate.bat"
 
     # Install h5py for tests
-    - "pip install --trusted-host www.silx.org --find-links http://www.silx.org/pub/wheelhouse/ --upgrade h5py"
+    - "pip install --pre h5py"
 
     # Install the generated wheel package to test it
-    # Make sure silx does not come from cache or pypi
+    # Make sure it does not come from cache or pypi
     # At this point all install_requires dependencies MUST be installed
     # as this is installing only from dist/
     - "pip install --pre --find-links dist/ --no-cache-dir --no-index hdf5plugin"
@@ -73,6 +75,11 @@ test_script:
 
     - "python test\\test.py"
 
+    # Run the tests with oldest available wheel of h5py
+    - "pip install --upgrade h5py==%H5PY_OLDER_VERSION%"
+    - "pip list"
+    - "python test\\test.py"
+
     # Leave test virtualenv
     - "%VENV_TEST_DIR%\\Scripts\\deactivate.bat"
     - "rmdir %VENV_TEST_DIR% /s /q"

ci/travis_osx.sh

@@ -34,6 +34,7 @@ else
     function travis_osx_install_begin {
         echo Mac OS X install begin: Install pip and setup build venv
         set -x  # echo on
+        export PATH=${PATH}:/Users/travis/Library/Python/2.7/bin
         curl -O https://bootstrap.pypa.io/get-pip.py
         python get-pip.py --user