Updated Bitshuffle filter to v0.4.2, Bitshuffle API change with deprecation warning

doc/contribute.rst

@@ -89,12 +89,18 @@ The meaning of those integers is filter dependent and is described below.
 bitshuffle
 ..........
 
-compression_opts: (**block_size**, **lz4 compression**)
+compression_opts: (**block_size**, **compression**, **level**)
 
 - **block size**: Number of elements (not bytes) per block.
   It MUST be a mulitple of 8.
   Default: 0 for a block size of about 8 kB.
-- **lz4 compression**: 0: disabled (default), 2: enabled.
+- **compression**:
+
+  * 0: No compression
+  * 2: LZ4
+  * 3: Zstd
+
+- **level**: Compression level, only used with Zstd compression.
 
 By default the filter uses bitshuffle, but does NOT compress with LZ4.
 

doc/information.rst

@@ -50,7 +50,7 @@ HDF5 filters and compression libraries
 HDF5 compression filters and compression libraries sources were obtained from:
 
 * LZ4 plugin (commit d48f960) and lz4 (v1.9.3): https://github.com/nexusformat/HDF5-External-Filter-Plugins and https://github.com/Blosc/c-blosc/tree/v1.21.1/internal-complibs/lz4-1.9.3
-* bitshuffle plugin (0.3.5): https://github.com/kiyo-masui/bitshuffle
+* bitshuffle plugin (0.4.2 + patch `PR #122 <https://github.com/kiyo-masui/bitshuffle/pull/122>`_) and zstd (v1.5.0): https://github.com/kiyo-masui/bitshuffle and https://github.com/Blosc/c-blosc/tree/v1.21.1/internal-complibs/zstd-1.5.0
 * bzip2 plugin (from PyTables v3.7.0) and bzip2 (v1.0.8): https://github.com/PyTables/PyTables/, https://sourceware.org/git/bzip2.git
 * hdf5-blosc plugin (v1.0.0), c-blosc (v1.21.1) and snappy (v1.1.9): https://github.com/Blosc/hdf5-blosc, https://github.com/Blosc/c-blosc and https://github.com/google/snappy
 * FCIDECOMP plugin (v1.0.2) and CharLS (branch 1.x-master SHA1 ID: 25160a42fb62e71e4b0ce081f5cb3f8bb73938b5):

setup.py

@@ -539,37 +539,6 @@ def prefix(directory, files):
 """Mapping plugin name to library name they depend on"""
 
 
-# bitshuffle (+lz4) plugin
-# Plugins from https://github.com/kiyo-masui/bitshuffle
-bithsuffle_dir = 'src/bitshuffle'
-
-# Set compile args for both MSVC and others, list is stripped at build time
-extra_compile_args = ['-O3', '-ffast-math', '-std=c99', '-fopenmp']
-extra_compile_args += ['/Ox', '/fp:fast', '/openmp']
-if platform.machine() == "ppc64le":
-    # Required on ppc64le
-    sse2_options = {'extra_compile_args': ['-DUSESSE2'] }
-else:
-    sse2_options = {}
-extra_link_args = ['-fopenmp', '/openmp']
-
-bithsuffle_plugin = HDF5PluginExtension(
-    "hdf5plugin.plugins.libh5bshuf",
-    sources=prefix(bithsuffle_dir,
-        ["src/bshuf_h5plugin.c", "src/bshuf_h5filter.c",
-         "src/bitshuffle.c", "src/bitshuffle_core.c",
-         "src/iochain.c", "lz4/lz4.c"]),
-    depends=prefix(bithsuffle_dir,
-        ["src/bitshuffle.h", "src/bitshuffle_core.h",
-         "src/iochain.h", 'src/bshuf_h5filter.h',
-         "lz4/lz4.h"]),
-    include_dirs=prefix(bithsuffle_dir, ['src/', 'lz4/']),
-    extra_compile_args=extra_compile_args,
-    extra_link_args=extra_link_args,
-    sse2=sse2_options,
-    )
-
-
 # blosc plugin
 # Plugin from https://github.com/Blosc/hdf5-blosc
 # c-blosc from https://github.com/Blosc/c-blosc
@@ -633,10 +602,14 @@ def prefix(directory, files):
 define_macros.append(('HAVE_ZLIB', 1))
 
 # zstd
-sources += glob(blosc_dir +'internal-complibs/zstd*/*/*.c')
-depends += glob(blosc_dir +'internal-complibs/zstd*/*/*.h')
-include_dirs += glob(blosc_dir + 'internal-complibs/zstd*')
-include_dirs += glob(blosc_dir + 'internal-complibs/zstd*/common')
+zstd_sources = glob(blosc_dir +'internal-complibs/zstd*/*/*.c')
+zstd_depends = glob(blosc_dir +'internal-complibs/zstd*/*/*.h')
+zstd_include_dirs = glob(blosc_dir + 'internal-complibs/zstd*')
+zstd_include_dirs += glob(blosc_dir + 'internal-complibs/zstd*/common')
+
+sources += zstd_sources
+depends += zstd_depends
+include_dirs += zstd_include_dirs
 define_macros.append(('HAVE_ZSTD', 1))
 
 extra_compile_args = ['-std=gnu99']  # Needed to build manylinux1 wheels
@@ -664,19 +637,50 @@ def prefix(directory, files):
 
 # HDF5Plugin-Zstandard
 zstandard_dir = os.path.join("src", "HDF5Plugin-Zstandard")
-zstandard_include_dirs = glob(blosc_dir + 'internal-complibs/zstd*')
-zstandard_include_dirs += glob(blosc_dir + 'internal-complibs/zstd*/common')
 zstandard_sources = [os.path.join(zstandard_dir, 'zstd_h5plugin.c')]
-zstandard_sources += glob(blosc_dir +'internal-complibs/zstd*/*/*.c')
+zstandard_sources += zstd_sources
 zstandard_depends = [os.path.join(zstandard_dir, 'zstd_h5plugin.h')]
-zstandard_depends += glob(blosc_dir +'internal-complibs/zstd*/*/*.h')
+zstandard_depends += zstd_depends
 zstandard_plugin = HDF5PluginExtension(
     "hdf5plugin.plugins.libh5zstd",
     sources=zstandard_sources,
     depends=zstandard_depends,
-    include_dirs=zstandard_include_dirs,
+    include_dirs=zstd_include_dirs,
     )
 
+# bitshuffle (+lz4 or zstd) plugin
+# Plugins from https://github.com/kiyo-masui/bitshuffle
+bithsuffle_dir = 'src/bitshuffle'
+
+# Set compile args for both MSVC and others, list is stripped at build time
+extra_compile_args = ['-O3', '-ffast-math', '-std=c99', '-fopenmp']
+extra_compile_args += ['/Ox', '/fp:fast', '/openmp']
+if platform.machine() == "ppc64le":
+    # Required on ppc64le
+    sse2_options = {'extra_compile_args': ['-DUSESSE2'] }
+else:
+    sse2_options = {}
+extra_link_args = ['-fopenmp', '/openmp']
+define_macros = [("ZSTD_SUPPORT", 1)]
+
+bithsuffle_plugin = HDF5PluginExtension(
+    "hdf5plugin.plugins.libh5bshuf",
+    sources=prefix(bithsuffle_dir,
+        ["src/bshuf_h5plugin.c", "src/bshuf_h5filter.c",
+         "src/bitshuffle.c", "src/bitshuffle_core.c",
+         "src/iochain.c", "lz4/lz4.c"]) + zstd_sources,
+    depends=prefix(bithsuffle_dir,
+        ["src/bitshuffle.h", "src/bitshuffle_core.h",
+         "src/iochain.h", 'src/bshuf_h5filter.h',
+         "lz4/lz4.h"]) + zstd_depends,
+    include_dirs=prefix(bithsuffle_dir, ['src/', 'lz4/']) + zstd_include_dirs,
+    define_macros=define_macros,
+    extra_compile_args=extra_compile_args,
+    extra_link_args=extra_link_args,
+    sse2=sse2_options,
+    )
+
+
 
 # lz4 plugin
 # Source from https://github.com/nexusformat/HDF5-External-Filter-Plugins

src/bitshuffle/.github/workflows/flake8_cython.cfg

@@ -0,0 +1,4 @@
+[flake8]
+filename=*.pyx,*.pxd
+select=E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411
+show_source=True

src/bitshuffle/.github/workflows/flake8_python.cfg

@@ -0,0 +1,3 @@
+[flake8]
+ignore=E501,E203,W503,E266
+show_source=True

src/bitshuffle/.github/workflows/install_hdf5.sh

@@ -0,0 +1,10 @@
+HDF5_VERSION=$1
+
+# Download and install HDF5 $HDF5_VERSION from source for building wheels
+curl https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VERSION%.*}/hdf5-$HDF5_VERSION/src/hdf5-$HDF5_VERSION.tar.gz -O -s
+tar -xzf hdf5-$HDF5_VERSION.tar.gz
+cd hdf5-$HDF5_VERSION
+./configure --prefix=/usr/local
+make -j 2
+make install
+cd ..

src/bitshuffle/.github/workflows/lint.yml

@@ -0,0 +1,32 @@
+name: bitshuffle-ci-build
+on:
+  pull_request:
+    branches:
+    - master
+  push:
+    branches:
+    - master
+
+jobs:
+
+  lint-code:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+
+      - name: Install pip dependencies
+        run: |
+          pip install black flake8
+
+      - name: Run flake8
+        run: |
+          flake8 --config $GITHUB_WORKSPACE/.github/workflows/flake8_python.cfg bitshuffle tests
+          flake8 --config $GITHUB_WORKSPACE/.github/workflows/flake8_cython.cfg bitshuffle tests
+
+      - name: Check code with black
+        run: black --check .

src/bitshuffle/.github/workflows/main.yml

@@ -0,0 +1,58 @@
+name: bitshuffle-ci-build
+on:
+  pull_request:
+    branches:
+    - master
+  push:
+    branches:
+    - master
+
+jobs:
+  run-tests:
+
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7", "3.10"]
+        os: [ubuntu-latest, macos-latest]
+        exclude:
+        - os: macos-latest
+          python-version: "3.6"
+
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install apt dependencies
+      if: ${{ matrix.os == 'ubuntu-latest' }}
+      run: |
+        sudo apt-get install -y libhdf5-serial-dev hdf5-tools pkg-config
+
+    - name: Install homebrew dependencies
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        brew install hdf5 pkg-config
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install h5py
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        pip install h5py
+
+    - name: Install pip dependencies
+      run: |
+        pip install Cython
+        pip install -r requirements.txt
+        pip install pytest
+
+        # Pull in ZSTD repo
+        git submodule update --init
+
+        # Installing the plugin to arbitrary directory to check the install script.
+        python setup.py install --h5plugin --h5plugin-dir ~/hdf5/lib --zstd
+
+    - name: Run tests
+      run: pytest -v .

src/bitshuffle/.github/workflows/wheels.yml

@@ -0,0 +1,98 @@
+name: Build bitshuffle wheels and upload to PyPI
+
+on:
+  workflow_dispatch:
+  release:
+    types:
+    - published
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }} and hdf5-${{ matrix.hdf5 }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        hdf5: ["1.10.7"]
+
+    steps:
+      # Checkout bitshuffle
+      - uses: actions/checkout@v2
+
+      # Build wheels for linux and x86 platforms
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.3.1
+        with:
+          output-dir: ./wheelhouse-hdf5-${{ matrix.hdf5}}
+        env:
+          CIBW_SKIP: "pp* *musllinux*"
+          CIBW_ARCHS_LINUX: "x86_64"
+          CIBW_BEFORE_ALL: |
+            chmod +x .github/workflows/install_hdf5.sh
+            .github/workflows/install_hdf5.sh ${{ matrix.hdf5 }}
+            git submodule update --init
+          CIBW_ENVIRONMENT: |
+            LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib ENABLE_ZSTD=1
+          CIBW_TEST_REQUIRES: pytest
+          # Install different version of HDF5 for unit tests to ensure the
+          # wheels are independent of HDF5 installation
+          # CIBW_BEFORE_TEST: |
+          #   chmod +x .github/workflows/install_hdf5.sh
+          #   .github/workflows/install_hdf5.sh 1.8.11
+          # Run units tests but disable test_h5plugin.py
+          CIBW_TEST_COMMAND: pytest {package}/tests
+
+      # Package wheels and host on CI
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse-hdf5-${{ matrix.hdf5 }}/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    strategy:
+      matrix:
+        python-version: ["3.8"]
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install apt dependencies
+        run: |
+          sudo apt-get install -y libhdf5-serial-dev hdf5-tools pkg-config
+
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install pip dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Build sdist
+        run: python setup.py sdist
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: dist/*.tar.gz
+
+  # Upload to PyPI
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    # Upload to PyPI on every tag
+    # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    # Alternatively, to publish when a GitHub Release is created, use the following rule:
+    if: github.event_name == 'release' && github.event.action == 'published'
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifact
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@v1.4.2
+        with:
+          user: __token__
+          password: ${{ secrets.pypi_password }}
+          # To test: repository_url: https://test.pypi.org/legacy/

src/bitshuffle/.gitignore

@@ -75,3 +75,5 @@ doc/generated
 bitshuffle/ext.c
 bitshuffle/h5.c
 
+# ItelliJ
+.idea

src/bitshuffle/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "zstd"]
+	path = zstd
+	url = https://github.com/facebook/zstd