Merge pull request #54 from t20100/snappy-c++11

LGTM

CHANGELOG.rst

@@ -1,3 +1,13 @@
+Unreleased
+----------
+
+- Added `--openmp=[False|True]` build option to compile bitshuffle filter with OpenMP. (PR #51)
+- Added `--sse2=[True|False]` build option to compile blosc and bitshuffle filters with SSE2 instructions if available. (PR #52)
+- Added `--avx2=[True|False]` build option to compile blosc and bitshuffle filters with AVX2 instructions if available. (PR #52)
+- Added `--native=[True|False]` build option to compile filters for native CPU architecture. This enables SSE2/AVX2 support for the bitshuffle filter if available. (PR #52)
+- Added snappy compression to the blosc filter if C++11 is available (`--cpp11=[True|False]` build option). (PR #54)
+- Improved wheel generation by using root_is_pure=True setting. (PR #49)
+
 2.0.0
 -----
 

README.rst

@@ -48,7 +48,7 @@ Sample code:
 
   # Compression
   f = h5py.File('test.h5', 'w')
-  f.create_dataset('data', data=numpy.arange(100), compression=hdf5plugin.LZ4_ID)
+  f.create_dataset('data', data=numpy.arange(100), **hdf5plugin.LZ4())
   f.close()
 
   # Decompression
@@ -58,18 +58,18 @@ Sample code:
 
 ``hdf5plugin`` provides:
 
-* The HDF5 filter ID of embedded plugins:
-
-  - ``BLOSC_ID``
-  - ``BSHUF_ID``
-  - ``LZ4_ID``
-
 * Compression option helper classes to prepare arguments to provide to ``h5py.Group.create_dataset``:
 
   - `Bitshuffle(nelems=0, lz4=True)`_
   - `Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE)`_
   - `LZ4(nbytes=0)`_
 
+* The HDF5 filter ID of embedded plugins:
+
+  - ``BLOSC_ID``
+  - ``BSHUF_ID``
+  - ``LZ4_ID``
+
 * ``FILTERS``: A dictionary mapping provided filters to their ID
 * ``PLUGINS_PATH``: The directory where the provided filters library are stored.
 
@@ -104,6 +104,7 @@ This class takes the following arguments and returns the compression options to
   * 'blosclz'
   * 'lz4' (default)
   * 'lz4hc'
+  * 'snappy' (optional, requires C++11)
   * 'zlib'
   * 'zstd'
 

hdf5plugin/__init__.py

@@ -111,6 +111,7 @@ class Blosc(_FilterRefClass):
 
     :param str cname:
         `blosclz`, `lz4` (default), `lz4hc`, `zlib`, `zstd`
+        Optional: `snappy`, depending on compilation (requires C++11).
     :param int clevel:
         Compression level from 0 no compression to 9 maximum compression.
         Default: 5.
@@ -135,7 +136,7 @@ class Blosc(_FilterRefClass):
         'blosclz': 0,
         'lz4': 1,
         'lz4hc': 2,
-        # Not built 'snappy': 3,
+        'snappy': 3,
         'zlib': 4,
         'zstd': 5,
     }

hdf5plugin/test.py

@@ -104,8 +104,6 @@ def testBlosc(self):
         for clevel in range(10):
             for shuffle in shuffles:
                 for compression_id, cname in enumerate(compress):
-                    if cname == 'snappy':
-                        continue  # Not provided
                     filter_ = self._test(
                         'blosc', cname=cname, clevel=clevel, shuffle=shuffle)
                     self.assertEqual(

setup.py

@@ -97,10 +97,13 @@ class Build(build):
         ('sse2=', None, "Whether or not to compile with SSE2 support if available."
          "Default: True"),
         ('avx2=', None, "Whether or not to compile with AVX2 support if available."
-         "Default: True")]
+         "Default: True"),
+        ('cpp11=', None, "Whether or not to compile C++11 code if available."
+         "Default: True"),
+        ]
     user_options.extend(build.user_options)
 
-    boolean_options = build.boolean_options + ['openmp', 'native', 'sse2', 'avx2']
+    boolean_options = build.boolean_options + ['openmp', 'native', 'sse2', 'avx2', 'cpp11']
 
     def initialize_options(self):
         build.initialize_options(self)
@@ -110,6 +113,7 @@ def initialize_options(self):
         self.native = True
         self.sse2 = True
         self.avx2 = True
+        self.cpp11 = True
 
 
 class PluginBuildExt(build_ext):
@@ -145,30 +149,39 @@ def build_extensions(self):
 
         # Check availability of compile flags
 
+        if build_cmd.cpp11:
+            if compiler_type == 'msvc':
+                with_cpp11 = sys.version_info[:2] >= (3, 5)
+            else:
+                with_cpp11 = self.__check_compile_flag('-std=c++11', extension='.cpp')
+        else:
+            with_cpp11 = False
+
         if build_cmd.sse2:
             if compiler_type == 'msvc':
                 with_sse2 = sys.version_info[0] >= 3
             else:
-                with_sse2 = self.__check_compile_args('-msse2')
+                with_sse2 = self.__check_compile_flag('-msse2')
         else:
             with_sse2 = False
 
         if build_cmd.avx2:
             if compiler_type == 'msvc':
                 with_avx2 = sys.version_info[:2] >= (3, 5)
             else:
-                with_avx2 = self.__check_compile_args('-mavx2')
+                with_avx2 = self.__check_compile_flag('-mavx2')
         else:
             with_avx2 = False
 
-        with_openmp = bool(build_cmd.openmp) and self.__check_compile_args(
+        with_openmp = bool(build_cmd.openmp) and self.__check_compile_flag(
             '/openmp' if compiler_type == 'msvc' else '-fopenmp')
 
         if build_cmd.native:
             is_cpu_sse2, is_cpu_avx2 = get_cpu_sse2_avx2()
             with_sse2 = with_sse2 and is_cpu_sse2
             with_avx2 = with_avx2 and is_cpu_avx2
 
+        logger.info("Building with C++11: %r", with_cpp11)
         logger.info('Building with native option: %r', bool(build_cmd.native))
         logger.info("Building extensions with SSE2: %r", with_sse2)
         logger.info("Building extensions with AVX2: %r", with_avx2)
@@ -180,6 +193,11 @@ def build_extensions(self):
             if isinstance(e, HDF5PluginExtension):
                 e.set_hdf5_dir(build_cmd.hdf5)
 
+                if with_cpp11:
+                    for name, value in e.cpp11.items():
+                        attribute = getattr(e, name)
+                        attribute += value
+
                 # Enable SSE2/AVX2 if available and add corresponding resources
                 if with_sse2:
                     e.extra_compile_args += ['-msse2'] # /arch:SSE2 is on by default
@@ -210,10 +228,11 @@ def build_extensions(self):
 
         build_ext.build_extensions(self)
 
-    def __check_compile_args(self, *args):
+    def __check_compile_flag(self, flag, extension='.c'):
         """Try to compile an empty file to check for compiler args
 
-        :param List[str] args: List of arguments to pass to compiler
+        :param str flag: Flag argument to pass to compiler
+        :param str extension: Source file extension (default: '.c')
         :returns: Whether or not compilation was successful
         :rtype: bool
         """
@@ -222,12 +241,12 @@ def __check_compile_args(self, *args):
 
         with tempfile.TemporaryDirectory() as tmp_dir:
             # Create empty source file
-            tmp_file = os.path.join(tmp_dir, 'source.c')
+            tmp_file = os.path.join(tmp_dir, 'source' + extension)
             with open(tmp_file, 'w') as f:
-                f.write('/*empty source file*/\n')
+                f.write('int main (int argc, char **argv) { return 0; }\n')
 
             try:
-                self.compiler.compile([tmp_file], output_dir=tmp_dir, extra_postargs=list(args))
+                self.compiler.compile([tmp_file], output_dir=tmp_dir, extra_postargs=[flag])
             except CompileError:
                 return False
             else:
@@ -237,7 +256,7 @@ def __check_compile_args(self, *args):
 class HDF5PluginExtension(Extension):
     """Extension adding specific things to build a HDF5 plugin"""
 
-    def __init__(self, name, sse2=None, avx2=None, **kwargs):
+    def __init__(self, name, sse2=None, avx2=None, cpp11=None, **kwargs):
         Extension.__init__(self, name, **kwargs)
 
         if sys.platform.startswith('win'):
@@ -254,6 +273,7 @@ def __init__(self, name, sse2=None, avx2=None, **kwargs):
 
         self.sse2 = sse2 if sse2 is not None else {}
         self.avx2 = avx2 if avx2 is not None else {}
+        self.cpp11 = cpp11 if cpp11 is not None else {}
 
     def set_hdf5_dir(self, hdf5_dir=None):
         """Set the HDF5 installation directory to use to build the plugins.
@@ -315,7 +335,6 @@ def prefix(directory, files):
 # blosc plugin
 # Plugin from https://github.com/Blosc/hdf5-blosc
 # c-blosc from https://github.com/Blosc/c-blosc
-# TODO snappy
 hdf5_blosc_dir = 'src/hdf5-blosc/src/'
 blosc_dir = 'src/c-blosc/'
 
@@ -326,11 +345,15 @@ def prefix(directory, files):
 include_dirs = [blosc_dir, blosc_dir + 'blosc']
 define_macros = []
 
-sse2_sources = [f for f in glob(blosc_dir + 'blosc/*.c') if 'sse2' in f]
-sse2_define_macros = [('SHUFFLE_SSE2_ENABLED', 1)]
+sse2_kwargs = {
+    'sources': [f for f in glob(blosc_dir + 'blosc/*.c') if 'sse2' in f],
+    'define_macros': [('SHUFFLE_SSE2_ENABLED', 1)],
+    }
 
-avx2_sources = [f for f in glob(blosc_dir + 'blosc/*.c') if 'avx2' in f]
-avx2_define_macros = [('SHUFFLE_AVX2_ENABLED', 1)]
+avx2_kwargs = {
+    'sources': [f for f in glob(blosc_dir + 'blosc/*.c') if 'avx2' in f],
+    'define_macros': [('SHUFFLE_AVX2_ENABLED', 1)],
+    }
 
 # compression libs
 # lz4
@@ -344,7 +367,12 @@ def prefix(directory, files):
 define_macros.append(('HAVE_LZ4', 1))
 
 # snappy
-# TODO
+cpp11_kwargs = {
+    'sources': glob(blosc_dir + 'internal-complibs/snappy*/*.cc'),
+    'include_dirs': glob(blosc_dir + 'internal-complibs/snappy*'),
+    'extra_compile_args': ['-std=c++11', '-lstdc++'],
+    'define_macros': [('HAVE_SNAPPY', 1)],
+    }
 
 #zlib
 sources += glob(blosc_dir + 'internal-complibs/zlib*/*.c')
@@ -368,8 +396,9 @@ def prefix(directory, files):
         prefix(hdf5_blosc_dir, ['blosc_filter.h', 'blosc_plugin.h']),
     include_dirs=include_dirs + [hdf5_blosc_dir],
     define_macros=define_macros,
-    sse2={'sources': sse2_sources, 'define_macros': sse2_define_macros},
-    avx2={'sources': avx2_sources, 'define_macros': avx2_define_macros},
+    sse2=sse2_kwargs,
+    avx2=avx2_kwargs,
+    cpp11=cpp11_kwargs,
     )