From 2a3c520168af5c09704b0db43ec1dd6a91f57d7b Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 20 Nov 2019 16:06:22 +0100 Subject: [PATCH 1/6] Add build of snappy if c++11 is available + cpp11 build option --- setup.py | 67 ++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index e8c6769c..1a16fb96 100644 --- a/setup.py +++ b/setup.py @@ -97,10 +97,13 @@ class Build(build): ('sse2=', None, "Whether or not to compile with SSE2 support if available." "Default: True"), ('avx2=', None, "Whether or not to compile with AVX2 support if available." - "Default: True")] + "Default: True"), + ('cpp11=', None, "Whether or not to compile C++11 code if available." + "Default: True"), + ] user_options.extend(build.user_options) - boolean_options = build.boolean_options + ['openmp', 'native', 'sse2', 'avx2'] + boolean_options = build.boolean_options + ['openmp', 'native', 'sse2', 'avx2', 'cpp11'] def initialize_options(self): build.initialize_options(self) @@ -110,6 +113,7 @@ def initialize_options(self): self.native = True self.sse2 = True self.avx2 = True + self.cpp11 = True class PluginBuildExt(build_ext): @@ -145,11 +149,19 @@ def build_extensions(self): # Check availability of compile flags + if build_cmd.cpp11: + if compiler_type == 'msvc': + with_cpp11 = sys.version_info[:2] >= (3, 5) + else: + with_cpp11 = self.__check_compile_flag('-std=c++11', extension='.cpp') + else: + with_cpp11 = False + if build_cmd.sse2: if compiler_type == 'msvc': with_sse2 = sys.version_info[0] >= 3 else: - with_sse2 = self.__check_compile_args('-msse2') + with_sse2 = self.__check_compile_flag('-msse2') else: with_sse2 = False @@ -157,11 +169,11 @@ def build_extensions(self): if compiler_type == 'msvc': with_avx2 = sys.version_info[:2] >= (3, 5) else: - with_avx2 = self.__check_compile_args('-mavx2') + with_avx2 = self.__check_compile_flag('-mavx2') else: with_avx2 = False - with_openmp = bool(build_cmd.openmp) and self.__check_compile_args( + with_openmp = bool(build_cmd.openmp) and self.__check_compile_flag( '/openmp' if compiler_type == 'msvc' else '-fopenmp') if build_cmd.native: @@ -169,6 +181,7 @@ def build_extensions(self): with_sse2 = with_sse2 and is_cpu_sse2 with_avx2 = with_avx2 and is_cpu_avx2 + logger.info("Building with C++11: %r", with_cpp11) logger.info('Building with native option: %r', bool(build_cmd.native)) logger.info("Building extensions with SSE2: %r", with_sse2) logger.info("Building extensions with AVX2: %r", with_avx2) @@ -180,6 +193,11 @@ def build_extensions(self): if isinstance(e, HDF5PluginExtension): e.set_hdf5_dir(build_cmd.hdf5) + if with_cpp11: + for name, value in e.cpp11.items(): + attribute = getattr(e, name) + attribute += value + # Enable SSE2/AVX2 if available and add corresponding resources if with_sse2: e.extra_compile_args += ['-msse2'] # /arch:SSE2 is on by default @@ -210,10 +228,11 @@ def build_extensions(self): build_ext.build_extensions(self) - def __check_compile_args(self, *args): + def __check_compile_flag(self, flag, extension='.c'): """Try to compile an empty file to check for compiler args - :param List[str] args: List of arguments to pass to compiler + :param str flag: Flag argument to pass to compiler + :param str extension: Source file extension (default: '.c') :returns: Whether or not compilation was successful :rtype: bool """ @@ -222,12 +241,12 @@ def __check_compile_args(self, *args): with tempfile.TemporaryDirectory() as tmp_dir: # Create empty source file - tmp_file = os.path.join(tmp_dir, 'source.c') + tmp_file = os.path.join(tmp_dir, 'source' + extension) with open(tmp_file, 'w') as f: - f.write('/*empty source file*/\n') + f.write('int main (int argc, char **argv) { return 0; }\n') try: - self.compiler.compile([tmp_file], output_dir=tmp_dir, extra_postargs=list(args)) + self.compiler.compile([tmp_file], output_dir=tmp_dir, extra_postargs=[flag]) except CompileError: return False else: @@ -237,7 +256,7 @@ def __check_compile_args(self, *args): class HDF5PluginExtension(Extension): """Extension adding specific things to build a HDF5 plugin""" - def __init__(self, name, sse2=None, avx2=None, **kwargs): + def __init__(self, name, sse2=None, avx2=None, cpp11=None, **kwargs): Extension.__init__(self, name, **kwargs) if sys.platform.startswith('win'): @@ -254,6 +273,7 @@ def __init__(self, name, sse2=None, avx2=None, **kwargs): self.sse2 = sse2 if sse2 is not None else {} self.avx2 = avx2 if avx2 is not None else {} + self.cpp11 = cpp11 if cpp11 is not None else {} def set_hdf5_dir(self, hdf5_dir=None): """Set the HDF5 installation directory to use to build the plugins. @@ -315,7 +335,6 @@ def prefix(directory, files): # blosc plugin # Plugin from https://github.com/Blosc/hdf5-blosc # c-blosc from https://github.com/Blosc/c-blosc -# TODO snappy hdf5_blosc_dir = 'src/hdf5-blosc/src/' blosc_dir = 'src/c-blosc/' @@ -326,11 +345,15 @@ def prefix(directory, files): include_dirs = [blosc_dir, blosc_dir + 'blosc'] define_macros = [] -sse2_sources = [f for f in glob(blosc_dir + 'blosc/*.c') if 'sse2' in f] -sse2_define_macros = [('SHUFFLE_SSE2_ENABLED', 1)] +sse2_kwargs = { + 'sources': [f for f in glob(blosc_dir + 'blosc/*.c') if 'sse2' in f], + 'define_macros': [('SHUFFLE_SSE2_ENABLED', 1)], + } -avx2_sources = [f for f in glob(blosc_dir + 'blosc/*.c') if 'avx2' in f] -avx2_define_macros = [('SHUFFLE_AVX2_ENABLED', 1)] +avx2_kwargs = { + 'sources': [f for f in glob(blosc_dir + 'blosc/*.c') if 'avx2' in f], + 'define_macros': [('SHUFFLE_AVX2_ENABLED', 1)], + } # compression libs # lz4 @@ -344,7 +367,12 @@ def prefix(directory, files): define_macros.append(('HAVE_LZ4', 1)) # snappy -# TODO +cpp11_kwargs = { + 'sources': glob(blosc_dir + 'internal-complibs/snappy*/*.cc'), + 'include_dirs': glob(blosc_dir + 'internal-complibs/snappy*'), + 'extra_compile_args': ['-std=c++11', '-lstdc++'], # TODO Windows + 'define_macros': [('HAVE_SNAPPY', 1)], + } #zlib sources += glob(blosc_dir + 'internal-complibs/zlib*/*.c') @@ -368,8 +396,9 @@ def prefix(directory, files): prefix(hdf5_blosc_dir, ['blosc_filter.h', 'blosc_plugin.h']), include_dirs=include_dirs + [hdf5_blosc_dir], define_macros=define_macros, - sse2={'sources': sse2_sources, 'define_macros': sse2_define_macros}, - avx2={'sources': avx2_sources, 'define_macros': avx2_define_macros}, + sse2=sse2_kwargs, + avx2=avx2_kwargs, + cpp11=cpp11_kwargs, ) From effb0c9c7bb94538d978b4ab9eda2813cf9d8da2 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 20 Nov 2019 16:38:08 +0100 Subject: [PATCH 2/6] add snappy to supported cname --- hdf5plugin/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hdf5plugin/__init__.py b/hdf5plugin/__init__.py index 41021753..5ad5f63e 100644 --- a/hdf5plugin/__init__.py +++ b/hdf5plugin/__init__.py @@ -111,6 +111,7 @@ class Blosc(_FilterRefClass): :param str cname: `blosclz`, `lz4` (default), `lz4hc`, `zlib`, `zstd` + Optional: `snappy`, depending on compilation (requires C++11). :param int clevel: Compression level from 0 no compression to 9 maximum compression. Default: 5. @@ -135,7 +136,7 @@ class Blosc(_FilterRefClass): 'blosclz': 0, 'lz4': 1, 'lz4hc': 2, - # Not built 'snappy': 3, + 'snappy': 3, 'zlib': 4, 'zstd': 5, } From a8635a1dad86fe871d0db5f65b96ffd9f0c43118 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 20 Nov 2019 16:38:26 +0100 Subject: [PATCH 3/6] test with snappy (test also work if not available: fallback to no compression) --- hdf5plugin/test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hdf5plugin/test.py b/hdf5plugin/test.py index 9e70b133..076f9c1e 100644 --- a/hdf5plugin/test.py +++ b/hdf5plugin/test.py @@ -104,8 +104,6 @@ def testBlosc(self): for clevel in range(10): for shuffle in shuffles: for compression_id, cname in enumerate(compress): - if cname == 'snappy': - continue # Not provided filter_ = self._test( 'blosc', cname=cname, clevel=clevel, shuffle=shuffle) self.assertEqual( From 5376b1a48551212e9e3dbf64277d38d5c5ebbc31 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Wed, 20 Nov 2019 16:38:58 +0100 Subject: [PATCH 4/6] update readme: add blosc snappy + rework sample code --- README.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index de725ea8..36baeafa 100644 --- a/README.rst +++ b/README.rst @@ -48,7 +48,7 @@ Sample code: # Compression f = h5py.File('test.h5', 'w') - f.create_dataset('data', data=numpy.arange(100), compression=hdf5plugin.LZ4_ID) + f.create_dataset('data', data=numpy.arange(100), **hdf5plugin.LZ4()) f.close() # Decompression @@ -58,18 +58,18 @@ Sample code: ``hdf5plugin`` provides: -* The HDF5 filter ID of embedded plugins: - - - ``BLOSC_ID`` - - ``BSHUF_ID`` - - ``LZ4_ID`` - * Compression option helper classes to prepare arguments to provide to ``h5py.Group.create_dataset``: - `Bitshuffle(nelems=0, lz4=True)`_ - `Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE)`_ - `LZ4(nbytes=0)`_ +* The HDF5 filter ID of embedded plugins: + + - ``BLOSC_ID`` + - ``BSHUF_ID`` + - ``LZ4_ID`` + * ``FILTERS``: A dictionary mapping provided filters to their ID * ``PLUGINS_PATH``: The directory where the provided filters library are stored. @@ -104,6 +104,7 @@ This class takes the following arguments and returns the compression options to * 'blosclz' * 'lz4' (default) * 'lz4hc' + * 'snappy' (optional, requires C++11) * 'zlib' * 'zstd' From d11213e56d00bf9bba0ada8809b806b0e3595f36 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 28 Nov 2019 11:03:11 +0100 Subject: [PATCH 5/6] remove TODO --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1a16fb96..cd2b9481 100644 --- a/setup.py +++ b/setup.py @@ -370,7 +370,7 @@ def prefix(directory, files): cpp11_kwargs = { 'sources': glob(blosc_dir + 'internal-complibs/snappy*/*.cc'), 'include_dirs': glob(blosc_dir + 'internal-complibs/snappy*'), - 'extra_compile_args': ['-std=c++11', '-lstdc++'], # TODO Windows + 'extra_compile_args': ['-std=c++11', '-lstdc++'], 'define_macros': [('HAVE_SNAPPY', 1)], } From 1e274207ec7fdc3726badfa334dec3cdc7fd0703 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Thu, 28 Nov 2019 11:03:21 +0100 Subject: [PATCH 6/6] Update changelog --- CHANGELOG.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a141784d..d5ecb653 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,13 @@ +Unreleased +---------- + +- Added `--openmp=[False|True]` build option to compile bitshuffle filter with OpenMP. (PR #51) +- Added `--sse2=[True|False]` build option to compile blosc and bitshuffle filters with SSE2 instructions if available. (PR #52) +- Added `--avx2=[True|False]` build option to compile blosc and bitshuffle filters with AVX2 instructions if available. (PR #52) +- Added `--native=[True|False]` build option to compile filters for native CPU architecture. This enables SSE2/AVX2 support for the bitshuffle filter if available. (PR #52) +- Added snappy compression to the blosc filter if C++11 is available (`--cpp11=[True|False]` build option). (PR #54) +- Improved wheel generation by using root_is_pure=True setting. (PR #49) + 2.0.0 -----