silx-kit · t20100 · Aug 27, 2024 · Aug 27, 2024 · Aug 27, 2024 · Aug 27, 2024
diff --git a/doc/information.rst b/doc/information.rst
@@ -65,33 +65,31 @@ HDF5 compression filters and compression libraries sources were obtained from:
   (commit `49e3b65 <https://github.com/nexusformat/HDF5-External-Filter-Plugins/tree/49e3b65eca772bca77af13ba047d8b577673afba>`_)
   using LZ4.
 * `bitshuffle plugin <https://github.com/kiyo-masui/bitshuffle>`_ (v0.5.1) using LZ4 and ZStd.
-* bzip2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.9.2)
+* bzip2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.10.1)
   using `BZip2 <https://sourceware.org/git/bzip2.git>`_ (v1.0.8).
 * `hdf5-blosc plugin <https://github.com/Blosc/hdf5-blosc>`_ (v1.0.1)
-  using `c-blosc <https://github.com/Blosc/c-blosc>`_ (v1.21.5), LZ4, Snappy, ZLib and ZStd.
-* hdf5-blosc2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.9.2)
-  using `c-blosc2 <https://github.com/Blosc/c-blosc2>`_ (v2.15.0), LZ4, ZLib and ZStd.
+  using `c-blosc <https://github.com/Blosc/c-blosc>`_ (v1.21.6), LZ4, Snappy, ZLib and ZStd.
+* hdf5-blosc2 plugin (from `PyTables <https://github.com/PyTables/PyTables/>`_ v3.10.1)
+  using `c-blosc2 <https://github.com/Blosc/c-blosc2>`_ (v2.15.1), LZ4, ZLib and ZStd.
 * `FCIDECOMP plugin <https://gitlab.eumetsat.int/open-source/data-tailor-plugins/fcidecomp>`_
-  (`v2.0.1 <https://gitlab.eumetsat.int/open-source/data-tailor-plugins/fcidecomp/-/tree/e88f83c03bafcd0769c167dca14aa7aabf728e1b>`_)
+  (`v2.1.1 <https://gitlab.eumetsat.int/open-source/data-tailor-plugins/fcidecomp/-/tree/2.1.1>`_)
   using `CharLS <https://github.com/team-charls/charls>`_ (v2.1.0).
 * `SZ plugin <https://github.com/szcompressor/SZ>`_
   (commit `f466775 <https://github.com/szcompressor/SZ/tree/f4667759ead6a902110e80ff838ccdfddbc8dcd7>`_)
   using `SZ <https://github.com/szcompressor/SZ>`_, ZLib and ZStd.
 * `H5Z-SPERR plugin <https://github.com/NCAR/H5Z-SPERR>`_ (v0.1.3) using `SPERR <https://github.com/NCAR/SPERR>`_ (v0.8.1).
-* `SZ3 plugin <https://github.com/szcompressor/SZ3>`_
-  (commit `4bbe9df7e4bcb <https://github.com/szcompressor/SZ3/commit/4bbe9df7e4bcb6ae6339fcb3033100da07fe7434>`_)
-  using `SZ3 <https://github.com/szcompressor/SZ3>`_ and ZStd.
+* `SZ3 plugin <https://github.com/szcompressor/SZ3>`_ (v3.2.0) using `SZ3 <https://github.com/szcompressor/SZ3>`_ and ZStd.
 * `HDF5-ZFP plugin <https://github.com/LLNL/H5Z-ZFP>`_ (v1.1.1)
   using `zfp <https://github.com/LLNL/zfp>`_ (v1.0.1).
 * `HDF5Plugin-Zstandard <https://github.com/aparamon/HDF5Plugin-Zstandard>`_
   (commit `d5afdb5 <https://github.com/aparamon/HDF5Plugin-Zstandard/tree/d5afdb5f04116d5c2d1a869dc9c7c0c72832b143>`_) using ZStd.
 
 Sources of compression libraries shared accross multiple filters were obtained from:
 
-* `LZ4 v1.9.4 <https://github.com/Blosc/c-blosc2/tree/v2.15.0/internal-complibs/lz4-1.9.4>`_
+* `LZ4 v1.10.0 <https://github.com/Blosc/c-blosc2/tree/v2.15.1/internal-complibs/lz4-1.10.0>`_
 * `Snappy v1.2.1 <https://github.com/google/snappy>`_
-* `ZStd v1.5.6 <https://github.com/Blosc/c-blosc2/tree/v2.15.0/internal-complibs/zstd-1.5.6>`_
-* `ZLib v1.2.13 <https://github.com/Blosc/c-blosc/tree/v1.21.5/internal-complibs/zlib-1.2.13>`_
+* `ZStd v1.5.6 <https://github.com/Blosc/c-blosc2/tree/v2.15.1/internal-complibs/zstd-1.5.6>`_
+* `ZLib v1.3.1 <https://github.com/Blosc/c-blosc/tree/v1.21.6/internal-complibs/zlib-1.3.1>`_
 
 When compiled with Intel IPP, the LZ4 compression library is replaced with `LZ4 v1.9.3 <https://github.com/lz4/lz4/releases/tag/v1.9.3>`_ patched with a patch from Intel IPP 2021.7.0.
 

diff --git a/setup.py b/setup.py
@@ -942,15 +942,17 @@ def get_blosc2_plugin():
     ]
     define_macros = [
         ('HAVE_PLUGINS', 1),
+        ('SHUFFLE_SSE2_ENABLED', 1),
+        ('SHUFFLE_AVX2_ENABLED', 1),
         ('SHUFFLE_AVX512_ENABLED', 1),
         ('SHUFFLE_NEON_ENABLED', 1),
+        ('SHUFFLE_ALTIVEC_ENABLED', 1),
     ]
     extra_compile_args = []
     extra_link_args = []
     libraries = []
 
     if platform.machine() == 'ppc64le':
-        define_macros.append(('SHUFFLE_ALTIVEC_ENABLED', 1))
         define_macros.append(('NO_WARN_X86_INTRINSICS', None))
     if HostConfig.ARCH == 'ARM_8':
         extra_compile_args += ['-flax-vector-conversions']
@@ -1184,8 +1186,8 @@ def get_sz3_plugin():
     h5z_sz3_dir = "src/SZ3/tools/H5Z-SZ3"
 
     include_dirs = [f"{sz3_dir}/include"]
-    include_dirs += glob(f"{sz3_dir}/include/SZ3/*/")
-    include_dirs += glob(f"{sz3_dir}/include/SZ3/utils/*/")
+    include_dirs += glob(f"{sz3_dir}/include/SZ3")
+
     # add version.hpp
     include_dirs.append("src/SZ3_extra")
     if sys.platform == 'darwin':

diff --git a/src/PyTables/hdf5-blosc2/src/blosc2_filter.c b/src/PyTables/hdf5-blosc2/src/blosc2_filter.c
@@ -298,6 +298,7 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
   /* Filter params that are only set for B2ND */
   int ndim = -1;
   int32_t chunkshape[BLOSC2_MAX_DIM];
+  size_t chunksize = typesize;
   if (cd_nelmts >= 8) {
     /* Get chunk shape for B2ND */
     ndim = cd_values[7];
@@ -321,6 +322,7 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
     }
     for (int i = 0; i < ndim; i++) {
       chunkshape[i] = cd_values[8 + i];
+      chunksize *= (size_t) cd_values[8 + i];
     }
   }
 
@@ -367,6 +369,14 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
 
     blosc2_storage storage = {.cparams=&cparams, .contiguous=false};
 
+    if (ndim > 1 && nbytes != chunksize) {
+      BLOSC_TRACE_INFO("Filter input size %lu does not match chunk data size %lu "
+                       "(e.g. Fletcher32 checksum added before compression step), "
+                       "using plain Blosc2 instead of B2ND",
+                       nbytes, chunksize);
+      ndim = -1;
+    }
+
     if (ndim > 1) {
 
       b2nd_context_t *ctx = NULL;
@@ -453,6 +463,15 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
     /* declare dummy variables */
     int32_t cbytes;
 
+    /* It would be cool to have a check to detect whether the buffer contains
+     * an unexpected amout of bytes because of the application of a previous
+     * filter (e.g. a Fletcher32 checksum), thus disabling B2ND optimizations.
+     * However, we cannot know before parsing the super-chunk, and that
+     * operation accepts trailing bytes without reporting their presence.
+     * Thus such a test can only happen outside of the filter, with more
+     * information available about the whole pipeline setup.
+     */
+
     blosc2_schunk* schunk = blosc2_schunk_from_buffer(*buf, (int64_t)nbytes, false);
     if (schunk == NULL) {
       PUSH_ERR("blosc2_filter", H5E_CALLBACK, "Cannot get super-chunk from buffer");
@@ -549,6 +568,7 @@ size_t blosc2_filter_function(unsigned flags, size_t cd_nelmts,
       }
 
       blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
+      dparams.schunk = schunk;
       dctx = blosc2_create_dctx(dparams);
       status = blosc2_decompress_ctx(dctx, chunk, cbytes, outbuf, (int32_t) outbuf_size);
       if (status <= 0) {

diff --git a/src/SZ3/CMakeLists.txt b/src/SZ3/CMakeLists.txt
@@ -1,107 +1,117 @@
 cmake_minimum_required(VERSION 3.18)
-project(SZ3 VERSION 3.1.7)
+project(SZ3 VERSION 3.2.0)
+
+#data version defines the version of the compressed data format
+#it is not always equal to the program version (e.g., SZ3 v3.1.0 and SZ3 v.3.1.1 may use the same data version of v.3.1.0)
+#only update data version if the new version of the program changes compressed data format
+set(SZ3_DATA_VERSION 3.2.0)
+
 include(GNUInstallDirs)
 include(CTest)
 
 option(BUILD_SHARED_LIBS "build shared libraries by default" ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 if (MSVC)
-    set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj")
-endif()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj")
+endif ()
 set(default_build_type "Release")
-if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
-  message(STATUS "No build type was set. Setting build type to ${default_build_type}.")
-  set(CMAKE_BUILD_TYPE ${default_build_type} CACHE
-          STRING "Choose the type to build" FORCE)
-  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release"
-          "MinSizeRel" "RelWithDebInfo")
-endif()
+if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+    message(STATUS "No build type was set. Setting build type to ${default_build_type}.")
+    set(CMAKE_BUILD_TYPE ${default_build_type} CACHE
+            STRING "Choose the type to build" FORCE)
+    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release"
+            "MinSizeRel" "RelWithDebInfo")
+endif ()
 
 find_package(PkgConfig)
-find_package(OpenMP)
 
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/SZ3/version.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp)
+configure_file(
+        ${CMAKE_CURRENT_SOURCE_DIR}/include/SZ3/version.hpp.in
+        ${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp)
 
 add_library(${PROJECT_NAME} INTERFACE)
 target_include_directories(
         ${PROJECT_NAME} INTERFACE
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
         $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/include>
         $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
-        )
+)
 target_compile_features(${PROJECT_NAME}
-    INTERFACE cxx_std_17
-  )
+        INTERFACE cxx_std_17
+)
 
-target_link_libraries(${PROJECT_NAME} INTERFACE OpenMP::OpenMP_CXX)
+find_package(OpenMP)
+if (OpenMP_FOUND)
+    target_link_libraries(${PROJECT_NAME} INTERFACE OpenMP::OpenMP_CXX)
+endif ()
 
 if (MSVC)
     option(SZ3_USE_BUNDLED_ZSTD "prefer the bundled version of Zstd" ON)
-else()
+else ()
     option(SZ3_USE_BUNDLED_ZSTD "prefer the bundled version of Zstd" OFF)
-endif()
-option(SZ3_DEBUG_TIMINGS "print debug timing information" ON)
+endif ()
 
-if(SZ3_DEBUG_TIMINGS)
-  target_compile_definitions(${PROJECT_NAME} INTERFACE SZ3_DEBUG_TIMINGS=1)
-else()
-  target_compile_definitions(${PROJECT_NAME} INTERFACE SZ3_DEBUG_TIMINGS=0)
-endif()
+option(SZ3_DEBUG_TIMINGS "print debug timing information" OFF)
+if (SZ3_DEBUG_TIMINGS)
+    target_compile_definitions(${PROJECT_NAME} INTERFACE SZ3_DEBUG_TIMINGS=1)
+else ()
+    target_compile_definitions(${PROJECT_NAME} INTERFACE SZ3_DEBUG_TIMINGS=0)
+endif ()
 
 pkg_search_module(ZSTD IMPORTED_TARGET libzstd)
-if(ZSTD_FOUND AND NOT SZ3_USE_BUNDLED_ZSTD)
-  target_link_libraries(${PROJECT_NAME} INTERFACE PkgConfig::ZSTD)
-else()
-  add_subdirectory(tools/zstd)
-  target_link_libraries(${PROJECT_NAME} INTERFACE zstd)
-endif()
+if (ZSTD_FOUND AND NOT SZ3_USE_BUNDLED_ZSTD)
+    target_link_libraries(${PROJECT_NAME} INTERFACE PkgConfig::ZSTD)
+else ()
+    add_subdirectory(tools/zstd)
+    target_link_libraries(${PROJECT_NAME} INTERFACE zstd)
+endif ()
 
 find_package(GSL)
 if (GSL_FOUND)
-  target_compile_definitions(${PROJECT_NAME} INTERFACE -DENABLE_GSL="TRUE")
-  target_link_libraries(${PROJECT_NAME} INTERFACE GSL::gsl)
-endif()
+    target_compile_definitions(${PROJECT_NAME} INTERFACE -DENABLE_GSL="TRUE")
+    target_link_libraries(${PROJECT_NAME} INTERFACE GSL::gsl)
+endif ()
 
 add_subdirectory(tools/sz3)
 add_subdirectory(tools/sz3c)
 
 
 option(BUILD_H5Z_FILTER "build the H5Z-SZ3 filter" OFF)
-if(BUILD_H5Z_FILTER)
-	add_subdirectory(tools/H5Z-SZ3)
-endif()
+if (BUILD_H5Z_FILTER)
+    add_subdirectory(tools/H5Z-SZ3)
+endif ()
 
 option(BUILD_MDZ "build MDZ (MMD-SZ)" OFF)
-if(BUILD_MDZ)
-  add_subdirectory(tools/mdz)
-endif()
+if (BUILD_MDZ)
+    add_subdirectory(tools/mdz)
+endif ()
 
 
-install(TARGETS ${PROJECT_NAME} 
-  EXPORT SZ3Targets
-  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-  )
+install(TARGETS ${PROJECT_NAME}
+        EXPORT SZ3Targets
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+)
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES
-  "${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp"
-  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/SZ3/"
-  )
+        "${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp"
+        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/SZ3/"
+)
 install(EXPORT SZ3Targets NAMESPACE SZ3:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3)
 include(CMakePackageConfigHelpers)
 configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/SZ3Config.cmake.in
-  "${CMAKE_CURRENT_BINARY_DIR}/SZ3Config.cmake"
-  INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3
-  )
+        "${CMAKE_CURRENT_BINARY_DIR}/SZ3Config.cmake"
+        INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3
+)
 write_basic_package_version_file(
-  "${CMAKE_CURRENT_BINARY_DIR}/SZ3ConfigVersion.cmake"
-  VERSION ${PROJECT_VERSION}
-  COMPATIBILITY AnyNewerVersion
-  )
+        "${CMAKE_CURRENT_BINARY_DIR}/SZ3ConfigVersion.cmake"
+        VERSION ${PROJECT_VERSION}
+        COMPATIBILITY AnyNewerVersion
+)
 install(FILES
-  ${CMAKE_CURRENT_BINARY_DIR}/SZ3Config.cmake
-  ${CMAKE_CURRENT_BINARY_DIR}/SZ3ConfigVersion.cmake
-  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3
-  )
+        ${CMAKE_CURRENT_BINARY_DIR}/SZ3Config.cmake
+        ${CMAKE_CURRENT_BINARY_DIR}/SZ3ConfigVersion.cmake
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3
+)
 #export sz3 target for external use
 export(TARGETS SZ3 FILE SZ3.cmake)