Refactor Blosc operator using c-blosc2 and add backward compatibility…

… for bp3 files written with ADIOS version 2.7 and before

CMakeLists.txt

@@ -139,7 +139,6 @@ endif()
 
 include(CTest)
 
-adios_option(Blosc      "Enable support for Blosc transforms" AUTO)
 adios_option(Blosc2     "Enable support for c-blosc-2 transforms" AUTO)
 adios_option(BZip2      "Enable support for BZip2 transforms" AUTO)
 adios_option(ZFP        "Enable support for ZFP transforms" AUTO)
@@ -161,10 +160,12 @@ adios_option(IME        "Enable support for DDN IME transport" AUTO)
 adios_option(Python     "Enable support for Python bindings" AUTO)
 adios_option(Fortran    "Enable support for Fortran bindings" AUTO)
 adios_option(SysVShMem  "Enable support for SysV Shared Memory IPC on *NIX" AUTO)
+adios_option(UCX        "Enable support for UCX DataPlane in SST" AUTO)
 adios_option(Profiling  "Enable support for profiling" AUTO)
 adios_option(Endian_Reverse "Enable support for Little/Big Endian Interoperability" AUTO)
 adios_option(Sodium     "Enable support for Sodium for encryption" AUTO)
 adios_option(Catalyst   "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO)
+adios_option(AWSSDK     "Enable support for S3 compatible storage using AWS SDK's S3 module" AUTO)
 include(${PROJECT_SOURCE_DIR}/cmake/DetectOptions.cmake)
 
 if(ADIOS2_HAVE_CUDA)
@@ -225,7 +226,9 @@ endif()
 
 
 set(ADIOS2_CONFIG_OPTS
-    BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem ZeroMQ Profiling Endian_Reverse GPU_Support
+    BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc Blosc2 
+    BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ 
+    Profiling Endian_Reverse GPU_Support AWSSDK
 )
 
 GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS})
@@ -397,7 +400,7 @@ foreach(opt IN LISTS ADIOS2_CONFIG_OPTS)
   endif()
 endforeach()
 
-if (ADIOS2_HAVE_SST AND ADIOS2_SST_HAVE_LIBFABRIC) 
+if (ADIOS2_HAVE_SST AND (ADIOS2_SST_HAVE_LIBFABRIC OR ADIOS2_SST_HAVE_UCX))
 message("    RDMA Transport for Staging: Available")
 else()
 message("    RDMA Transport for Staging: Unconfigured")

cmake/DetectOptions.cmake

@@ -67,16 +67,6 @@ function(lists_get_prefix listVars outVar)
   set(${outVar} "${prefix}" PARENT_SCOPE)
 endfunction()
 
-# Blosc
-if(ADIOS2_USE_Blosc STREQUAL AUTO)
-  find_package(Blosc 1.7)
-elseif(ADIOS2_USE_Blosc)
-  find_package(Blosc 1.7 REQUIRED)
-endif()
-if(BLOSC_FOUND)
-  set(ADIOS2_HAVE_Blosc TRUE)
-endif()
-
 # Blosc2
 if(ADIOS2_USE_Blosc2 STREQUAL AUTO)
   find_package(Blosc2 2.4)
@@ -405,6 +395,16 @@ if(ADIOS2_USE_SST AND NOT WIN32)
     ADIOS2_HAVE_MPI_CLIENT_SERVER)
     unset(CMAKE_REQUIRED_LIBRARIES)
   endif()
+  # UCX
+  if(ADIOS2_USE_UCX STREQUAL AUTO)
+    find_package(UCX 1.9.0)
+  elseif(ADIOS2_USE_UCX)
+    find_package(UCX 1.9.0)
+  endif()
+  if(UCX_FOUND)
+    set(ADIOS2_SST_HAVE_UCX TRUE)
+    set(ADIOS2_HAVE_UCX TRUE)
+  endif()
 endif()
 
 # DAOS
@@ -466,6 +466,16 @@ if(catalyst_FOUND)
   set(ADIOS2_HAVE_Catalyst TRUE)
 endif()
 
+# AWS S3
+if(ADIOS2_USE_AWSSDK STREQUAL AUTO)
+    find_package(AWSSDK QUIET COMPONENTS s3)
+elseif(ADIOS2_USE_AWSSDK)
+    find_package(AWSSDK REQUIRED COMPONENTS s3)
+endif()
+if(AWSSDK_FOUND)
+    set(ADIOS2_HAVE_AWSSDK TRUE)
+endif()
+
 # Multithreading
 find_package(Threads REQUIRED)
 

cmake/adios2-config-common.cmake.in

@@ -76,11 +76,6 @@ else()
 endif()
 
 if(NOT @BUILD_SHARED_LIBS@)
-  set(ADIOS2_HAVE_Blosc @ADIOS2_HAVE_Blosc@)
-  if(ADIOS2_HAVE_Blosc)
-    find_dependency(Blosc)
-  endif()
-
   set(ADIOS2_HAVE_Blosc2 @ADIOS2_HAVE_Blosc2@)
   if(ADIOS2_HAVE_Blosc2)
     find_dependency(Blosc2)
@@ -142,6 +137,12 @@ if(NOT @BUILD_SHARED_LIBS@)
     find_dependency(catalyst)
   endif()
 
+  set(ADIOS2_HAVE_AWSSDK @ADIOS2_HAVE_AWSSDK@)
+  if(ADIOS2_HAVE_AWSSDK)
+    find_dependency(AWSSDK)
+  endif()
+
+
   adios2_add_thirdparty_target(pugixml)
   set(ADIOS2_USE_EXTERNAL_PUGIXML @ADIOS2_USE_EXTERNAL_PUGIXML@)
   if(ADIOS2_USE_EXTERNAL_PUGIXML)
@@ -168,6 +169,11 @@ if(NOT @BUILD_SHARED_LIBS@)
       find_dependency(LIBFABRIC)
     endif()
 
+    set(ADIOS2_SST_HAVE_UCX @ADIOS2_SST_HAVE_UCX@)
+    if(ADIOS2_SST_HAVE_UCX)
+      find_dependency(UCX 1.9.0)
+    endif()
+
     find_dependency(EVPath)
     add_library(adios2::thirdparty::EVPath INTERFACE IMPORTED)
     set_target_properties(adios2::thirdparty::EVPath PROPERTIES

docs/user_guide/source/engines/sst.rst

@@ -157,7 +157,7 @@ the underlying network communication mechanism to use for exchanging
 data in SST.  Generally this is chosen by SST based upon what is
 available on the current platform.  However, specifying this engine
 parameter allows overriding SST's choice.  Current allowed values are
-**"MPI"**, **"RDMA"**, and **"WAN"**.  (**ib** and **fabric** are accepted as
+**"UCX"**,**"MPI"**, **"RDMA"**, and **"WAN"**.  (**ib** and **fabric** are accepted as
 equivalent to **RDMA** and **evpath** is equivalent to **WAN**.)
 Generally both the reader and writer should be using the same network
 transport, and the network transport chosen may be dictated by the
@@ -280,15 +280,15 @@ single reader, but only upon request (with a request being initiated
 by the reader doing BeginStep()).  Normal reader-side rules (like
 BeginStep timeouts) and writer-side rules (like queue limit behavior) apply.
 
-============================= ===================== ================================================
+============================= ===================== ====================================================
  **Key**                        **Value Format**      **Default** and Examples
-============================= ===================== ================================================
+============================= ===================== ====================================================
  RendezvousReaderCount           integer             **1**
  RegistrationMethod              string              **File**, Screen
  QueueLimit                      integer             **0** (no queue limits)
  QueueFullPolicy                 string              **Block**, Discard
  ReserveQueueLimit               integer             **0** (no queue limits)
- DataTransport                   string              **default varies by platform**, MPI, RDMA, WAN
+ DataTransport                   string              **default varies by platform**, UCX, MPI, RDMA, WAN
  WANDataTransport                string              **sockets**, enet, ib
  ControlTransport                string              **TCP**, Scalable
  NetworkInterface                string              **NULL**
@@ -299,4 +299,4 @@ BeginStep timeouts) and writer-side rules (like queue limit behavior) apply.
  OpenTimeoutSecs                 integer             **60**
  SpeculativePreloadMode          string              **AUTO**, ON, OFF
  SpecAutoNodeThreshold           integer             **1**
-============================= ===================== ================================================
+============================= ===================== =====================================================

docs/user_guide/source/setting_up/source/cmake.rst

@@ -86,7 +86,7 @@ VAR                            VALUE                     Description
 ``ADIOS2_USE_HDF5``            **ON**/OFF      `HDF5 <https://www.hdfgroup.org>`_ engine. If HDF5 is not on the syspath, it can be set using ``-DHDF5_ROOT=/path/to/hdf5``
 ``ADIOS2_USE_Python``          **ON**/OFF      Python bindings. Python 3 will be used if found. If you want to specify a particular python version use  ``-DPYTHON_EXECUTABLE=/path/to/interpreter/python``
 ``ADIOS2_USE_Fortran``         **ON**/OFF      Bindings for Fortran 90 or above.
-``ADIOS2_USE_SST``             **ON**/OFF      Simplified Staging Engine (SST) and its dependencies, requires MPI. Can optionally use LibFabric for RDMA transport. Specify the LibFabric install manually with the -DLIBFABRIC_ROOT=... option.
+``ADIOS2_USE_SST``             **ON**/OFF      Simplified Staging Engine (SST) and its dependencies, requires MPI. Can optionally use LibFabric/UCX for RDMA transport. You can specify the LibFabric/UCX path manually with the -DLIBFABRIC_ROOT=... or -DUCX_ROOT=... option.
 ``ADIOS2_USE_BZip2``           **ON**/OFF      `BZIP2 <http://www.bzip.org>`_ compression.
 ``ADIOS2_USE_ZFP``             **ON**/OFF      `ZFP <https://github.com/LLNL/zfp>`_ compression (experimental).
 ``ADIOS2_USE_SZ``              **ON**/OFF      `SZ <https://github.com/disheng222/SZ>`_ compression (experimental).

source/adios2/CMakeLists.txt

@@ -138,6 +138,11 @@ if(UNIX)
   target_sources(adios2_core PRIVATE toolkit/transport/file/FilePOSIX.cpp)
 endif()
 
+if(ADIOS2_HAVE_AWSSDK)
+  target_sources(adios2_core PRIVATE toolkit/transport/file/FileAWSSDK.cpp)
+  target_link_libraries(adios2_core PRIVATE ${AWSSDK_LINK_LIBRARIES})
+endif()
+
 if (ADIOS2_HAVE_BP5)
   target_sources(adios2_core PRIVATE
     engine/bp5/BP5Engine.cpp
@@ -239,13 +244,8 @@ if(ADIOS2_HAVE_DataSpaces)
   target_link_libraries(adios2_core_mpi PRIVATE DataSpaces::DataSpaces)
 endif()
 
-if(ADIOS2_HAVE_Blosc)
-  target_sources(adios2_core PRIVATE operator/compress/CompressBlosc.cpp)
-  target_link_libraries(adios2_core PRIVATE Blosc::Blosc)
-endif()
-
 if(ADIOS2_HAVE_Blosc2)
-  target_sources(adios2_core PRIVATE operator/compress/CompressBlosc2.cpp)
+  target_sources(adios2_core PRIVATE operator/compress/CompressBlosc.cpp)
   target_link_libraries(adios2_core PRIVATE Blosc2::Blosc2)
 endif()
 

source/adios2/common/ADIOSTypes.h

@@ -480,8 +480,8 @@ constexpr char blockSize100k_9[] = "9";
 } // end namespace bzip2
 #endif
 
-// BBlosc PARAMETERS
-#ifdef ADIOS2_HAVE_BLOSC
+// Blosc PARAMETERS
+#ifdef ADIOS2_HAVE_BLOSC2
 
 constexpr char LosslessBlosc[] = "blosc";
 namespace blosc
@@ -528,54 +528,6 @@ constexpr char doshuffle_bitshuffle[] = "BLOSC_BITSHUFFLE";
 
 #endif
 
-// Blosc2 PARAMETERS
-#ifdef ADIOS2_HAVE_BLOSC2
-
-constexpr char LosslessBlosc2[] = "blosc2";
-namespace blosc2
-{
-
-namespace key
-{
-constexpr char nthreads[] = "nthreads";
-constexpr char compressor[] = "compressor";
-constexpr char clevel[] = "clevel";
-constexpr char doshuffle[] = "doshuffle";
-constexpr char blocksize[] = "blocksize";
-constexpr char threshold[] = "threshold";
-}
-
-namespace value
-{
-
-constexpr char compressor_blosclz[] = "blosclz";
-constexpr char compressor_lz4[] = "lz4";
-constexpr char compressor_lz4hc[] = "lz4hc";
-constexpr char compressor_snappy[] = "snappy";
-constexpr char compressor_zlib[] = "zlib";
-constexpr char compressor_zstd[] = "zstd";
-
-constexpr char clevel_0[] = "0";
-constexpr char clevel_1[] = "1";
-constexpr char clevel_2[] = "2";
-constexpr char clevel_3[] = "3";
-constexpr char clevel_4[] = "4";
-constexpr char clevel_5[] = "5";
-constexpr char clevel_6[] = "6";
-constexpr char clevel_7[] = "7";
-constexpr char clevel_8[] = "8";
-constexpr char clevel_9[] = "9";
-
-constexpr char doshuffle_shuffle[] = "BLOSC_SHUFFLE";
-constexpr char doshuffle_noshuffle[] = "BLOSC_NOSHUFFLE";
-constexpr char doshuffle_bitshuffle[] = "BLOSC_BITSHUFFLE";
-
-} // end namespace value
-
-} // end namespace blosc2
-
-#endif
-
 } // end namespace ops
 
 } // end namespace adios2