From c7ec1b2d9179248715abc2e0823a75ba962ca674 Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Tue, 2 Mar 2021 18:46:14 +0000 Subject: [PATCH 1/4] DataSpaces engine update for DataSpaces2 --- cmake/FindDataSpaces.cmake | 63 +++++++++++++------ cmake/FindLIBFABRIC.cmake | 4 +- source/adios2/CMakeLists.txt | 1 + .../engine/dataspaces/DataSpacesReader.cpp | 37 +++++++++++ .../engine/dataspaces/DataSpacesReader.tcc | 12 +++- .../engine/dataspaces/DataSpacesWriter.cpp | 23 +++++-- .../engine/dataspaces/DataSpacesWriter.tcc | 17 +++++ .../adios2/toolkit/dataspaces/CMakeLists.txt | 8 +++ .../toolkit/dataspaces/DSpacesConfig.h.in | 12 ++++ source/adios2/toolkit/dataspaces/ds.h | 7 +++ source/adios2/toolkit/dataspaces/ds_writer.c | 34 +++++++++- 11 files changed, 187 insertions(+), 31 deletions(-) create mode 100644 source/adios2/toolkit/dataspaces/CMakeLists.txt create mode 100644 source/adios2/toolkit/dataspaces/DSpacesConfig.h.in diff --git a/cmake/FindDataSpaces.cmake b/cmake/FindDataSpaces.cmake index 6025fc52ed..482f510520 100644 --- a/cmake/FindDataSpaces.cmake +++ b/cmake/FindDataSpaces.cmake @@ -34,7 +34,33 @@ if(NOT DATASPACES_FOUND) get_filename_component(DATASPACES_ROOT "${DSPACES_CONF}/../.." ABSOLUTE) endif() endif() - if(DATASPACES_ROOT) + find_package(PkgConfig) + if(PKG_CONFIG_FOUND) + set(_CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH}) + if(DATASPACES_ROOT) + list(INSERT CMAKE_PREFIX_PATH 0 "${DATASPACES_ROOT}") + elseif(NOT ENV{DATASPACES_ROOT} STREQUAL "") + list(INSERT CMAKE_PREFIX_PATH 0 "$ENV{DATASPACES_ROOT}") + endif() + set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON) + pkg_check_modules(PC_DSPACES dspaces) + set(CMAKE_PREFIX_PATH ${_CMAKE_PREFIX_PATH}) + unset(_CMAKE_PREFIX_PATH) + if(PC_DSPACES_FOUND) + if(BUILD_SHARED_LIBS) + set(_PC_TYPE) + else() + set(_PC_TYPE _STATIC) + endif() + set(DATASPACES_LIBRARIES ${PC_DSPACES${_PC_TYPE}_LINK_LIBRARIES}) + set(DATASPACES_LIBRARY_HINT ${PC_DSPACES${_PC_TYPE}_LIBRARY_DIRS}) + set(DATASPACES_INCLUDE_DIR ${PC_DSPACES${_PC_TYPE}_INCLUDE_DIRS}) + set(DATASPACES_VERSION ${PC_DSPACES_VERSION}) + find_library(DATASPACES_LIBRARY dspaces HINTS ${DATASPACES_LIBRARY_HINT}) + set(HAVE_DSPACES2 TRUE) + endif() + endif() + if(DATASPACES_ROOT AND NOT PC_DSPACES_FOUND) find_program(DSPACES_CONF dspaces_config ${DATASPACES_ROOT}/bin) if(DSPACES_CONF) execute_process(COMMAND ${DSPACES_CONF} -l @@ -50,50 +76,47 @@ if(NOT DATASPACES_FOUND) foreach(LOOP_VAR ${LINK_LIBS}) STRING(FIND ${LOOP_VAR} "-u" DEL_FLG) if(("${DEL_FLG}" EQUAL "-1")) - STRING(FIND ${LOOP_VAR} "/" HINT_FLG) - if(NOT("${HINT_FLG}" EQUAL "-1")) + STRING(FIND ${LOOP_VAR} "/" HINT_FLG) + if(NOT("${HINT_FLG}" EQUAL "-1")) list(APPEND DATASPACES_LIBRARY_HINT ${LOOP_VAR}) - else() + else() unset(LOCAL_LIBRARY CACHE) unset(LOCAL_LIBRARY-NOTFOUND CACHE) STRING(FIND ${LOOP_VAR} "stdc++" CPP_FLG) if("${CPP_FLG}" EQUAL "-1") - find_library(LOCAL_LIBRARY NAMES "${LOOP_VAR}" HINTS ${DATASPACES_LIBRARY_HINT}) - if(LOCAL_LIBRARY) + find_library(LOCAL_LIBRARY NAMES "${LOOP_VAR}" HINTS ${DATASPACES_LIBRARY_HINT}) + if(LOCAL_LIBRARY) list(APPEND DATASPACES_LIBRARIES ${LOCAL_LIBRARY}) - else() + else() list(APPEND DATASPACES_LIBRARIES ${LOOP_VAR}) - endif() + endif() endif() - endif() - endif() - endforeach() - execute_process(COMMAND ${DSPACES_CONF} -v + endif() + endif() + endforeach() + execute_process(COMMAND ${DSPACES_CONF} -v RESULT_VARIABLE RESULT_VAR OUTPUT_VARIABLE DATASPACES_VERSION ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - endif () - + endif() set(DATASPACES_INCLUDE_OPTS HINTS ${DATASPACES_ROOT}/include) - + find_path(DATASPACES_INCLUDE_DIR dataspaces.h ${DATASPACES_INCLUDE_OPTS}) + find_library(DATASPACES_LIBRARY dspaces HINTS ${DATASPACES_LIBRARY_HINT}) endif() - find_path(DATASPACES_INCLUDE_DIR dataspaces.h ${DATASPACES_INCLUDE_OPTS}) - find_library(DSPACES_LIBRARY dspaces HINTS ${DATASPACES_LIBRARY_HINT}) - include(FindPackageHandleStandardArgs) find_package_handle_standard_args(DataSpaces FOUND_VAR DATASPACES_FOUND VERSION_VAR DATASPACES_VERSION REQUIRED_VARS DATASPACES_VERSION DATASPACES_INCLUDE_DIR - DATASPACES_LIBRARIES DSPACES_LIBRARY + DATASPACES_LIBRARIES #DATASPACES_LIBRARY ) if(DATASPACES_FOUND) if(DATASPACES_FOUND AND NOT TARGET DataSpaces::DataSpaces) add_library(DataSpaces::DataSpaces UNKNOWN IMPORTED) set_target_properties(DataSpaces::DataSpaces PROPERTIES - IMPORTED_LOCATION "${DSPACES_LIBRARY}" + IMPORTED_LOCATION "${DATASPACES_LIBRARY}" INTERFACE_LINK_LIBRARIES "${DATASPACES_LIBRARIES}" INTERFACE_INCLUDE_DIRECTORIES "${DATASPACES_INCLUDE_DIR}" ) diff --git a/cmake/FindLIBFABRIC.cmake b/cmake/FindLIBFABRIC.cmake index d9c42f6695..4f518bddce 100644 --- a/cmake/FindLIBFABRIC.cmake +++ b/cmake/FindLIBFABRIC.cmake @@ -30,11 +30,11 @@ if(NOT (PC_LIBFABRIC_FOUND STREQUAL "IGNORE")) if(BUILD_SHARED_LIBS) set(_PC_TYPE) else() - set(_PF_TYPE _STATIC) + set(_PC_TYPE _STATIC) endif() set(LIBFABRIC_INCLUDE_DIRS ${PC_LIBFABRIC${_PC_TYPE}_INCLUDE_DIRS}) set(LIBFABRIC_LIBRARIES ${PC_LIBFABRIC${_PC_TYPE}_LINK_LIBRARIES}) - set(LIBFABRIC_DEFINITIONS ${PC_LIBFABRIC${PC_TYPE}_CFLAGS_OTHER}) + set(LIBFABRIC_DEFINITIONS ${PC_LIBFABRIC${_PC_TYPE}_CFLAGS_OTHER}) endif() endif() endif() diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 2b3a844474..b60b310bd4 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -215,6 +215,7 @@ if(ADIOS2_HAVE_SST) endif() if(ADIOS2_HAVE_DataSpaces) + add_subdirectory(toolkit/dataspaces) target_sources(adios2_core_mpi PRIVATE toolkit/dataspaces/ds_writer.c engine/dataspaces/DataSpacesWriter.cpp engine/dataspaces/DataSpacesWriter.tcc diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.cpp b/source/adios2/engine/dataspaces/DataSpacesReader.cpp index 2e5a6d33a8..0d90f253e1 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesReader.cpp @@ -14,8 +14,13 @@ #include "DataSpacesReader.tcc" #include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //CSVToVector +#include "adios2/toolkit/dataspaces/DSpacesConfig.h" #include "adios2/toolkit/dataspaces/ds_data.h" +#ifdef HAVE_DSPACES2 +#include "dspaces.h" +#else #include "dataspaces.h" +#endif /* HAVE_DSPACES2 */ namespace adios2 { @@ -83,23 +88,53 @@ StepStatus DataSpacesReader::BeginStep(StepMode mode, const float timeout_sec) strcpy(meta_lk, lk_name.c_str()); MPI_Comm lock_comm = m_data.mpi_comm; +#ifndef HAVE_DSPACES2 dspaces_lock_on_read(meta_lk, &lock_comm); +#endif /* HAVE_DSPACES2 */ int nVars = 0; if (!m_ProvideLatest) { if (rank == 0) { +#ifdef HAVE_DSPACES2 + dspaces_client_t *client = get_client_handle(); + char meta_str[256]; + sprintf(meta_str, "VARMETA@%s", fstr); + unsigned int metalen; + int var_name_max_length = 128; + int var_size = 2 * sizeof(int) + MAX_DS_NDIM * sizeof(uint64_t) + + var_name_max_length * sizeof(char); + int err = dspaces_get_meta(*client, meta_str, META_MODE_NEXT, + m_CurrentStep, &bcast_array[1], + (void **)&buffer, &metalen); + bcast_array[0] = metalen / var_size; +#else buffer = dspaces_get_next_meta(m_CurrentStep, fstr, &bcast_array[0], &bcast_array[1]); +#endif /* HAVE_DSPACES2 */ } } else { if (rank == 0) { +#ifdef HAVE_DSPACES2 + dspaces_client_t *client = get_client_handle(); + char meta_str[256]; + sprintf(meta_str, "VARMETA@%s", fstr); + unsigned int metalen; + int var_name_max_length = 128; + int var_size = 2 * sizeof(int) + MAX_DS_NDIM * sizeof(uint64_t) + + var_name_max_length * sizeof(char); + int err = dspaces_get_meta(*client, meta_str, META_MODE_LAST, + m_CurrentStep, &bcast_array[1], + (void **)&buffer, &metalen); + bcast_array[0] = metalen / var_size; +#else buffer = dspaces_get_latest_meta(m_CurrentStep, fstr, &bcast_array[0], &bcast_array[1]); +#endif /* HAVE_DSPACES2 */ } } MPI_Bcast(bcast_array, 2, MPI_INT, 0, m_data.mpi_comm); @@ -228,7 +263,9 @@ void DataSpacesReader::EndStep() strcpy(meta_lk, lk_name.c_str()); MPI_Comm lock_comm = m_data.mpi_comm; +#ifndef HAVE_DSPACES2 dspaces_unlock_on_read(meta_lk, &lock_comm); +#endif /* HAVE_DSPACES2 */ } void DataSpacesReader::DoClose(const int transportIndex) diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.tcc b/source/adios2/engine/dataspaces/DataSpacesReader.tcc index 75eb20f89f..9f2cae04aa 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.tcc +++ b/source/adios2/engine/dataspaces/DataSpacesReader.tcc @@ -15,7 +15,12 @@ #include "DataSpacesReader.h" #include "adios2/helper/adiosFunctions.h" //CSVToVector +#include "adios2/toolkit/dataspaces/DSpacesConfig.h" +#ifdef HAVE_DSPACES2 +#include "dspaces.h" +#else #include "dataspaces.h" +#endif /* HAVE_DSPACES2 */ namespace adios2 { @@ -67,7 +72,9 @@ void DataSpacesReader::ReadDsData(Variable &variable, T *data, int version) */ if (variable.m_Shape.size() == 0) { +#ifndef HAVE_DSPACES2 gdims_in[0] = dspaces_get_num_space_server(); +#endif lb_in[0] = 0; ub_in[0] = 0; ndims = 1; @@ -108,11 +115,12 @@ void DataSpacesReader::ReadDsData(Variable &variable, T *data, int version) std::string l_Name = ds_in_name + std::to_string(version); char *cstr = new char[l_Name.length() + 1]; strcpy(cstr, l_Name.c_str()); - +#ifdef HAVE_DSPACES2 +#else dspaces_define_gdim(var_str, ndims, gdims_in); dspaces_get(var_str, version, variable.m_ElementSize, ndims, lb_in, ub_in, (void *)data); - +#endif /* HAVE_DSPACES2 */ delete[] cstr; delete[] var_str; } diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp index 17589f8628..4c81caba70 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp @@ -15,8 +15,13 @@ #include "DataSpacesWriter.tcc" #include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //CSVToVector +#include "adios2/toolkit/dataspaces/DSpacesConfig.h" #include "adios2/toolkit/dataspaces/ds_data.h" +#ifdef HAVE_DSPACES2 +#include "dspaces.h" +#else #include "dataspaces.h" +#endif /* HAVE_DSPACES2 */ namespace adios2 { @@ -73,10 +78,13 @@ void DataSpacesWriter::EndStep() char *meta_lk = new char[local_file_var.length() + 1]; strcpy(meta_lk, local_file_var.c_str()); MPI_Comm lock_comm = m_data.mpi_comm; - +#ifndef HAVE_DSPACES2 dspaces_lock_on_write(meta_lk, &lock_comm); +#endif /* HAVE_DSPACES2 */ WriteVarInfo(); +#ifndef HAVE_DSPACES2 dspaces_unlock_on_write(meta_lk, &lock_comm); +#endif /* HAVE_DSPACES2 */ } void DataSpacesWriter::Flush(const int transportIndex) {} @@ -88,8 +96,10 @@ void DataSpacesWriter::DoClose(const int transportIndex) char *meta_lk = new char[local_file_var.length() + 1]; strcpy(meta_lk, local_file_var.c_str()); +#ifndef HAVE_DSPACES2 dspaces_lock_on_write(meta_lk, &(m_data.mpi_comm)); dspaces_unlock_on_write(meta_lk, &(m_data.mpi_comm)); +#endif /* HAVE_DSPACES2 */ globals_adios_set_dataspaces_disconnected_from_writer(); } @@ -126,9 +136,9 @@ void DataSpacesWriter::WriteVarInfo() std::string ds_file_var; int var_num = ndim_vector.size(); int var_name_max_length = 128; - int buf_len = var_num * sizeof(int) + var_num * sizeof(int) + - MAX_DS_NDIM * var_num * sizeof(uint64_t) + - var_num * var_name_max_length * sizeof(char); + int buf_len = + var_num * (2 * sizeof(int) + MAX_DS_NDIM * sizeof(uint64_t) + + var_name_max_length * sizeof(char)); int *dim_meta, *elemSize_meta; uint64_t *gdim_meta; dim_meta = (int *)malloc(var_num * sizeof(int)); @@ -174,12 +184,17 @@ void DataSpacesWriter::WriteVarInfo() ndim = 1; lb[0] = 0; ub[0] = buf_len - 1; +#ifdef HAVE_DSPACES2 + dspaces_client_t *client = get_client_handle(); + dspaces_put_meta(*client, local_str, m_CurrentStep, buffer, buf_len); +#else gdims[0] = (ub[0] - lb[0] + 1) * dspaces_get_num_space_server(); dspaces_define_gdim(local_str, ndim, gdims); dspaces_put(local_str, m_CurrentStep, elemsize, ndim, lb, ub, buffer); dspaces_put_sync(); // wait on previous put to finish +#endif /* HAVE_DSPACES2 */ delete[] local_str; free(dim_meta); free(elemSize_meta); diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.tcc b/source/adios2/engine/dataspaces/DataSpacesWriter.tcc index c9feaab041..b705122a93 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.tcc +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.tcc @@ -15,8 +15,14 @@ #include "DataSpacesWriter.h" #include "adios2/helper/adiosFunctions.h" +#include "adios2/toolkit/dataspaces/DSpacesConfig.h" +#include "adios2/toolkit/dataspaces/ds.h" #include "adios2/toolkit/dataspaces/ds_data.h" +#ifdef HAVE_DSPACES2 +#include "dspaces.h" +#else #include "dataspaces.h" +#endif /* HAVE_DSPACES2 */ namespace adios2 { @@ -48,7 +54,11 @@ void DataSpacesWriter::DoPutSyncCommon(Variable &variable, const T *values) */ if (variable.m_SingleValue) { +#ifdef HAVE_DSPACES2 + gdims_in[0] = 1; +#else gdims_in[0] = dspaces_get_num_space_server(); +#endif /* HAVE_DSPACES2 */ lb_in[0] = 0; ub_in[0] = 0; ndims = 1; @@ -110,11 +120,18 @@ void DataSpacesWriter::DoPutSyncCommon(Variable &variable, const T *values) char *cstr = new char[l_Name.length() + 1]; strcpy(cstr, l_Name.c_str()); +#ifdef HAVE_DSPACES2 + dspaces_client_t *client = get_client_handle(); + dspaces_define_gdim(*client, var_str, ndims, gdims_in); + dspaces_put(*client, var_str, version, variable.m_ElementSize, ndims, lb_in, + ub_in, values); +#else dspaces_define_gdim(var_str, ndims, gdims_in); dspaces_put(var_str, version, variable.m_ElementSize, ndims, lb_in, ub_in, values); dspaces_put_sync(); dspaces_put_sync(); +#endif /* HAVE_DSPACES2 */ delete[] cstr; delete[] var_str; } diff --git a/source/adios2/toolkit/dataspaces/CMakeLists.txt b/source/adios2/toolkit/dataspaces/CMakeLists.txt new file mode 100644 index 0000000000..20a2c8a08d --- /dev/null +++ b/source/adios2/toolkit/dataspaces/CMakeLists.txt @@ -0,0 +1,8 @@ +configure_file( + ${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/dataspaces/DSpacesConfig.h.in + ${ADIOS2_BINARY_DIR}/source/adios2/toolkit/dataspaces/DSpacesConfig.h.in + ) + configure_file( + ${ADIOS2_BINARY_DIR}/source/adios2/toolkit/dataspaces/DSpacesConfig.h.in + ${ADIOS2_BINARY_DIR}/source/adios2/toolkit/dataspaces/DSpacesConfig.h + ) diff --git a/source/adios2/toolkit/dataspaces/DSpacesConfig.h.in b/source/adios2/toolkit/dataspaces/DSpacesConfig.h.in new file mode 100644 index 0000000000..e526a07d56 --- /dev/null +++ b/source/adios2/toolkit/dataspaces/DSpacesConfig.h.in @@ -0,0 +1,12 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ + +#ifndef DSPACESCONFIG_H_ +#define DSPACESCONFIG_H_ + +/* Everything past this line is programatically generated */ +#cmakedefine HAVE_DSPACES2 + +#endif /* DSPACESCONFIG_H_ */ diff --git a/source/adios2/toolkit/dataspaces/ds.h b/source/adios2/toolkit/dataspaces/ds.h index 781fa6ad7d..d8ec557591 100644 --- a/source/adios2/toolkit/dataspaces/ds.h +++ b/source/adios2/toolkit/dataspaces/ds.h @@ -16,14 +16,21 @@ extern "C" { #endif +#include "adios2/toolkit/dataspaces/DSpacesConfig.h" #include "ds_data.h" #include +#ifdef HAVE_DSPACES2 +#include +#endif /* HAVE_DSPACES2 */ typedef struct adios_ds_data_struct DsData; extern int adios_dataspaces_init(void *comm, DsData *md); extern int adios_read_dataspaces_init(void *comm, DsData *data); +#ifdef HAVE_DSPACES2 +extern dspaces_client_t *get_client_handle(); +#endif /* HAVE_DSPACES2 */ extern void adios_dataspaces_open(char *fname, DsData *md); extern void globals_adios_set_dataspaces_connected_from_reader(); extern void globals_adios_set_dataspaces_disconnected_from_reader(); diff --git a/source/adios2/toolkit/dataspaces/ds_writer.c b/source/adios2/toolkit/dataspaces/ds_writer.c index 7f63353838..6e8a29bd91 100644 --- a/source/adios2/toolkit/dataspaces/ds_writer.c +++ b/source/adios2/toolkit/dataspaces/ds_writer.c @@ -20,12 +20,23 @@ #include #include "adios2/common/ADIOSConfig.h" +#include "adios2/toolkit/dataspaces/DSpacesConfig.h" +#ifdef HAVE_DSPACES2 +#include "dspaces.h" +#else #include "dataspaces.h" +#endif /* HAVE_DSPACES2 */ #include "ds.h" #include #define MAX_DS_NAMELEN 128 +#ifdef HAVE_DSPACES2 +static dspaces_client_t client = NULL; + +dspaces_client_t *get_client_handle() { return (&client); } +#endif /* HAVE_DSPACES2 */ + static int globals_adios_appid = -1; static int globals_adios_was_set = 0; void globals_adios_set_application_id(int id) @@ -129,13 +140,21 @@ int adios_dataspaces_init(void *comm, DsData *md) "the Writer in DATASPACES\n"); } +#ifdef HAVE_DSPACES2 + err = dspaces_init(rank, &client); + if (err != dspaces_SUCCESS) + { + fprintf(stderr, "Failed to connect to the DataSpaces server.\n"); + return (err); + } +#else err = dspaces_init(nproc, md->appid, (MPI_Comm *)comm, NULL); if (err < 0) { fprintf(stderr, "Failed to connect with DATASPACES\n"); return err; } - +#endif /* HAVE_DSPACES2 */ md->rank = rank; md->mpi_comm = *(MPI_Comm *)comm; } @@ -149,8 +168,9 @@ void adios_dataspaces_open(char *fname, DsData *md) // application that we need to tell DATASPACES MPI_Comm_rank(md->mpi_comm, &(md->rank)); MPI_Comm_size(md->mpi_comm, &(md->peers)); - +#ifndef HAVE_DSPACES2 dspaces_lock_on_write(fname, &md->mpi_comm); +#endif /* HAVE_DSPACES2 */ } int adios_read_dataspaces_init(void *comm, DsData *md) @@ -169,13 +189,21 @@ int adios_read_dataspaces_init(void *comm, DsData *md) fprintf(stderr, "AppID not found in xml file. Setting it as 2 for " "the Reader in DATASPACES\n"); } +#ifdef HAVE_DSPACES2 + err = dspaces_init(rank, &client); + if (err != dspaces_SUCCESS) + { + fprintf(stderr, "Failed to connect to the DataSpaces server.\n"); + return (err); + } +#else err = dspaces_init(nproc, md->appid, (MPI_Comm *)comm, NULL); if (err < 0) { fprintf(stderr, "Failed to connect with DATASPACES\n"); return err; } - +#endif /* HAVE_DSPACES2 */ md->rank = rank; md->mpi_comm = *(MPI_Comm *)comm; } From 22bed5658c3e307848f45e8fe3ce84683f76357f Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Thu, 4 Mar 2021 16:37:54 +0000 Subject: [PATCH 2/4] Added zero-sided metadata update to close file --- .../adios2/engine/dataspaces/DataSpacesReader.cpp | 14 ++++---------- .../adios2/engine/dataspaces/DataSpacesWriter.cpp | 13 ++++++++++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.cpp b/source/adios2/engine/dataspaces/DataSpacesReader.cpp index 0d90f253e1..dabb3bcd1e 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesReader.cpp @@ -100,15 +100,12 @@ StepStatus DataSpacesReader::BeginStep(StepMode mode, const float timeout_sec) #ifdef HAVE_DSPACES2 dspaces_client_t *client = get_client_handle(); char meta_str[256]; - sprintf(meta_str, "VARMETA@%s", fstr); unsigned int metalen; - int var_name_max_length = 128; - int var_size = 2 * sizeof(int) + MAX_DS_NDIM * sizeof(uint64_t) + - var_name_max_length * sizeof(char); + sprintf(meta_str, "VARMETA@%s", fstr); int err = dspaces_get_meta(*client, meta_str, META_MODE_NEXT, m_CurrentStep, &bcast_array[1], (void **)&buffer, &metalen); - bcast_array[0] = metalen / var_size; + bcast_array[0] = metalen; #else buffer = dspaces_get_next_meta(m_CurrentStep, fstr, &bcast_array[0], &bcast_array[1]); @@ -122,15 +119,12 @@ StepStatus DataSpacesReader::BeginStep(StepMode mode, const float timeout_sec) #ifdef HAVE_DSPACES2 dspaces_client_t *client = get_client_handle(); char meta_str[256]; - sprintf(meta_str, "VARMETA@%s", fstr); unsigned int metalen; - int var_name_max_length = 128; - int var_size = 2 * sizeof(int) + MAX_DS_NDIM * sizeof(uint64_t) + - var_name_max_length * sizeof(char); + sprintf(meta_str, "VARMETA@%s", fstr); int err = dspaces_get_meta(*client, meta_str, META_MODE_LAST, m_CurrentStep, &bcast_array[1], (void **)&buffer, &metalen); - bcast_array[0] = metalen / var_size; + bcast_array[0] = metalen; #else buffer = dspaces_get_latest_meta(m_CurrentStep, fstr, &bcast_array[0], &bcast_array[1]); diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp index 4c81caba70..346dcb171f 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp @@ -96,7 +96,18 @@ void DataSpacesWriter::DoClose(const int transportIndex) char *meta_lk = new char[local_file_var.length() + 1]; strcpy(meta_lk, local_file_var.c_str()); -#ifndef HAVE_DSPACES2 +#ifdef HAVE_DSPACES2 + int rank; + MPI_Comm_rank(m_data.mpi_comm, &rank); + if(rank == 0) { + local_file_var = "VARMETA@" + f_Name; + char *local_str = new char[local_file_var.length() + 1]; + strcpy(local_str, local_file_var.c_str()); + dspaces_client_t *client = get_client_handle(); + dspaces_put_meta(*client, local_str, m_CurrentStep+1, NULL, 0); + delete[] local_str; + } +#else dspaces_lock_on_write(meta_lk, &(m_data.mpi_comm)); dspaces_unlock_on_write(meta_lk, &(m_data.mpi_comm)); #endif /* HAVE_DSPACES2 */ From ca6214c7a0b7a08e6ac5242ce9bc2d15b1fb6a58 Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Thu, 4 Mar 2021 23:27:11 +0000 Subject: [PATCH 3/4] Send kill to server on engine close --- .../engine/dataspaces/DataSpacesReader.cpp | 1 - .../engine/dataspaces/DataSpacesReader.tcc | 6 ++++++ .../engine/dataspaces/DataSpacesWriter.cpp | 8 ++++---- source/adios2/toolkit/dataspaces/ds_writer.c | 19 +++++++++++++++++++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.cpp b/source/adios2/engine/dataspaces/DataSpacesReader.cpp index dabb3bcd1e..f2c066f10b 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesReader.cpp @@ -264,7 +264,6 @@ void DataSpacesReader::EndStep() void DataSpacesReader::DoClose(const int transportIndex) { - globals_adios_set_dataspaces_disconnected_from_reader(); } diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.tcc b/source/adios2/engine/dataspaces/DataSpacesReader.tcc index 9f2cae04aa..6c589dde2a 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.tcc +++ b/source/adios2/engine/dataspaces/DataSpacesReader.tcc @@ -74,6 +74,8 @@ void DataSpacesReader::ReadDsData(Variable &variable, T *data, int version) { #ifndef HAVE_DSPACES2 gdims_in[0] = dspaces_get_num_space_server(); +#else + gdims_in[0] = 1; #endif lb_in[0] = 0; ub_in[0] = 0; @@ -116,6 +118,10 @@ void DataSpacesReader::ReadDsData(Variable &variable, T *data, int version) char *cstr = new char[l_Name.length() + 1]; strcpy(cstr, l_Name.c_str()); #ifdef HAVE_DSPACES2 + dspaces_client_t *client = get_client_handle(); + dspaces_define_gdim(*client, var_str, ndims, gdims_in); + dspaces_get(*client, var_str, version, variable.m_ElementSize, ndims, lb_in, + ub_in, (void *)data, -1); #else dspaces_define_gdim(var_str, ndims, gdims_in); dspaces_get(var_str, version, variable.m_ElementSize, ndims, lb_in, ub_in, diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp index 346dcb171f..ee5cb9a41d 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp @@ -99,19 +99,20 @@ void DataSpacesWriter::DoClose(const int transportIndex) #ifdef HAVE_DSPACES2 int rank; MPI_Comm_rank(m_data.mpi_comm, &rank); - if(rank == 0) { + if (rank == 0) + { local_file_var = "VARMETA@" + f_Name; char *local_str = new char[local_file_var.length() + 1]; strcpy(local_str, local_file_var.c_str()); dspaces_client_t *client = get_client_handle(); - dspaces_put_meta(*client, local_str, m_CurrentStep+1, NULL, 0); + dspaces_put_meta(*client, local_str, m_CurrentStep + 1, NULL, 0); delete[] local_str; } + MPI_Barrier(m_data.mpi_comm); #else dspaces_lock_on_write(meta_lk, &(m_data.mpi_comm)); dspaces_unlock_on_write(meta_lk, &(m_data.mpi_comm)); #endif /* HAVE_DSPACES2 */ - globals_adios_set_dataspaces_disconnected_from_writer(); } @@ -143,7 +144,6 @@ void DataSpacesWriter::WriteVarInfo() if (rank == 0) { - std::string ds_file_var; int var_num = ndim_vector.size(); int var_name_max_length = 128; diff --git a/source/adios2/toolkit/dataspaces/ds_writer.c b/source/adios2/toolkit/dataspaces/ds_writer.c index 6e8a29bd91..be891fbb33 100644 --- a/source/adios2/toolkit/dataspaces/ds_writer.c +++ b/source/adios2/toolkit/dataspaces/ds_writer.c @@ -37,6 +37,7 @@ static dspaces_client_t client = NULL; dspaces_client_t *get_client_handle() { return (&client); } #endif /* HAVE_DSPACES2 */ +static int globals_rank; static int globals_adios_appid = -1; static int globals_adios_was_set = 0; void globals_adios_set_application_id(int id) @@ -74,7 +75,15 @@ void globals_adios_set_dataspaces_disconnected_from_reader() { if (globals_adios_connected_to_dataspaces == dataspaces_connected_from_reader) + { globals_adios_connected_to_dataspaces = dataspaces_disconnected; +#ifdef HAVE_DSPACES2 + if (globals_rank == 0) + { + dspaces_kill(client); + } +#endif + } else if (globals_adios_connected_to_dataspaces == dataspaces_connected_from_both) globals_adios_connected_to_dataspaces = @@ -93,7 +102,15 @@ void globals_adios_set_dataspaces_disconnected_from_writer() { if (globals_adios_connected_to_dataspaces == dataspaces_connected_from_writer) + { globals_adios_connected_to_dataspaces = dataspaces_disconnected; +#ifdef HAVE_DSPACES2 + if (globals_rank == 0) + { + dspaces_kill(client); + } +#endif + } else if (globals_adios_connected_to_dataspaces == dataspaces_connected_from_both) globals_adios_connected_to_dataspaces = @@ -132,6 +149,7 @@ int adios_dataspaces_init(void *comm, DsData *md) MPI_Comm_rank(*(MPI_Comm *)comm, &rank); MPI_Comm_size(*(MPI_Comm *)comm, &nproc); + globals_rank = rank; if (md->appid == 0) { @@ -182,6 +200,7 @@ int adios_read_dataspaces_init(void *comm, DsData *md) MPI_Comm_rank(*(MPI_Comm *)comm, &rank); MPI_Comm_size(*(MPI_Comm *)comm, &nproc); + globals_rank = rank; if (md->appid == 0) { From 1c487e26e1cd4bc034e7f09625554de7c653c158 Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Fri, 5 Mar 2021 14:22:03 +0000 Subject: [PATCH 4/4] Bumped DataSpaces dependency to 2.1.1 --- cmake/DetectOptions.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 3b8652d2db..d720bf711a 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -216,9 +216,9 @@ endif() # DataSpaces if(ADIOS2_USE_DataSpaces STREQUAL AUTO) - find_package(DataSpaces 1.8) + find_package(DataSpaces 2.1.1) elseif(ADIOS2_USE_DataSpaces) - find_package(DataSpaces 1.8 REQUIRED) + find_package(DataSpaces 2.1.1 REQUIRED) endif() if(DATASPACES_FOUND) set(ADIOS2_HAVE_DataSpaces TRUE)