Skip to content

Commit

Permalink
Merge pull request #2643 from philip-davis/dataspaces2
Browse files Browse the repository at this point in the history
Update DataSpaces engine to support DataSpaces2
  • Loading branch information
philip-davis authored Mar 26, 2021
2 parents e4146e8 + 1c487e2 commit 528c799
Show file tree
Hide file tree
Showing 12 changed files with 221 additions and 36 deletions.
4 changes: 2 additions & 2 deletions cmake/DetectOptions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -216,9 +216,9 @@ endif()

# DataSpaces
if(ADIOS2_USE_DataSpaces STREQUAL AUTO)
find_package(DataSpaces 1.8)
find_package(DataSpaces 2.1.1)
elseif(ADIOS2_USE_DataSpaces)
find_package(DataSpaces 1.8 REQUIRED)
find_package(DataSpaces 2.1.1 REQUIRED)
endif()
if(DATASPACES_FOUND)
set(ADIOS2_HAVE_DataSpaces TRUE)
Expand Down
63 changes: 43 additions & 20 deletions cmake/FindDataSpaces.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,33 @@ if(NOT DATASPACES_FOUND)
get_filename_component(DATASPACES_ROOT "${DSPACES_CONF}/../.." ABSOLUTE)
endif()
endif()
if(DATASPACES_ROOT)
find_package(PkgConfig)
if(PKG_CONFIG_FOUND)
set(_CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH})
if(DATASPACES_ROOT)
list(INSERT CMAKE_PREFIX_PATH 0 "${DATASPACES_ROOT}")
elseif(NOT ENV{DATASPACES_ROOT} STREQUAL "")
list(INSERT CMAKE_PREFIX_PATH 0 "$ENV{DATASPACES_ROOT}")
endif()
set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON)
pkg_check_modules(PC_DSPACES dspaces)
set(CMAKE_PREFIX_PATH ${_CMAKE_PREFIX_PATH})
unset(_CMAKE_PREFIX_PATH)
if(PC_DSPACES_FOUND)
if(BUILD_SHARED_LIBS)
set(_PC_TYPE)
else()
set(_PC_TYPE _STATIC)
endif()
set(DATASPACES_LIBRARIES ${PC_DSPACES${_PC_TYPE}_LINK_LIBRARIES})
set(DATASPACES_LIBRARY_HINT ${PC_DSPACES${_PC_TYPE}_LIBRARY_DIRS})
set(DATASPACES_INCLUDE_DIR ${PC_DSPACES${_PC_TYPE}_INCLUDE_DIRS})
set(DATASPACES_VERSION ${PC_DSPACES_VERSION})
find_library(DATASPACES_LIBRARY dspaces HINTS ${DATASPACES_LIBRARY_HINT})
set(HAVE_DSPACES2 TRUE)
endif()
endif()
if(DATASPACES_ROOT AND NOT PC_DSPACES_FOUND)
find_program(DSPACES_CONF dspaces_config ${DATASPACES_ROOT}/bin)
if(DSPACES_CONF)
execute_process(COMMAND ${DSPACES_CONF} -l
Expand All @@ -50,50 +76,47 @@ if(NOT DATASPACES_FOUND)
foreach(LOOP_VAR ${LINK_LIBS})
STRING(FIND ${LOOP_VAR} "-u" DEL_FLG)
if(("${DEL_FLG}" EQUAL "-1"))
STRING(FIND ${LOOP_VAR} "/" HINT_FLG)
if(NOT("${HINT_FLG}" EQUAL "-1"))
STRING(FIND ${LOOP_VAR} "/" HINT_FLG)
if(NOT("${HINT_FLG}" EQUAL "-1"))
list(APPEND DATASPACES_LIBRARY_HINT ${LOOP_VAR})
else()
else()
unset(LOCAL_LIBRARY CACHE)
unset(LOCAL_LIBRARY-NOTFOUND CACHE)
STRING(FIND ${LOOP_VAR} "stdc++" CPP_FLG)
if("${CPP_FLG}" EQUAL "-1")
find_library(LOCAL_LIBRARY NAMES "${LOOP_VAR}" HINTS ${DATASPACES_LIBRARY_HINT})
if(LOCAL_LIBRARY)
find_library(LOCAL_LIBRARY NAMES "${LOOP_VAR}" HINTS ${DATASPACES_LIBRARY_HINT})
if(LOCAL_LIBRARY)
list(APPEND DATASPACES_LIBRARIES ${LOCAL_LIBRARY})
else()
else()
list(APPEND DATASPACES_LIBRARIES ${LOOP_VAR})
endif()
endif()
endif()
endif()
endif()
endforeach()
execute_process(COMMAND ${DSPACES_CONF} -v
endif()
endif()
endforeach()
execute_process(COMMAND ${DSPACES_CONF} -v
RESULT_VARIABLE RESULT_VAR
OUTPUT_VARIABLE DATASPACES_VERSION
ERROR_QUIET
OUTPUT_STRIP_TRAILING_WHITESPACE)
endif ()

endif()
set(DATASPACES_INCLUDE_OPTS HINTS ${DATASPACES_ROOT}/include)

find_path(DATASPACES_INCLUDE_DIR dataspaces.h ${DATASPACES_INCLUDE_OPTS})
find_library(DATASPACES_LIBRARY dspaces HINTS ${DATASPACES_LIBRARY_HINT})
endif()

find_path(DATASPACES_INCLUDE_DIR dataspaces.h ${DATASPACES_INCLUDE_OPTS})
find_library(DSPACES_LIBRARY dspaces HINTS ${DATASPACES_LIBRARY_HINT})

include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(DataSpaces
FOUND_VAR DATASPACES_FOUND
VERSION_VAR DATASPACES_VERSION
REQUIRED_VARS DATASPACES_VERSION DATASPACES_INCLUDE_DIR
DATASPACES_LIBRARIES DSPACES_LIBRARY
DATASPACES_LIBRARIES #DATASPACES_LIBRARY
)
if(DATASPACES_FOUND)
if(DATASPACES_FOUND AND NOT TARGET DataSpaces::DataSpaces)
add_library(DataSpaces::DataSpaces UNKNOWN IMPORTED)
set_target_properties(DataSpaces::DataSpaces PROPERTIES
IMPORTED_LOCATION "${DSPACES_LIBRARY}"
IMPORTED_LOCATION "${DATASPACES_LIBRARY}"
INTERFACE_LINK_LIBRARIES "${DATASPACES_LIBRARIES}"
INTERFACE_INCLUDE_DIRECTORIES "${DATASPACES_INCLUDE_DIR}"
)
Expand Down
4 changes: 2 additions & 2 deletions cmake/FindLIBFABRIC.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ if(NOT (PC_LIBFABRIC_FOUND STREQUAL "IGNORE"))
if(BUILD_SHARED_LIBS)
set(_PC_TYPE)
else()
set(_PF_TYPE _STATIC)
set(_PC_TYPE _STATIC)
endif()
set(LIBFABRIC_INCLUDE_DIRS ${PC_LIBFABRIC${_PC_TYPE}_INCLUDE_DIRS})
set(LIBFABRIC_LIBRARIES ${PC_LIBFABRIC${_PC_TYPE}_LINK_LIBRARIES})
set(LIBFABRIC_DEFINITIONS ${PC_LIBFABRIC${PC_TYPE}_CFLAGS_OTHER})
set(LIBFABRIC_DEFINITIONS ${PC_LIBFABRIC${_PC_TYPE}_CFLAGS_OTHER})
endif()
endif()
endif()
Expand Down
1 change: 1 addition & 0 deletions source/adios2/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ if(ADIOS2_HAVE_SST)
endif()

if(ADIOS2_HAVE_DataSpaces)
add_subdirectory(toolkit/dataspaces)
target_sources(adios2_core_mpi PRIVATE
toolkit/dataspaces/ds_writer.c
engine/dataspaces/DataSpacesWriter.cpp engine/dataspaces/DataSpacesWriter.tcc
Expand Down
32 changes: 31 additions & 1 deletion source/adios2/engine/dataspaces/DataSpacesReader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,13 @@
#include "DataSpacesReader.tcc"
#include "adios2/helper/adiosCommMPI.h"
#include "adios2/helper/adiosFunctions.h" //CSVToVector
#include "adios2/toolkit/dataspaces/DSpacesConfig.h"
#include "adios2/toolkit/dataspaces/ds_data.h"
#ifdef HAVE_DSPACES2
#include "dspaces.h"
#else
#include "dataspaces.h"
#endif /* HAVE_DSPACES2 */

namespace adios2
{
Expand Down Expand Up @@ -83,23 +88,47 @@ StepStatus DataSpacesReader::BeginStep(StepMode mode, const float timeout_sec)
strcpy(meta_lk, lk_name.c_str());

MPI_Comm lock_comm = m_data.mpi_comm;
#ifndef HAVE_DSPACES2
dspaces_lock_on_read(meta_lk, &lock_comm);
#endif /* HAVE_DSPACES2 */

int nVars = 0;
if (!m_ProvideLatest)
{
if (rank == 0)
{
#ifdef HAVE_DSPACES2
dspaces_client_t *client = get_client_handle();
char meta_str[256];
unsigned int metalen;
sprintf(meta_str, "VARMETA@%s", fstr);
int err = dspaces_get_meta(*client, meta_str, META_MODE_NEXT,
m_CurrentStep, &bcast_array[1],
(void **)&buffer, &metalen);
bcast_array[0] = metalen;
#else
buffer = dspaces_get_next_meta(m_CurrentStep, fstr, &bcast_array[0],
&bcast_array[1]);
#endif /* HAVE_DSPACES2 */
}
}
else
{
if (rank == 0)
{
#ifdef HAVE_DSPACES2
dspaces_client_t *client = get_client_handle();
char meta_str[256];
unsigned int metalen;
sprintf(meta_str, "VARMETA@%s", fstr);
int err = dspaces_get_meta(*client, meta_str, META_MODE_LAST,
m_CurrentStep, &bcast_array[1],
(void **)&buffer, &metalen);
bcast_array[0] = metalen;
#else
buffer = dspaces_get_latest_meta(m_CurrentStep, fstr,
&bcast_array[0], &bcast_array[1]);
#endif /* HAVE_DSPACES2 */
}
}
MPI_Bcast(bcast_array, 2, MPI_INT, 0, m_data.mpi_comm);
Expand Down Expand Up @@ -228,12 +257,13 @@ void DataSpacesReader::EndStep()
strcpy(meta_lk, lk_name.c_str());

MPI_Comm lock_comm = m_data.mpi_comm;
#ifndef HAVE_DSPACES2
dspaces_unlock_on_read(meta_lk, &lock_comm);
#endif /* HAVE_DSPACES2 */
}

void DataSpacesReader::DoClose(const int transportIndex)
{

globals_adios_set_dataspaces_disconnected_from_reader();
}

Expand Down
18 changes: 16 additions & 2 deletions source/adios2/engine/dataspaces/DataSpacesReader.tcc
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,12 @@

#include "DataSpacesReader.h"
#include "adios2/helper/adiosFunctions.h" //CSVToVector
#include "adios2/toolkit/dataspaces/DSpacesConfig.h"
#ifdef HAVE_DSPACES2
#include "dspaces.h"
#else
#include "dataspaces.h"
#endif /* HAVE_DSPACES2 */

namespace adios2
{
Expand Down Expand Up @@ -67,7 +72,11 @@ void DataSpacesReader::ReadDsData(Variable<T> &variable, T *data, int version)
*/
if (variable.m_Shape.size() == 0)
{
#ifndef HAVE_DSPACES2
gdims_in[0] = dspaces_get_num_space_server();
#else
gdims_in[0] = 1;
#endif
lb_in[0] = 0;
ub_in[0] = 0;
ndims = 1;
Expand Down Expand Up @@ -108,11 +117,16 @@ void DataSpacesReader::ReadDsData(Variable<T> &variable, T *data, int version)
std::string l_Name = ds_in_name + std::to_string(version);
char *cstr = new char[l_Name.length() + 1];
strcpy(cstr, l_Name.c_str());

#ifdef HAVE_DSPACES2
dspaces_client_t *client = get_client_handle();
dspaces_define_gdim(*client, var_str, ndims, gdims_in);
dspaces_get(*client, var_str, version, variable.m_ElementSize, ndims, lb_in,
ub_in, (void *)data, -1);
#else
dspaces_define_gdim(var_str, ndims, gdims_in);
dspaces_get(var_str, version, variable.m_ElementSize, ndims, lb_in, ub_in,
(void *)data);

#endif /* HAVE_DSPACES2 */
delete[] cstr;
delete[] var_str;
}
Expand Down
38 changes: 32 additions & 6 deletions source/adios2/engine/dataspaces/DataSpacesWriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,13 @@
#include "DataSpacesWriter.tcc"
#include "adios2/helper/adiosCommMPI.h"
#include "adios2/helper/adiosFunctions.h" //CSVToVector
#include "adios2/toolkit/dataspaces/DSpacesConfig.h"
#include "adios2/toolkit/dataspaces/ds_data.h"
#ifdef HAVE_DSPACES2
#include "dspaces.h"
#else
#include "dataspaces.h"
#endif /* HAVE_DSPACES2 */

namespace adios2
{
Expand Down Expand Up @@ -73,10 +78,13 @@ void DataSpacesWriter::EndStep()
char *meta_lk = new char[local_file_var.length() + 1];
strcpy(meta_lk, local_file_var.c_str());
MPI_Comm lock_comm = m_data.mpi_comm;

#ifndef HAVE_DSPACES2
dspaces_lock_on_write(meta_lk, &lock_comm);
#endif /* HAVE_DSPACES2 */
WriteVarInfo();
#ifndef HAVE_DSPACES2
dspaces_unlock_on_write(meta_lk, &lock_comm);
#endif /* HAVE_DSPACES2 */
}
void DataSpacesWriter::Flush(const int transportIndex) {}

Expand All @@ -88,9 +96,23 @@ void DataSpacesWriter::DoClose(const int transportIndex)
char *meta_lk = new char[local_file_var.length() + 1];
strcpy(meta_lk, local_file_var.c_str());

#ifdef HAVE_DSPACES2
int rank;
MPI_Comm_rank(m_data.mpi_comm, &rank);
if (rank == 0)
{
local_file_var = "VARMETA@" + f_Name;
char *local_str = new char[local_file_var.length() + 1];
strcpy(local_str, local_file_var.c_str());
dspaces_client_t *client = get_client_handle();
dspaces_put_meta(*client, local_str, m_CurrentStep + 1, NULL, 0);
delete[] local_str;
}
MPI_Barrier(m_data.mpi_comm);
#else
dspaces_lock_on_write(meta_lk, &(m_data.mpi_comm));
dspaces_unlock_on_write(meta_lk, &(m_data.mpi_comm));

#endif /* HAVE_DSPACES2 */
globals_adios_set_dataspaces_disconnected_from_writer();
}

Expand Down Expand Up @@ -122,13 +144,12 @@ void DataSpacesWriter::WriteVarInfo()

if (rank == 0)
{

std::string ds_file_var;
int var_num = ndim_vector.size();
int var_name_max_length = 128;
int buf_len = var_num * sizeof(int) + var_num * sizeof(int) +
MAX_DS_NDIM * var_num * sizeof(uint64_t) +
var_num * var_name_max_length * sizeof(char);
int buf_len =
var_num * (2 * sizeof(int) + MAX_DS_NDIM * sizeof(uint64_t) +
var_name_max_length * sizeof(char));
int *dim_meta, *elemSize_meta;
uint64_t *gdim_meta;
dim_meta = (int *)malloc(var_num * sizeof(int));
Expand Down Expand Up @@ -174,12 +195,17 @@ void DataSpacesWriter::WriteVarInfo()
ndim = 1;
lb[0] = 0;
ub[0] = buf_len - 1;
#ifdef HAVE_DSPACES2
dspaces_client_t *client = get_client_handle();
dspaces_put_meta(*client, local_str, m_CurrentStep, buffer, buf_len);
#else
gdims[0] = (ub[0] - lb[0] + 1) * dspaces_get_num_space_server();
dspaces_define_gdim(local_str, ndim, gdims);

dspaces_put(local_str, m_CurrentStep, elemsize, ndim, lb, ub, buffer);

dspaces_put_sync(); // wait on previous put to finish
#endif /* HAVE_DSPACES2 */
delete[] local_str;
free(dim_meta);
free(elemSize_meta);
Expand Down
17 changes: 17 additions & 0 deletions source/adios2/engine/dataspaces/DataSpacesWriter.tcc
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,14 @@

#include "DataSpacesWriter.h"
#include "adios2/helper/adiosFunctions.h"
#include "adios2/toolkit/dataspaces/DSpacesConfig.h"
#include "adios2/toolkit/dataspaces/ds.h"
#include "adios2/toolkit/dataspaces/ds_data.h"
#ifdef HAVE_DSPACES2
#include "dspaces.h"
#else
#include "dataspaces.h"
#endif /* HAVE_DSPACES2 */

namespace adios2
{
Expand Down Expand Up @@ -48,7 +54,11 @@ void DataSpacesWriter::DoPutSyncCommon(Variable<T> &variable, const T *values)
*/
if (variable.m_SingleValue)
{
#ifdef HAVE_DSPACES2
gdims_in[0] = 1;
#else
gdims_in[0] = dspaces_get_num_space_server();
#endif /* HAVE_DSPACES2 */
lb_in[0] = 0;
ub_in[0] = 0;
ndims = 1;
Expand Down Expand Up @@ -110,11 +120,18 @@ void DataSpacesWriter::DoPutSyncCommon(Variable<T> &variable, const T *values)
char *cstr = new char[l_Name.length() + 1];
strcpy(cstr, l_Name.c_str());

#ifdef HAVE_DSPACES2
dspaces_client_t *client = get_client_handle();
dspaces_define_gdim(*client, var_str, ndims, gdims_in);
dspaces_put(*client, var_str, version, variable.m_ElementSize, ndims, lb_in,
ub_in, values);
#else
dspaces_define_gdim(var_str, ndims, gdims_in);
dspaces_put(var_str, version, variable.m_ElementSize, ndims, lb_in, ub_in,
values);
dspaces_put_sync();
dspaces_put_sync();
#endif /* HAVE_DSPACES2 */
delete[] cstr;
delete[] var_str;
}
Expand Down
Loading

0 comments on commit 528c799

Please sign in to comment.