diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index c9b5e4a5ee10..de6f00db8674 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -27,6 +27,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 76df09cfd93b..6bb8c9b31704 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -27,6 +27,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 144c17e7820d..c041a86cff97 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -27,6 +27,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 1df2a9fd23b3..d391acb86524 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -27,6 +27,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-16.sh b/.jenkins/lsu/env-clang-16.sh index 393e42c03e54..c489ea51e204 100644 --- a/.jenkins/lsu/env-clang-16.sh +++ b/.jenkins/lsu/env-clang-16.sh @@ -27,6 +27,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-17.sh b/.jenkins/lsu/env-clang-17.sh index 1b6ac8d26294..3381bbe651e5 100644 --- a/.jenkins/lsu/env-clang-17.sh +++ b/.jenkins/lsu/env-clang-17.sh @@ -27,6 +27,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 9c4064825a0a..e93f5087821f 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -27,6 +27,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 48a6887c1ba6..7c40c3fd3274 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -26,6 +26,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 4a10458aa535..3861f5585fb6 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -26,6 +26,11 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c934e21328b..9d3bdd5bb402 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1191,6 +1191,26 @@ if(HPX_WITH_NETWORKING) ) endif() + hpx_option( + HPX_WITH_PARCELPORT_OPENSHMEM BOOL "Enable the OPENSHMEM based parcelport." + OFF CATEGORY "Parcelport" + ) + hpx_option( + HPX_WITH_FETCH_OPENSHMEM + BOOL + "Download an OpenSHMEM if one is not available" OFF + CATEGORY "Build Targets" + ADVANCED + ) + if(HPX_WITH_PARCELPORT_OPENSHMEM) + hpx_add_config_define(HPX_HAVE_PARCELPORT_OPENSHMEM) + hpx_option( + HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT STRING + "Define which conduit to use for the OpenSHMEM parcelport" "sos" + STRINGS "sos;ucx;mpi" + ) + endif() + hpx_option( HPX_WITH_PARCELPORT_TCP BOOL "Enable the TCP based parcelport." ON CATEGORY "Parcelport" @@ -2197,6 +2217,10 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) include(HPX_SetupGasnet) hpx_setup_gasnet() endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() +endif() # Setup packages and subprojects include(HPX_SetupSYCL) diff --git a/cmake/FindOpenShmemPmi.cmake b/cmake/FindOpenShmemPmi.cmake new file mode 100644 index 000000000000..5f6814a509db --- /dev/null +++ b/cmake/FindOpenShmemPmi.cmake @@ -0,0 +1,65 @@ +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +find_package(PkgConfig QUIET) +# look for cray pmi... +pkg_check_modules(PC_PMI_CRAY QUIET cray-pmi) +# look for the rest if we couldn't find the cray package +if(NOT PC_PMI_CRAY_FOUND) + pkg_check_modules(PC_PMI QUIET pmi) +endif() + +find_path( + PMI_INCLUDE_DIR pmi2.h + HINTS ${PMI_ROOT} + ENV + PMI_ROOT + ${PMI_DIR} + ENV + PMI_DIR + ${PC_PMI_CRAY_INCLUDEDIR} + ${PC_PMI_CRAY_INCLUDE_DIRS} + ${PC_PMI_INCLUDEDIR} + ${PC_PMI_INCLUDE_DIRS} + PATH_SUFFIXES include +) + +find_library( + PMI_LIBRARY + NAMES pmi + HINTS ${PMI_ROOT} + ENV + PMI_ROOT + ${PC_PMI_CRAY_LIBDIR} + ${PC_PMI_CRAY_LIBRARY_DIRS} + ${PC_PMI_LIBDIR} + ${PC_PMI_LIBRARY_DIRS} + PATH_SUFFIXES lib lib64 +) + +# Set PMI_ROOT in case the other hints are used +if(PMI_ROOT) + # The call to file is for compatibility with windows paths + file(TO_CMAKE_PATH ${PMI_ROOT} PMI_ROOT) +elseif("$ENV{PMI_ROOT}") + file(TO_CMAKE_PATH $ENV{PMI_ROOT} PMI_ROOT) +else() + file(TO_CMAKE_PATH "${PMI_INCLUDE_DIR}" PMI_INCLUDE_DIR) + string(REPLACE "/include" "" PMI_ROOT "${PMI_INCLUDE_DIR}") +endif() + +if(NOT PMI_LIBRARY OR NOT PMI_INCLUDE_DIR) + set(PMI_FOUND=OFF) + return() +endif() + +# hpx_error( "PMI_LIBRARY OR PMI_INCLUDE_DIR not found, please install PMI or +# set \ the right PMI_ROOT path" ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(PMI DEFAULT_MSG PMI_LIBRARY PMI_INCLUDE_DIR) + +mark_as_advanced(PMI_ROOT PMI_LIBRARY PMI_INCLUDE_DIR) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index cd90dc0d0972..1fd012651b20 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -5,8 +5,14 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) function(add_hpx_test category name) - set(options FAILURE_EXPECTED RUN_SERIAL NO_PARCELPORT_TCP NO_PARCELPORT_MPI - NO_PARCELPORT_LCI NO_PARCELPORT_GASNET + set(options + FAILURE_EXPECTED + RUN_SERIAL + NO_PARCELPORT_TCP + NO_PARCELPORT_MPI + NO_PARCELPORT_LCI + NO_PARCELPORT_GASNET + NO_PARCELPORT_OPENSHMEM ) set(one_value_args EXECUTABLE LOCALITIES THREADS_PER_LOCALITY TIMEOUT RUNWRAPPER @@ -196,7 +202,7 @@ function(add_hpx_test category name) if(_add_test) set(_full_name "${category}.distributed.gasnet.${name}") add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" - "gasnet-smp" ${args} + "amudprun" ${args} ) set_tests_properties( "${_full_name}" @@ -211,6 +217,29 @@ function(add_hpx_test category name) endif() endif() endif() + if(HPX_WITH_PARCELPORT_OPENSHMEM AND NOT ${name}_NO_PARCELPORT_OPENSHMEM) + set(_add_test FALSE) + if(DEFINED ${name}_PARCELPORTS) + set(PP_FOUND -1) + list(FIND ${name}_PARCELPORTS "openshmem" PP_FOUND) + if(NOT PP_FOUND EQUAL -1) + set(_add_test TRUE) + endif() + else() + set(_add_test TRUE) + endif() + if(_add_test) + set(_full_name "${category}.distributed.openshmem.${name}") + add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "openshmem" "-r" + "oshrun" ${args} + ) + if(${name}_TIMEOUT) + set_tests_properties( + "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} + ) + endif() + endif() + endif() if(HPX_WITH_PARCELPORT_TCP AND NOT ${${name}_NO_PARCELPORT_TCP}) set(_add_test FALSE) if(DEFINED ${name}_PARCELPORTS) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 61690c08ddae..9c961dde464c 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -119,7 +119,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-udp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-udp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -129,7 +129,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-smp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-smp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -139,7 +139,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-ofi --with-ofi-home=${OFI_DIR} && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -149,7 +149,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-ucx --with-ucx-home=${UCX_DIR} && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -185,7 +185,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --with-mpi-libs=${MPI_C_LIBRARIES} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=\"-fPIC\" CCFLAGS=\"-fPIC\" CXXFLAGS=\"-fPIC\" ./configure --prefix=${GASNET_DIR}/install --with-cflags=\"-fPIC\" --with-cxxflags=\"-fPIC\" --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --with-mpi-libs=${MPI_C_LIBRARIES} && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake new file mode 100644 index 000000000000..244be1879a1b --- /dev/null +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -0,0 +1,1153 @@ +# Copyright (c) 2019-2023 Ste||ar Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) + +macro(hpx_setup_openshmem) + + if(NOT TARGET PkgConfig::OPENSHMEM) + + set(OPENSHMEM_PC "") + if(HPX_WITH_PARCELPORT_MPI + AND HPX_WITH_PARCELPORT_OPENSHMEM + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) + message( + FATAL + "HPX_WITH_PARCELPORT_MPI & HPX_WITH_PARCELPORT_OPENSHMEM both set to ON and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT != 'mpi'" + ) + + elseif( + HPX_WITH_PARCELPORT_MPI + AND HPX_WITH_PARCELPORT_OPENSHMEM + AND HPX_WITH_FETCH_OPENSHMEM + AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + ) + message( + FATAL_ERROR + "HPX_WITH_FETCH_OPENSHMEM for HPX_WITH_PARCELPORT_OPENSHMEM and HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT='mpi' not supported" + ) + + elseif( + HPX_WITH_PARCELPORT_OPENSHMEM + AND (NOT HPX_WITH_FETCH_OPENSHMEM) + AND ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + ) + set(ENV{PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}:${MPI_LIBDIR}/pkgconfig") + + set(OPENSHMEM_PC "oshmem") + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) + + if(NOT OPENSHMEM_FOUND) + find_program(OSHMEM_INFO NAMES oshmem_info ompi_info REQUIRED) + + if(NOT OSHMEM_INFO) + message( + FATAL_ERROR + "oshmem_info and/or ompi_info not found! pkg-config cannot find OpenMPI's `${OPENSHMEM_PC}.pc`" + ) + endif() + + set(OSHMEM_INFO_OUTPUT + "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_stdout.log" + ) + set(OSHMEM_INFO_ERROR + "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_error.log" + ) + + execute_process( + COMMAND bash -c "${OSHMEM_INFO} --path libdir" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE OSHMEM_INFO_STATUS + OUTPUT_FILE ${OSHMEM_INFO_OUTPUT} + ERROR_FILE ${OSHMEM_INFO_ERROR} + ) + + if(OSHMEM_INFO_STATUS) + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Program status code: ${OSHMEM_INFO_STATUS}" + ) + endif() + + file(READ ${OSHMEM_INFO_OUTPUT} OSHMEM_INFO_OUTPUT_CONTENT) + + if(NOT DEFINED OSHMEM_INFO_OUTPUT_CONTENT) + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}\n${OSHMEM_INFO_OUTPUT_CONTENT}" + ) + endif() + + if("${OSHMEM_INFO_OUTPUT_CONTENT}" STREQUAL "") + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_ERROR}\n${OSHMEM_INFO_OUTPUT_CONTENT}" + ) + endif() + + string(REGEX MATCH "(\/.*)" OSHMEM_LIBDIR_PATH + ${OSHMEM_INFO_OUTPUT_CONTENT} + ) + + string(STRIP ${OSHMEM_LIBDIR_PATH} OSHMEM_LIBDIR_PATH) + + set(ENV{PKG_CONFIG_PATH} + "$ENV{PKG_CONFIG_PATH}:${OSHMEM_LIBDIR_PATH}/pkgconfig" + ) + + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) + + if(NOT OPENSHMEM_FOUND) + + set(OSHMEM_INFO_INCOUTPUT + "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_stdout_inc.log" + ) + set(OSHMEM_INFO_INCERROR + "${CMAKE_CURRENT_SOURCE_DIR}/oshmem_info_error_inc.log" + ) + + execute_process( + COMMAND bash -c "${OSHMEM_INFO} --path incdir" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE OSHMEM_INFO_INCSTATUS + OUTPUT_FILE ${OSHMEM_INFO_INCOUTPUT} + ERROR_FILE ${OSHMEM_INFO_INCERROR} + ) + + if(OSHMEM_INFO_INCSTATUS) + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Program status code: ${OSHMEM_INFO_INCSTATUS}" + ) + endif() + + file(READ ${OSHMEM_INFO_INCOUTPUT} OSHMEM_INFO_OUTPUT_INCCONTENT) + + if(NOT DEFINED OSHMEM_INFO_OUTPUT_INCCONTENT) + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_INCERROR}" + ) + endif() + + if("${OSHMEM_INFO_OUTPUT_INCCONTENT}" STREQUAL "") + message( + FATAL_ERROR + "${OSHMEM_INFO} Failed! Check: ${OSHMEM_INFO_INCERROR}\n${OSHMEM_INFO_OUTPUT_INCCONTENT}" + ) + endif() + + string(REGEX MATCH "(\/.*)" OSHMEM_INCDIR_PATH + ${OSHMEM_INFO_OUTPUT_INCCONTENT} + ) + + string(STRIP ${OSHMEM_INCDIR_PATH} OSHMEM_INCDIR_PATH) + + set(OPENSHMEM_CFLAGS + "-I${OSHMEM_INCDIR_PATH} -pthread -I${OSHMEM_LIBDIR_PATH}" + ) + set(OPENSHMEM_LDFLAGS "-loshmem") + set(OPENSHMEM_LIBRARY_DIRS "${OSHMEM_LIBDIR_PATH}") + + add_library(PkgConfig::OPENSHMEM INTERFACE IMPORTED GLOBAL) + + set(OPENSHMEM_FOUND ON) + endif() + endif() + elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") + set(OPENSHMEM_PC "ucx") + + pkg_search_module(UCX IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) + + if(NOT UCX_FOUND) + message( + FATAL + "HPX_WITH_PARCELPORT_OPENSHMEM=ucx selected but UCX is unavailable" + ) + endif() + + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) + elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") + set(OPENSHMEM_PC "sandia-openshmem") + + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) + endif() + endif() + + if((NOT OPENSHMEM_FOUND) + AND HPX_WITH_FETCH_OPENSHMEM + AND NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + ) + if(NOT CMAKE_C_COMPILER) + message( + FATAL_ERROR + "HPX_WITH_FETCH_OPENSHMEM requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset." + ) + endif() + if(NOT CMAKE_CXX_COMPILER) + message( + FATAL_ERROR + "HPX_WITH_FETCH_OPENSHMEM requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset." + ) + endif() + + find_program(MAKE_EXECUTABLE NAMES gmake make mingw32-make REQUIRED) + + include(FindOpenShmemPmi) + + set(PMI_AUTOCONF_OPTS "") + if(NOT PMI_LIBRARY OR NOT PMI_FOUND) + set(PMI_AUTOCONF_OPTS "--enable-pmi-simple") + else() + string(REGEX MATCH "(^\/[^\/]+)" PMI_INCLUDE_DIR_ROOT_PATH + ${PMI_INCLUDE_DIR} + ) + string(REGEX MATCH "(^\/[^\/]+)" PMI_LIBRARY_ROOT_PATH ${PMI_LIBRARY}) + set(PMI_AUTOCONF_OPTS + "--with-pmi=${PMI_INCLUDE_DIR_ROOT_PATH} --with-pmi-libdir=${PMI_LIBRARY_ROOT_PATH}" + ) + endif() + + include(FetchContent) + + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") + + message(STATUS "Fetching OSSS-UCX-OpenSHMEM") + + fetchcontent_declare( + openshmem + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://github.com/openshmem-org/osss-ucx/archive/refs/tags/v1.0.2.tar.gz + ) + + message( + STATUS + "Building OSSS-UCX (OpenSHMEM on UCX) and installing into ${CMAKE_INSTALL_PREFIX}" + ) + + elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") + + message(STATUS "Fetching Sandia-OpenSHMEM") + + fetchcontent_declare( + openshmem + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://github.com/Sandia-OpenSHMEM/SOS/archive/refs/tags/v1.5.2.tar.gz + ) + + message( + STATUS + "Building and installing Sandia OpenSHMEM into ${CMAKE_INSTALL_PREFIX}" + ) + + else() + message( + FATAL_ERROR "HPX_WITH_PARCELPORT_OPENSHMEM is not set to `ucx` or `sos`" + ) + endif() + + fetchcontent_getproperties(openshmem) + if(NOT openshmem) + fetchcontent_populate(openshmem) + endif() + + set(OPENSHMEM_DIR "${openshmem_SOURCE_DIR}") + set(OPENSHMEM_BUILD_OUTPUT "${OPENSHMEM_DIR}/build.log") + set(OPENSHMEM_ERROR_FILE "${OPENSHMEM_DIR}/error.log") + + set(ENV{PKG_CONFIG_PATH} + "$ENV{PKG_CONFIG_PATH}:${OPENSHMEM_DIR}/install/lib/pkgconfig" + ) + + execute_process( + COMMAND + bash -c + "./autogen.sh && CC=${CMAKE_C_COMPILER} ./configure --prefix=${OPENSHMEM_DIR}/install --enable-shared --disable-fortran ${PMI_AUTOCONF_OPTS} && make && make install" + WORKING_DIRECTORY ${OPENSHMEM_DIR} + RESULT_VARIABLE OPENSHMEM_BUILD_STATUS + OUTPUT_FILE ${OPENSHMEM_BUILD_OUTPUT} + ERROR_FILE ${OPENSHMEM_ERROR_FILE} + ) + + if(OPENSHMEM_BUILD_STATUS) + file(READ ${OPENSHMEM_BUILD_OUTPUT} OPENSHMEM_BUILD_OUTPUT_CONTENT) + + message( + FATAL_ERROR + "OpenSHMEM build result = ${OPENSHMEM_BUILD_STATUS} - see ${OPENSHMEM_BUILD_OUTPUT} for more details\nOpenSHMEM build command\t:./autogen.sh && CC=${CMAKE_C_COMPILER} ./configure --prefix=${OPENSHMEM_DIR}/install --enable-shared --disable-fortran ${PMI_AUTOCONF_OPTS} && make && make install\n${OPENSHMEM_BUILD_OUTPUT_CONTENT}" + ) + else() + + find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND ${OPENSHMEM_PC}.pc + ${OPENSHMEM_DIR}/install/lib/pkgconfig + ) + + if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) + message( + FATAL_ERROR + "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}; ${OPENSHMEM_PC}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" + ) + endif() + + install(CODE "set(OPENSHMEM_PKGCONF \"${OPENSHMEM_PC}\")") + install(CODE "set(OPENSHMEM_PATH \"${OPENSHMEM_DIR}\")") + + install( + CODE [[ + file(READ + ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PKGCONF}.pc + OPENSHMEM_PKGCONFIG_FILE_CONTENT) + + if(NOT OPENSHMEM_PKGCONFIG_FILE_CONTENT) + message(FATAL_ERROR "ERROR INSTALLING OPENSHMEM") + endif() + + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_PKGCONFIG_FILE_CONTENT + ${OPENSHMEM_PKGCONFIG_FILE_CONTENT}) + + file(WRITE + ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PKGCONF}.pc + ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} + ) + + file(GLOB_RECURSE OPENSHMEM_FILES ${OPENSHMEM_PATH}/install/*) + + if(NOT OPENSHMEM_FILES) + message(STATUS "ERROR INSTALLING OPENSHMEM") + endif() + + foreach(OPENSHMEM_FILE ${OPENSHMEM_FILES}) + set(OPENSHMEM_FILE_CACHED "${OPENSHMEM_FILE}") + + string(REGEX MATCH "(^\/.*\/)" OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE}) + + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_FILE ${OPENSHMEM_FILE} + ) + + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE_PATH} + ) + + file(MAKE_DIRECTORY ${OPENSHMEM_FILE_PATH}) + + string(LENGTH ${OPENSHMEM_FILE_PATH} OPENSHMEM_FILE_PATH_SIZE) + math(EXPR OPENSHMEM_FILE_PATH_SIZE "${OPENSHMEM_FILE_PATH_SIZE}-1") + + string(SUBSTRING ${OPENSHMEM_FILE_PATH} 0 ${OPENSHMEM_FILE_PATH_SIZE} + OPENSHMEM_FILE_PATH) + + file(COPY ${OPENSHMEM_FILE_CACHED} DESTINATION ${OPENSHMEM_FILE_PATH}) + endforeach() + ]] + ) + endif() + + set(CMAKE_PREFIX_PATH "${OPENSHMEM_DIR}/install/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} + "$ENV{PKG_CONFIG_PATH}:${OPENSHMEM_DIR}/install/lib/pkgconfig" + ) + + pkg_search_module(OPENSHMEM REQUIRED IMPORTED_TARGET GLOBAL ${OPENSHMEM_PC}) + + if(NOT OPENSHMEM_FOUND) + message( + FATAL_ERROR + "OpenSHMEM downloaded, compiled, but cannot be found in ${CMAKE_INSTALL_PREFIX}" + ) + endif() + elseif((NOT OPENSHMEM_FOUND) AND (NOT HPX_WITH_FETCH_OPENSHMEM)) + message( + FATAL_ERROR "OpenSHMEM not found and HPX_WITH_FETCH_OPENSHMEM not set!" + ) + endif() + + if(OPENSHMEM_CFLAGS AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" + STREQUAL "mpi")) + ) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" + ) + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_CFLAGS "${X}") + endforeach() + endif() + + if(OPENSHMEM_CFLAGS_OTHER AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" + STREQUAL "mpi")) + ) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" + ) + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(OPENSHMEM_LDFLAGS AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" + STREQUAL "mpi")) + ) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(SKIP 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" + ) + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive" + ) + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " + ) + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(OPENSHMEM_LDFLAGS_OTHER + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" + ) + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive" + ) + + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " + ) + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + endif() + + if(OPENSHMEM_STATIC_CFLAGS + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" + ) + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_CFLAGS "${X}") + endforeach() + endif() + + if(OPENSHMEM_STATIC_CFLAGS_OTHER + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" + ) + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(OPENSHMEM_STATIC_LDFLAGS + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + string(FIND "${X}" "-loshmem" IDX) + else() + string(FIND "${X}" "-lsma" IDX) + endif() + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" + ) + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive" + ) + + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_STATIC_LDFLAGS "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " + ) + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(OPENSHMEM_STATIC_LDFLAGS_OTHER + AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi")) + ) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "mpi") + string(FIND "${X}" "-loshmem" IDX) + else() + string(FIND "${X}" "-lsma" IDX) + endif() + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" + ) + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} + ${X}" + ) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive" + ) + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " + ) + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(OPENSHMEM_DIR AND (NOT ("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL + "mpi")) + ) + list(TRANSFORM OPENSHMEM_CFLAGS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + list(TRANSFORM OPENSHMEM_LDFLAGS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + list(TRANSFORM OPENSHMEM_LIBRARY_DIRS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + + message(STATUS "OPENSHMEM_CFLAGS:\t${OPENSHMEM_CFLAGS}") + message(STATUS "OPENSHMEM_LDFLAGS:\t${OPENSHMEM_LDFLAGS}") + message(STATUS "OPENSHMEM_LIBRARY_DIRS:\t${OPENSHMEM_LIBRARY_DIRS}") + + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS + "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS + "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS}" + ) + else() + message(STATUS "OPENSHMEM_CFLAGS:\t${OPENSHMEM_CFLAGS}") + message(STATUS "OPENSHMEM_LDFLAGS:\t${OPENSHMEM_LDFLAGS}") + message(STATUS "OPENSHMEM_LIBRARY_DIRS:\t${OPENSHMEM_LIBRARY_DIRS}") + + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS + "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS + "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS}" + ) + endif() +endmacro() diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index c3f0ae31b8f7..156240aebcf6 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -52,6 +52,11 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) endif() endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() +endif() + if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) include(HPX_SetupGasnet) hpx_setup_gasnet() diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 3bc50eacd078..240f1b31e5d9 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -195,19 +195,26 @@ def run_mpi(cmd, localities, verbose): print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) -def run_gasnet(cmd, localities, verbose): - os.environ['GASNET_PSHM_NODES'] = str(localities) - os.environ['GASNET_QUIET'] = 'yes' +def run_amudprun(cmd, localities, nodes, verbose): + assert len(nodes) == localities, "Number of localities must match number of nodes." + os.environ['GASNET_SPAWNFN'] = 'S' + os.environ['GASNET_SSH_SERVERS'] = ' '.join([ str(node) for node in nodes ]) + os.environ['SSH_SERVERS'] = os.environ['GASNET_SSH_SERVERS'] os.environ['GASNET_ROUTE_OUTPUT'] = '0' - run_mpi(cmd, localities, verbose) -def run_gasnet_smp(cmd, localities, verbose): - os.environ['GASNET_PSHM_NODES'] = str(localities) - os.environ['GASNET_BARRIER'] = 'PSHM' - os.environ['GASNET_QUIET'] = 'yes' - os.environ['GASNET_ROUTE_OUTPUT'] = '0' - # No launcher needed - run_none(cmd, localities, nodes, verbose) + exec_cmd = ['amudprun', '-np', len(nodes), cmd] + + if verbose: + print('Executing command: ' + ' '.join(exec_cmd)) + subproc(exec_cmd) + +# Run with oshrun +# This is executing amudprun with the "-np" option set to the number of localities +def run_oshrun(cmd, localities, verbose): + exec_cmd = ['oshrun', '-np', str(localities)] + cmd + if verbose: + print('Executing command: ' + ' '.join(exec_cmd)) + subproc(exec_cmd) # Run with srun # This is executing srun with the '-n' option set to the number of localities @@ -232,12 +239,11 @@ def run(cmd, runwrapper, localities, nodes, verbose): if runwrapper == 'mpi': assert nodes is None, "nodes option only valid with tcp parcelport." run_mpi(cmd, localities, verbose) - if runwrapper == 'gasnet': + if runwrapper == 'amudprun': + run_amudprun(cmd, localities, nodes, verbose) + if runwrapper == 'oshrun': assert nodes is None, "nodes option only valid with tcp parcelport." - run_gasnet(cmd, localities, verbose) - if runwrapper == 'gasnet-smp': - assert nodes is None, "nodes option only valid with smp parcelport." - run_gasnet_smp(cmd, localities, verbose) + run_oshrun(cmd, localities, verbose) if runwrapper == 'srun': assert nodes is None, "nodes option only valid with tcp parcelport." run_srun(cmd, localities, verbose) @@ -260,6 +266,7 @@ def build_cmd(options, args): ['--hpx:ini=hpx.parcel.mpi.priority=1000', '--hpx:ini=hpx.parcel.mpi.enable=1', '--hpx:ini=hpx.parcel.bootstrap=mpi'] if pp == 'mpi' else ['--hpx:ini=hpx.parcel.lci.priority=1000', '--hpx:ini=hpx.parcel.lci.enable=1', '--hpx:ini=hpx.parcel.bootstrap=lci'] if pp == 'lci' else ['--hpx:ini=hpx.parcel.gasnet.priority=1000', '--hpx:ini=hpx.parcel.gasnet.enable=1', '--hpx:ini=hpx.parcel.bootstrap=gasnet'] if pp == 'gasnet' + else ['--hpx:ini=hpx.parcel.openshmem.priority=1000', '--hpx:ini=hpx.parcel.openshmem.enable=1', '--hpx:ini=hpx.parcel.bootstrap=openshmem'] if pp == 'openshmem' else ['--hpx:ini=hpx.parcel.tcp.priority=1000', '--hpx:ini=hpx.parcel.tcp.enable=1'] if pp == 'tcp' else []) cmd += select_parcelport(options.parcelport) @@ -296,14 +303,14 @@ def check_options(parser, options, args): print('Can not start less than one thread per locality', sys.stderr) sys.exit(1) - check_valid_parcelport = (lambda x: x == 'mpi' or x == 'lci' or x == 'gasnet' or x == 'tcp' or x == 'none'); + check_valid_parcelport = (lambda x: x == 'mpi' or x == 'lci' or x == 'gasnet' or x == 'tcp' or x == 'openshmem' or x == 'none'); if not check_valid_parcelport(options.parcelport): print('Error: Parcelport option not valid\n', sys.stderr) parser.print_help() sys.exit(1) check_valid_runwrapper = (lambda x: - x == 'none' or x == 'mpi' or x == 'srun' or x =='jsrun' or x == 'gasnet' or x == 'gasnet-smp'); + x == 'none' or x == 'mpi' or x == 'srun' or x =='jsrun' or x == 'amudprun' or x == 'oshrun'); if not check_valid_runwrapper(options.runwrapper): print('Error: Runwrapper option not valid\n', sys.stderr) parser.print_help() @@ -382,13 +389,13 @@ Used by the tcp parcelport only. parser.add_option('-p', '--parcelport' , action='store', type='string' , dest='parcelport', default=default_env('HPXRUN_PARCELPORT', 'tcp') - , help='Which parcelport to use (Options are: mpi, lci, gasnet, tcp) ' + , help='Which parcelport to use (Options are: mpi, lci, openshmem, gasnet, tcp) ' '(environment variable HPXRUN_PARCELPORT') parser.add_option('-r', '--runwrapper' , action='store', type='string' , dest='runwrapper', default=default_env('HPXRUN_RUNWRAPPER', 'none') - , help='Which runwrapper to use (Options are: none, mpi, gasnet, gasnet-smp, srun, jsrun) ' + , help='Which runwrapper to use (Options are: none, mpi, oshrun, amudprun, srun, jsrun) ' '(environment variable HPXRUN_RUNWRAPPER)') parser.add_option('-e', '--expected' diff --git a/docs/sphinx/manual.rst b/docs/sphinx/manual.rst index 256456efd864..097257ed4f9c 100644 --- a/docs/sphinx/manual.rst +++ b/docs/sphinx/manual.rst @@ -32,6 +32,7 @@ information on how to build and use |hpx| in different scenarios. manual/debugging_hpx_applications manual/optimizing_hpx_applications manual/using_the_lci_parcelport + manual/using_the_openshmem_parcelport manual/hpx_runtime_and_resources manual/miscellaneous manual/troubleshooting diff --git a/docs/sphinx/manual/building_hpx.rst b/docs/sphinx/manual/building_hpx.rst index 3e974517ddfa..7032b15d9167 100644 --- a/docs/sphinx/manual/building_hpx.rst +++ b/docs/sphinx/manual/building_hpx.rst @@ -94,6 +94,12 @@ used CMake options. this experimental parcelport may provide better performance than the MPI parcelport. Please refer to :ref:`using_the_lci_parcelport` for more information about the LCI parcelport. +.. option:: HPX_WITH_PARCELPORT_OPENSHMEM + + Enable the OPENSHMEM parcelport. This enables the use of OpenSHMEM for the networking operations in the HPX runtime. + The default value is ``OFF`` because it's not available on all systems and/or requires another dependency. Please + refer to :ref:`using_the_openshmem_parcelport` for more information about the OpenSHMEM parcelport. + .. option:: HPX_WITH_APEX Enable APEX integration. `APEX `_ can be used to profile |hpx| diff --git a/docs/sphinx/manual/using_the_openshmem_parcelport.rst b/docs/sphinx/manual/using_the_openshmem_parcelport.rst new file mode 100644 index 000000000000..edf4dceb3980 --- /dev/null +++ b/docs/sphinx/manual/using_the_openshmem_parcelport.rst @@ -0,0 +1,85 @@ +.. + Copyright (c) 2023 Christopher Taylor + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +.. _using_the_openshmem_parcelport: + +============================== +Using the OpenSHMEM parcelport +============================== + +.. _info_openshmem: + +Basic information +================= + +OpenSHMEM is an effort to create an API specification standardizing parallel +programming in the Partitioned Global Address Space model. The specification +effort also creates a reference implementation of the API. This implementation +aims to be portable, allowing it to be deployed in multiple environments, and +to be a starting point for implementations targeted to particular hardware +platforms. The reference implementation will serves as a springboard for future +development of the API. + +OpenSHMEM provides one-sided communications, atomics, collectives, and again +implements a partitioned global address space (PGAS) distributed memory +computing model. The parcelport uses put_signal (put and atomic_put), wait_until, +and get from the OpenSHMEM API. + +.. _`OpenSHMEM`: http://www.openshmem.org/site/ + +.. _build_openshmem_pp: + +Build |hpx| with the OpenSHMEM parcelport +=================================== + +While building |hpx|, you can specify a set of |cmake| variables to enable +and configure the OpenSHMEM parcelport. Below, there is a set of the most important +and frequently used CMake variables. + +.. option:: HPX_WITH_PARCELPORT_OPENSHMEM + + Enable the OpenSHMEM parcelport. This enables the use of OpenSHMEM for networking operations in the |hpx| runtime. + The default value is ``OFF`` because it's not available on all systems and/or requires another dependency. + You must set this variable to ``ON`` in order to use the OpenSHMEM parcelport. All the following variables only + make sense when this variable is set to ``ON``. + +.. option:: HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT + + Defines which OpenSHMEM to utilize. The options are `sos;ucx;mpi`. This feature tells cmake how to compile the + parcelport against a specific implementation of OpenSHMEM. + +.. option:: HPX_WITH_FETCH_OPENSHMEM + + Use FetchContent to fetch OpenSHMEM. The default value is ``OFF``. + If this option is set to ``OFF``. This feature tells |hpx| to fetch and build OpenSHMEM for you. + |hpx| will download either `ucx` or `sos` OpenSHMEM based on the value provided in + `HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT`. This feature requires the user to set `CMAKE_C_COMPILER`. The + OpenSHMEM downloaded will be installed into `CMAKE_INSTALL_PREFIX`. PMI support will be compiled + into the parcelport if it's available. If PMI is not available a barebones implementation for SMP + systems will be used. + +.. _run_openshmem_pp: + +Run |hpx| with the OpenSHMEM parcelport +================================= + +We use the same mechanisms as MPI to launch OpenSHMEM, so you can use the same way you run MPI parcelport to run OpenSHMEM +parcelport. Typically, it would be ``hpxrun``, ``mpirun``, ``srun``, or ``oshrun``. + +If you are using ``hpxrun.py``, just pass ``--parcelport openshmem`` to the scripts. + +If you are using ``mpirun``, ``oshrun``, or ``srun``, you can just pass +``--hpx:ini=hpx.parcel.openshmem.priority=1000``, ``--hpx:ini=hpx.parcel.openshmem.enable=1``, and +``--hpx:ini=hpx.parcel.bootstrap=openshmem`` to the |hpx| applications. + +If you are running on a Cray machine, you need to pass `--mpi=pmix` or `--mpi=pmi2` to srun +to enable the PMIx or PMI2 support of SLURM since OpenSHMEM does not support the default Cray PMI. +For example, + +.. code-block:: shell-session + + $ srun --mpi=pmix [hpx application] diff --git a/libs/core/CMakeLists.txt b/libs/core/CMakeLists.txt index 3e6550e93f25..a55d1eaac8e9 100644 --- a/libs/core/CMakeLists.txt +++ b/libs/core/CMakeLists.txt @@ -56,6 +56,7 @@ set(_hpx_core_modules logging memory mpi_base + openshmem_base pack_traversal plugin prefix diff --git a/libs/core/openshmem_base/CMakeLists.txt b/libs/core/openshmem_base/CMakeLists.txt new file mode 100644 index 000000000000..c6164fcfa68e --- /dev/null +++ b/libs/core/openshmem_base/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM)) + return() +endif() + +include(HPX_SetupOpenSHMEM) +hpx_setup_openshmem() + +# Default location is $HPX_ROOT/libs/openshmem_base/include +set(openshmem_base_headers hpx/openshmem_base/openshmem.hpp + hpx/openshmem_base/openshmem_environment.hpp +) + +set(openshmem_base_sources openshmem_environment.cpp) + +include(HPX_AddModule) +add_hpx_module( + core openshmem_base + GLOBAL_HEADER_GEN ON + SOURCES ${openshmem_base_sources} + HEADERS ${openshmem_base_headers} + DEPENDENCIES PkgConfig::OPENSHMEM + MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_string_util + hpx_threading_base hpx_util + CMAKE_SUBDIRS examples tests +) diff --git a/libs/core/openshmem_base/examples/CMakeLists.txt b/libs/core/openshmem_base/examples/CMakeLists.txt new file mode 100644 index 000000000000..ff6189b08a6f --- /dev/null +++ b/libs/core/openshmem_base/examples/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(HPX_WITH_EXAMPLES) + add_hpx_pseudo_target(examples.modules.openshmem_base) + add_hpx_pseudo_dependencies(examples.modules examples.modules.openshmem_base) + if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) + add_hpx_pseudo_target(tests.examples.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.examples.modules tests.examples.modules.openshmem_base + ) + endif() +endif() diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp new file mode 100644 index 000000000000..60413db0dbdc --- /dev/null +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp @@ -0,0 +1,30 @@ +// Copyright (c) 2023 Christoper Taylor +// Copyright (c) 2017 Mikael Simberg +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +#ifndef SHMEM_H +#include +#endif + +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +#endif diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp new file mode 100644 index 000000000000..af57d81b0fe4 --- /dev/null +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -0,0 +1,126 @@ +// Copyright (c) 2013-2015 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#if !defined(OPENSHMEM_PAGESIZE) +// GASNet default pagesize is 16384 +#define OPENSHMEM_PER_RANK_PAGESIZE 16384 +#endif + +namespace hpx { namespace util { + + struct openshmem_seginfo_t + { + std::uint8_t* addr; + std::uint8_t* size; + }; + + struct HPX_CORE_EXPORT openshmem_environment + { + static bool check_openshmem_environment( + runtime_configuration const& cfg); + + static int init(int* argc, char*** argv, int& provided); + static void init(int* argc, char*** argv, runtime_configuration& cfg); + static void finalize(); + + static bool enabled(); + static bool multi_threaded(); + static bool has_called_init(); + + static int rank(); + static int size(); + + static std::string get_processor_name(); + + static void put_signal(const std::uint8_t* addr, const int rank, + std::uint8_t* raddr, const std::size_t size, unsigned int* sigaddr); + + static void wait_until(const unsigned int value, unsigned int* sigaddr); + + static void get(std::uint8_t* addr, const int rank, + const std::uint8_t* raddr, const std::size_t size); + + static void global_barrier(); + + struct HPX_CORE_EXPORT scoped_lock + { + scoped_lock(); + scoped_lock(scoped_lock const&) = delete; + scoped_lock& operator=(scoped_lock const&) = delete; + ~scoped_lock(); + void unlock(); + }; + + struct HPX_CORE_EXPORT scoped_try_lock + { + scoped_try_lock(); + scoped_try_lock(scoped_try_lock const&) = delete; + scoped_try_lock& operator=(scoped_try_lock const&) = delete; + ~scoped_try_lock(); + void unlock(); + bool locked; + }; + + typedef hpx::spinlock mutex_type; + + public: + static hpx::spinlock pollingLock; + static hpx::mutex mtx_; + + static bool enabled_; + static bool has_called_init_; + static int provided_threading_flag_; + + static int is_initialized_; + + static hpx::mutex dshm_mut; + static int init_val_; + static hpx::mutex* segment_mutex; + static openshmem_seginfo_t* segments; + static std::uint8_t* shmem_buffer; + static unsigned int rcv; + static unsigned int xmt; + }; +}} // namespace hpx::util + +#include + +#else + +#include + +#include + +namespace hpx { namespace util { + struct HPX_CORE_EXPORT openshmem_environment + { + static bool check_openshmem_environment( + runtime_configuration const& cfg); + }; +}} // namespace hpx::util + +#include + +#endif diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp new file mode 100644 index 000000000000..6345a722c5ad --- /dev/null +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -0,0 +1,387 @@ +// Copyright (c) 2013-2015 Thomas Heller +// Copyright (c) 2020 Google +// Copyright (c) 2022 Patrick Diehl +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +namespace hpx::util { + + namespace detail { + + bool detect_openshmem_environment( + util::runtime_configuration const& cfg, char const* default_env) + { + std::string openshmem_environment_strings = + cfg.get_entry("hpx.parcel.openshmem.env", default_env); + + hpx::string_util::char_separator sep(";,: "); + hpx::string_util::tokenizer tokens( + openshmem_environment_strings, sep); + for (auto const& tok : tokens) + { + char* env = std::getenv(tok.c_str()); + if (env) + { + LBT_(debug) + << "Found OPENSHMEM environment variable: " << tok + << "=" << std::string(env) + << ", enabling OPENSHMEM support\n"; + return true; + } + } + + LBT_(info) + << "No known OPENSHMEM environment variable found, disabling " + "OPENSHMEM support\n"; + return false; + } + } // namespace detail + + bool openshmem_environment::check_openshmem_environment( + [[maybe_unused]] util::runtime_configuration const& cfg) + { +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + // We disable the OPENSHMEM parcelport if any of these hold: + // + // - The parcelport is explicitly disabled + // - The application is not run in an OPENSHMEM environment + // - The TCP parcelport is enabled and has higher priority + // + if (get_entry_as(cfg, "hpx.parcel.openshmem.enable", 1) == 0 || + (get_entry_as(cfg, "hpx.parcel.tcp.enable", 1) && + (get_entry_as(cfg, "hpx.parcel.tcp.priority", 1) > + get_entry_as(cfg, "hpx.parcel.openshmem.priority", 0))) || + (get_entry_as(cfg, "hpx.parcel.openshmem.enable", 1) && + (get_entry_as(cfg, "hpx.parcel.openshmem.priority", 1) > + get_entry_as(cfg, "hpx.parcel.mpi.priority", 0)))) + { + LBT_(info) + << "OpenSHMEM support disabled via configuration settings\n"; + return false; + } + + return true; +#else + return false; +#endif + } +} // namespace hpx::util + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_OPENSHMEM_BASE)) + +namespace hpx::util { + + hpx::spinlock openshmem_environment::pollingLock{}; + hpx::mutex openshmem_environment::dshm_mut{}; + hpx::mutex openshmem_environment::mtx_{}; + bool openshmem_environment::enabled_ = false; + bool openshmem_environment::has_called_init_ = false; + int openshmem_environment::provided_threading_flag_ = 0; + int openshmem_environment::is_initialized_ = -1; + int openshmem_environment::init_val_ = 0; + hpx::mutex* openshmem_environment::segment_mutex = nullptr; + openshmem_seginfo_t* openshmem_environment::segments = nullptr; + std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; + unsigned int openshmem_environment::rcv = 0; + unsigned int openshmem_environment::xmt = 0; + + /////////////////////////////////////////////////////////////////////////// + int openshmem_environment::init([[maybe_unused]] int* argc, + [[maybe_unused]] char*** argv, [[maybe_unused]] int& provided) + { + if (!has_called_init_) + { + shmem_init(); + openshmem_environment::init_val_ = 1; + has_called_init_ = true; + } + + if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM initialization error"); + } + else if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM resource error"); + } + else if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM bad argument error"); + } + else if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM not ready error"); + } + + // create a number of segments equal to the number of hardware + // threads per machine (locality) + // + //segments.resize(hpx::threads::hardware_concurrency() * size()); + // + openshmem_environment::segments = new openshmem_seginfo_t[size()]; + openshmem_environment::segment_mutex = new hpx::mutex[size()]; + + hpx::util::openshmem_environment::shmem_buffer = + static_cast(shmem_calloc( + OPENSHMEM_PER_RANK_PAGESIZE, sizeof(std::uint8_t))); + + for (int i = 0; i < size(); ++i) + { + segments[i].addr = hpx::util::openshmem_environment::shmem_buffer + + (i * OPENSHMEM_PER_RANK_PAGESIZE); + segments[i].size = static_cast(segments[i].addr) + + OPENSHMEM_PER_RANK_PAGESIZE; + } + + shmem_barrier_all(); + + return openshmem_environment::init_val_; + } + + /////////////////////////////////////////////////////////////////////////// + void openshmem_environment::init([[maybe_unused]] int* argc, + [[maybe_unused]] char*** argv, + [[maybe_unused]] util::runtime_configuration& rtcfg) + { + if (enabled_) + return; // don't call twice + + int this_rank = -1; + has_called_init_ = false; + + // We assume to use the OpenSHMEM parcelport if it is not explicitly disabled + enabled_ = check_openshmem_environment(rtcfg); + if (!enabled_) + { + rtcfg.add_entry("hpx.parcel.openshmem.enable", "0"); + return; + } + + rtcfg.add_entry("hpx.parcel.bootstrap", "openshmem"); + + int retval = init(argc, argv, provided_threading_flag_); + if (1 != retval) + { + // explicitly disable openshmem if not run by openshmemrun + rtcfg.add_entry("hpx.parcel.openshmem.enable", "0"); + + enabled_ = false; + + std::string msg( + "openshmem_environment::init: openshmem_init failed"); + throw std::runtime_error(msg.c_str()); + } + + if (provided_threading_flag_ != 1) + { + // explicitly disable openshmem if not run by openshmemrun + rtcfg.add_entry("hpx.parcel.openshmem.multithreaded", "0"); + } + + this_rank = rank(); + +#if defined(HPX_HAVE_NETWORKING) + if (this_rank == 0) + { + rtcfg.mode_ = hpx::runtime_mode::console; + } + else + { + rtcfg.mode_ = hpx::runtime_mode::worker; + } +#elif defined(HPX_HAVE_DISTRIBUTED_RUNTIME) + rtcfg.mode_ = hpx::runtime_mode::console; +#else + rtcfg.mode_ = hpx::runtime_mode::local; +#endif + + rtcfg.add_entry("hpx.parcel.openshmem.rank", std::to_string(this_rank)); + rtcfg.add_entry( + "hpx.parcel.openshmem.processorname", get_processor_name()); + } + + std::string openshmem_environment::get_processor_name() + { + char name[1024 + 1] = {'\0'}; + const std::string rnkstr = std::to_string(rank()); + const int len = rnkstr.size(); + if (1025 < len) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::get_processor_name", + "openshmem processor name is larger than 1025"); + } + std::copy(std::begin(rnkstr), std::end(rnkstr), name); + return name; + } + + void openshmem_environment::put_signal(const std::uint8_t* addr, + const int node, std::uint8_t* raddr, const std::size_t size, + unsigned int* sigaddr) + { + if (rank() == node) + { + const std::lock_guard lk(segment_mutex[node]); + std::memmove(raddr, addr, size); + } + else + { + const std::lock_guard lk(segment_mutex[node]); + +#if !defined(SHMEM_SIGNAL_SET) + shmem_uint8_put(raddr, addr, size, node); + shmem_fence(); + shmem_uint8_put(reinterpret_cast(sigaddr), + reinterpret_cast(sigaddr), sizeof(sigaddr), + node); +#else + shmem_uint8_put_signal(raddr, addr, size, + reinterpret_cast(sigaddr), 1, SHMEM_SIGNAL_SET, + node); +#endif + } + } + + void openshmem_environment::wait_until( + const unsigned int value, unsigned int* sigaddr) + { + shmem_uint_wait_until(sigaddr, SHMEM_CMP_EQ, value); + } + + void openshmem_environment::get(std::uint8_t* addr, const int node, + const std::uint8_t* raddr, const std::size_t size) + { + if (rank() == node) + { + std::memmove(addr, raddr, size); + } + else + { + shmem_uint8_get( + addr, raddr, size, node); // dest, node, src, size + } + } + + void openshmem_environment::global_barrier() + { + shmem_barrier_all(); + } + + void openshmem_environment::finalize() + { + if (enabled() && has_called_init()) + { + shmem_finalize(); + + delete segments; + delete segment_mutex; + shmem_free(hpx::util::openshmem_environment::shmem_buffer); + segments = nullptr; + segment_mutex = nullptr; + shmem_buffer = nullptr; + } + } + + bool openshmem_environment::enabled() + { + return enabled_; + } + + bool openshmem_environment::multi_threaded() + { + return provided_threading_flag_ != 0; + } + + bool openshmem_environment::has_called_init() + { + return has_called_init_; + } + + int openshmem_environment::size() + { + int res(-1); + if (enabled()) + res = static_cast(shmem_n_pes()); + return res; + } + + int openshmem_environment::rank() + { + int res(-1); + if (enabled()) + res = static_cast(shmem_my_pe()); + return res; + } + + openshmem_environment::scoped_lock::scoped_lock() + { + if (!multi_threaded()) + mtx_.lock(); + } + + openshmem_environment::scoped_lock::~scoped_lock() + { + if (!multi_threaded()) + mtx_.unlock(); + } + + void openshmem_environment::scoped_lock::unlock() + { + if (!multi_threaded()) + mtx_.unlock(); + } + + openshmem_environment::scoped_try_lock::scoped_try_lock() + : locked(true) + { + if (!multi_threaded()) + { + locked = mtx_.try_lock(); + } + } + + openshmem_environment::scoped_try_lock::~scoped_try_lock() + { + if (!multi_threaded() && locked) + mtx_.unlock(); + } + + void openshmem_environment::scoped_try_lock::unlock() + { + if (!multi_threaded() && locked) + { + locked = false; + mtx_.unlock(); + } + } +} // namespace hpx::util + +#endif diff --git a/libs/core/openshmem_base/tests/CMakeLists.txt b/libs/core/openshmem_base/tests/CMakeLists.txt new file mode 100644 index 000000000000..76ce5efbb376 --- /dev/null +++ b/libs/core/openshmem_base/tests/CMakeLists.txt @@ -0,0 +1,44 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) +include(HPX_Option) + +if(HPX_WITH_TESTS) + if(HPX_WITH_TESTS_UNIT) + add_hpx_pseudo_target(tests.unit.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.unit.modules tests.unit.modules.openshmem_base + ) + add_subdirectory(unit) + endif() + + if(HPX_WITH_TESTS_REGRESSIONS) + add_hpx_pseudo_target(tests.regressions.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.regressions.modules tests.regressions.modules.openshmem_base + ) + add_subdirectory(regressions) + endif() + + if(HPX_WITH_TESTS_BENCHMARKS) + add_hpx_pseudo_target(tests.performance.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.performance.modules tests.performance.modules.openshmem_base + ) + add_subdirectory(performance) + endif() + + if(HPX_WITH_TESTS_HEADERS) + add_hpx_header_tests( + modules.openshmem_base + HEADERS ${openshmem_base_headers} + HEADER_ROOT ${PROJECT_SOURCE_DIR}/include + NOLIBS + DEPENDENCIES hpx_openshmem_base + ) + endif() +endif() diff --git a/libs/core/openshmem_base/tests/performance/CMakeLists.txt b/libs/core/openshmem_base/tests/performance/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/openshmem_base/tests/performance/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/openshmem_base/tests/regressions/CMakeLists.txt b/libs/core/openshmem_base/tests/regressions/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/openshmem_base/tests/regressions/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/openshmem_base/tests/unit/CMakeLists.txt b/libs/core/openshmem_base/tests/unit/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/openshmem_base/tests/unit/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/version/CMakeLists.txt b/libs/core/version/CMakeLists.txt index 5a82861e5ec2..470fe90c852b 100644 --- a/libs/core/version/CMakeLists.txt +++ b/libs/core/version/CMakeLists.txt @@ -20,6 +20,18 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() + set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) +endif() +if(HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_OPENSHMEM +) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() + set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) +endif() if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT diff --git a/libs/core/version/src/version.cpp b/libs/core/version/src/version.cpp index 96d1662efdf2..0a48cdc99c08 100644 --- a/libs/core/version/src/version.cpp +++ b/libs/core/version/src/version.cpp @@ -46,6 +46,14 @@ #include #endif +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) +#include + +#define OPENSHMEM_CONDUIT_NAME_STR_HELPER(x) #x +#define OPENSHMEM_CONDUIT_NAME_STR \ + OPENSHMEM_CONDUIT_NAME_STR_HELPER(HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT) +#endif + #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) #include @@ -136,6 +144,25 @@ namespace hpx { } #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + std::string openshmem_version() + { + int major = 0; + int minor = 0; + shmem_info_get_version(&major, &minor); + char vendor_cstr[SHMEM_MAX_NAME_LEN]; + shmem_info_get_name(vendor_cstr); + + std::ostringstream strm; + strm << "OPENSHMEM_VENDOR:" << vendor_cstr << ':' << major << ':' + << minor << '-' + << "OPENSHMEM_CONDUIT:" << OPENSHMEM_CONDUIT_NAME_STR; + return strm.str(); + } +#endif + #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) std::string gasnet_version() @@ -312,6 +339,11 @@ namespace hpx { defined(HPX_HAVE_MODULE_LCI_BASE) " LCI: {}\n" #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + " OPENSHMEM: {}\n" +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) " GASNET: {}\n" @@ -332,6 +364,11 @@ namespace hpx { defined(HPX_HAVE_MODULE_LCI_BASE) lci_version(), #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + openshmem_version(), +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) gasnet_version(), diff --git a/libs/full/CMakeLists.txt b/libs/full/CMakeLists.txt index e4a7eb24b923..d4f1b25c0e69 100644 --- a/libs/full/CMakeLists.txt +++ b/libs/full/CMakeLists.txt @@ -35,6 +35,7 @@ set(_hpx_full_modules parcelport_lci parcelport_libfabric parcelport_mpi + parcelport_openshmem parcelport_tcp parcelports parcelset diff --git a/libs/full/command_line_handling/CMakeLists.txt b/libs/full/command_line_handling/CMakeLists.txt index 966ba221406e..cf4cd5d16212 100644 --- a/libs/full/command_line_handling/CMakeLists.txt +++ b/libs/full/command_line_handling/CMakeLists.txt @@ -34,6 +34,11 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() +if(HPX_WITH_PARCELPORT_OPENSHMEM) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() + set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) +endif() if(HPX_WITH_PARCELPORT_GASNET) include(HPX_SetupGasnet) hpx_setup_gasnet() diff --git a/libs/full/command_line_handling/src/command_line_handling.cpp b/libs/full/command_line_handling/src/command_line_handling.cpp index b8b7abbf5743..23d5b914b5f4 100644 --- a/libs/full/command_line_handling/src/command_line_handling.cpp +++ b/libs/full/command_line_handling/src/command_line_handling.cpp @@ -21,6 +21,9 @@ #if defined(HPX_HAVE_MODULE_LCI_BASE) #include #endif +#if defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) +#include +#endif #if defined(HPX_HAVE_MODULE_GASNET_BASE) #include #endif @@ -1006,6 +1009,18 @@ namespace hpx::util { node_ = static_cast(util::lci_environment::rank()); } #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + if (util::openshmem_environment::check_openshmem_environment(rtcfg_)) + { + util::openshmem_environment::init(&argc, &argv, rtcfg_); + num_localities_ = + static_cast(util::openshmem_environment::size()); + node_ = + static_cast(util::openshmem_environment::rank()); + } +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) // better to put GASNET init after MPI init, since GASNET will also diff --git a/libs/full/parcelport_openshmem/CMakeLists.txt b/libs/full/parcelport_openshmem/CMakeLists.txt new file mode 100644 index 000000000000..064fd9d70301 --- /dev/null +++ b/libs/full/parcelport_openshmem/CMakeLists.txt @@ -0,0 +1,44 @@ +# Copyright (c) 2023 Chistopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(NOT + (HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_OPENSHMEM + AND HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT) +) + return() +endif() + +include(HPX_SetupOpenSHMEM) +hpx_setup_openshmem() + +set(parcelport_openshmem_headers + hpx/parcelport_openshmem/header.hpp + hpx/parcelport_openshmem/locality.hpp + hpx/parcelport_openshmem/receiver.hpp + hpx/parcelport_openshmem/receiver_connection.hpp + hpx/parcelport_openshmem/sender.hpp + hpx/parcelport_openshmem/sender_connection.hpp + hpx/parcelport_openshmem/tag_provider.hpp +) + +set(parcelport_openshmem_sources locality.cpp parcelport_openshmem.cpp) + +include(HPX_AddModule) +add_hpx_module( + full parcelport_openshmem + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_openshmem_sources} + HEADERS ${parcelport_openshmem_headers} + DEPENDENCIES hpx_core hpx_openshmem_base PkgConfig::OPENSHMEM + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + CMAKE_SUBDIRS examples tests +) + +set(HPX_STATIC_PARCELPORT_PLUGINS + ${HPX_STATIC_PARCELPORT_PLUGINS} parcelport_openshmem + CACHE INTERNAL "" FORCE +) diff --git a/libs/full/parcelport_openshmem/examples/CMakeLists.txt b/libs/full/parcelport_openshmem/examples/CMakeLists.txt new file mode 100644 index 000000000000..d14c5d1103c2 --- /dev/null +++ b/libs/full/parcelport_openshmem/examples/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(HPX_WITH_EXAMPLES) + add_hpx_pseudo_target(examples.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + examples.modules examples.modules.parcelport_openshmem + ) + if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) + add_hpx_pseudo_target(tests.examples.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.examples.modules tests.examples.modules.parcelport_openshmem + ) + endif() +endif() diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp new file mode 100644 index 000000000000..3712581d08e2 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp @@ -0,0 +1,150 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2013-2021 Hartmut Kaiser +// Copyright (c) 2013-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include + +#include + +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct header + { + using value_type = int; + + enum data_pos + { + pos_tag = 0 * sizeof(value_type), + pos_size = 1 * sizeof(value_type), + pos_numbytes = 2 * sizeof(value_type), + pos_numchunks_first = 3 * sizeof(value_type), + pos_numchunks_second = 4 * sizeof(value_type), + pos_piggy_back_flag = 5 * sizeof(value_type), + pos_piggy_back_data = 5 * sizeof(value_type) + 1 + }; + + static constexpr int data_size_ = 512; + + template + header(Buffer const& buffer, int tag) noexcept + { + std::int64_t size = static_cast(buffer.size_); + std::int64_t numbytes = + static_cast(buffer.data_size_); + + HPX_ASSERT(size <= (std::numeric_limits::max)()); + HPX_ASSERT(numbytes <= (std::numeric_limits::max)()); + + set(tag); + set(static_cast(size)); + set(static_cast(numbytes)); + set( + static_cast(buffer.num_chunks_.first)); + set( + static_cast(buffer.num_chunks_.second)); + + if (buffer.data_.size() <= (data_size_ - pos_piggy_back_data)) + { + data_[pos_piggy_back_flag] = 1; + std::memcpy(&data_[pos_piggy_back_data], &buffer.data_[0], + buffer.data_.size()); + } + else + { + data_[pos_piggy_back_flag] = 0; + } + } + + header() noexcept + { + reset(); + } + + void reset() noexcept + { + std::memset(&data_[0], -1, data_size_); + data_[pos_piggy_back_flag] = 1; + } + + bool valid() const noexcept + { + return data_[0] != -1; + } + + void assert_valid() const noexcept + { + HPX_ASSERT(tag() != -1); + HPX_ASSERT(size() != -1); + HPX_ASSERT(numbytes() != -1); + HPX_ASSERT(num_chunks().first != -1); + HPX_ASSERT(num_chunks().second != -1); + } + + constexpr char* data() noexcept + { + return &data_[0]; + } + + value_type tag() const noexcept + { + return get(); + } + + value_type size() const noexcept + { + return get(); + } + + value_type numbytes() const noexcept + { + return get(); + } + + std::pair num_chunks() const noexcept + { + return std::make_pair( + get(), get()); + } + + constexpr char* piggy_back() noexcept + { + if (data_[pos_piggy_back_flag]) + return &data_[pos_piggy_back_data]; + return nullptr; + } + + private: + std::array data_; + + template + void set(T const& t) noexcept + { + std::memcpy(&data_[Pos], &t, sizeof(t)); + } + + template + value_type get() const noexcept + { + value_type res; + std::memcpy(&res, &data_[Pos], sizeof(res)); + return res; + } + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp new file mode 100644 index 000000000000..d0fe8b8eecf4 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp @@ -0,0 +1,71 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2013-2014 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include + +#include + +namespace hpx::parcelset::policies::openshmem { + + class locality + { + public: + constexpr locality() noexcept + : rank_(-1) + { + } + + explicit constexpr locality(std::int32_t rank) noexcept + : rank_(rank) + { + } + + constexpr std::int32_t rank() const noexcept + { + return static_cast(rank_); + } + + static constexpr const char* type() noexcept + { + return "openshmem"; + } + + explicit constexpr operator bool() const noexcept + { + return rank_ != -1; + } + + HPX_EXPORT void save(serialization::output_archive& ar) const; + HPX_EXPORT void load(serialization::input_archive& ar); + + private: + friend bool operator==( + locality const& lhs, locality const& rhs) noexcept + { + return lhs.rank_ == rhs.rank_; + } + + friend bool operator<(locality const& lhs, locality const& rhs) noexcept + { + return lhs.rank_ < rhs.rank_; + } + + friend HPX_EXPORT std::ostream& operator<<( + std::ostream& os, locality const& loc) noexcept; + + int rank_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp new file mode 100644 index 000000000000..2c828f54e643 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -0,0 +1,170 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + template + struct receiver + { + using header_list = std::list>; + using handles_header_type = std::set>; + using connection_type = receiver_connection; + using connection_ptr = std::shared_ptr; + using connection_list = std::deque; + + struct exp_backoff + { + int numTries; + const static int maxRetries = 10; + + void operator()(unsigned int* addr) + { + if (numTries <= maxRetries) + { + if (shmem_uint_test(addr, SHMEM_CMP_EQ, 1)) + { + return; + } + hpx::this_thread::suspend( + std::chrono::microseconds(1 << numTries)); + } + else + { + numTries = 0; + } + } + }; + + explicit constexpr receiver(Parcelport& pp) noexcept + : pp_(pp) + , bo() + { + } + + void run() noexcept + { + util::openshmem_environment::scoped_lock l; + new_header(); + } + + bool background_work() noexcept + { + // We first try to accept a new connection + connection_ptr connection = accept(); + + // If we don't have a new connection, try to handle one of the + // already accepted ones. + if (!connection) + { + std::unique_lock l(connections_mtx_, std::try_to_lock); + if (l.owns_lock() && !connections_.empty()) + { + connection = HPX_MOVE(connections_.front()); + connections_.pop_front(); + } + } + + if (connection) + { + receive_messages(HPX_MOVE(connection)); + return true; + } + + return false; + } + + void receive_messages(connection_ptr connection) noexcept + { + if (!connection->receive()) + { + std::unique_lock l(connections_mtx_); + connections_.push_back(HPX_MOVE(connection)); + } + } + + connection_ptr accept() noexcept + { + std::unique_lock l(headers_mtx_, std::try_to_lock); + if (l.owns_lock()) + { + return accept_locked(l); + } + return connection_ptr(); + } + + template + connection_ptr accept_locked(Lock& header_lock) noexcept + { + connection_ptr res; + util::openshmem_environment::scoped_try_lock l; + + if (l.locked) + { + header h = new_header(); + l.unlock(); + header_lock.unlock(); + + // remote localities 'put' into the openshmem shared + // memory segment on this machine + // + res.reset(new connection_type( + hpx::util::openshmem_environment::rank(), h, pp_)); + return res; + } + return res; + } + + header new_header() noexcept + { + header h = rcv_header_; + rcv_header_.reset(); + + while (rcv_header_.data() == 0) + { + bo(&hpx::util::openshmem_environment::rcv); + } + + hpx::util::openshmem_environment::rcv = 0; + return h; + } + + Parcelport& pp_; + + hpx::spinlock headers_mtx_; + header rcv_header_; + + hpx::spinlock handles_header_mtx_; + handles_header_type handles_header_; + + hpx::spinlock connections_mtx_; + connection_list connections_; + exp_backoff bo; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp new file mode 100644 index 000000000000..b2166090b437 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -0,0 +1,269 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2007-2021 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + template + struct receiver_connection + { + private: + enum connection_state + { + initialized, + rcvd_transmission_chunks, + rcvd_data, + rcvd_chunks, + sent_release_tag + }; + + using data_type = std::vector; + using buffer_type = parcel_buffer; + + public: + receiver_connection(int src, header h, Parcelport& pp) noexcept + : state_(initialized) + , src_(src) + , tag_(h.tag()) + , header_(h) + , request_ptr_(false) + , chunks_idx_(0) + , pp_(pp) + { + header_.assert_valid(); +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + parcelset::data_point& data = buffer_.data_point_; + data.time_ = timer_.elapsed_nanoseconds(); + data.bytes_ = static_cast(header_.numbytes()); +#endif + buffer_.data_.resize(static_cast(header_.size())); + buffer_.num_chunks_ = header_.num_chunks(); + } + + bool receive(std::size_t num_thread = -1) + { + switch (state_) + { + case initialized: + return receive_transmission_chunks(num_thread); + + case rcvd_transmission_chunks: + return receive_data(num_thread); + + case rcvd_data: + return receive_chunks(num_thread); + + case rcvd_chunks: + return send_release_tag(num_thread); + + case sent_release_tag: + return done(); + + default: + HPX_ASSERT(false); + } + return false; + } + + bool receive_transmission_chunks(std::size_t num_thread = -1) + { + auto self_ = hpx::util::openshmem_environment::rank(); + + // determine the size of the chunk buffer + std::size_t num_zero_copy_chunks = static_cast( + static_cast(buffer_.num_chunks_.first)); + std::size_t num_non_zero_copy_chunks = static_cast( + static_cast(buffer_.num_chunks_.second)); + buffer_.transmission_chunks_.resize( + num_zero_copy_chunks + num_non_zero_copy_chunks); + if (num_zero_copy_chunks != 0) + { + buffer_.chunks_.resize(num_zero_copy_chunks); + { + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast( + buffer_.transmission_chunks_.data()), + self_, + hpx::util::openshmem_environment::segments[self_].addr, + static_cast(buffer_.transmission_chunks_.size() * + sizeof(buffer_type::transmission_chunk_type))); + + request_ptr_ = true; + } + } + + state_ = rcvd_transmission_chunks; + + return receive_data(num_thread); + } + + bool receive_data(std::size_t num_thread = -1) + { + if (!request_done()) + { + return false; + } + + char* piggy_back = header_.piggy_back(); + if (piggy_back) + { + std::memcpy( + &buffer_.data_[0], piggy_back, buffer_.data_.size()); + } + else + { + auto self_ = hpx::util::openshmem_environment::rank(); + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast(buffer_.data_.data()), + self_, + hpx::util::openshmem_environment::segments[self_].addr, + buffer_.data_.size()); + + request_ptr_ = true; + } + + state_ = rcvd_data; + + return receive_chunks(num_thread); + } + + bool receive_chunks(std::size_t num_thread = -1) + { + while (chunks_idx_ < buffer_.chunks_.size()) + { + if (!request_done()) + { + return false; + } + + std::size_t idx = chunks_idx_++; + std::size_t chunk_size = + buffer_.transmission_chunks_[idx].second; + + data_type& c = buffer_.chunks_[idx]; + c.resize(chunk_size); + { + auto self_ = hpx::util::openshmem_environment::rank(); + hpx::util::openshmem_environment::scoped_lock l; + + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast(c.data()), self_, + hpx::util::openshmem_environment::segments[self_].addr, + c.size()); + + hpx::util::openshmem_environment::put_signal(nullptr, src_, + nullptr, 0, &hpx::util::openshmem_environment::xmt); + + request_ptr_ = true; + } + } + + state_ = rcvd_chunks; + + return send_release_tag(num_thread); + } + + bool send_release_tag(std::size_t num_thread = -1) + { + if (!request_done()) + { + return false; + } +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + parcelset::data_point& data = buffer_.data_point_; + data.time_ = timer_.elapsed_nanoseconds() - data.time_; +#endif + { + auto self_ = hpx::util::openshmem_environment::rank(); + hpx::util::openshmem_environment::scoped_lock l; + + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast(&tag_), self_, + hpx::util::openshmem_environment::segments[self_].addr, + sizeof(int)); + + request_ptr_ = true; + } + + decode_parcels(pp_, HPX_MOVE(buffer_), num_thread); + + state_ = sent_release_tag; + + return done(); + } + + bool done() noexcept + { + return request_done(); + } + + bool request_done() noexcept + { + hpx::util::openshmem_environment::scoped_try_lock l; + if (!l.locked) + { + return false; + } + + return request_ptr_; + } + +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + hpx::chrono::high_resolution_timer timer_; +#endif + connection_state state_; + + int src_; + int tag_; + header header_; + buffer_type buffer_; + + bool request_ptr_; + std::size_t chunks_idx_; + + Parcelport& pp_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp new file mode 100644 index 000000000000..fe75167a4208 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -0,0 +1,162 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct sender + { + using connection_type = sender_connection; + using connection_ptr = std::shared_ptr; + using connection_list = std::deque; + + // different versions of clang-format disagree + // clang-format off + sender() noexcept + : next_free_tag_(-1) + { + } + // clang-format on + + void run() noexcept + { + hpx::util::openshmem_environment::scoped_lock l; + get_next_free_tag(); + } + + connection_ptr create_connection(int dest, parcelset::parcelport* pp) + { + return std::make_shared(this, dest, pp); + } + + void add(connection_ptr const& ptr) + { + std::unique_lock l(connections_mtx_); + connections_.push_back(ptr); + } + + int acquire_tag() noexcept + { + return tag_provider_.acquire(); + } + + void send_messages(connection_ptr connection) + { + // Check if sending has been completed.... + if (connection->send()) + { + error_code ec(throwmode::lightweight); + hpx::move_only_function + postprocess_handler; + std::swap( + postprocess_handler, connection->postprocess_handler_); + postprocess_handler(ec, connection->destination(), connection); + } + else + { + std::unique_lock l(connections_mtx_); + connections_.push_back(HPX_MOVE(connection)); + } + } + + bool background_work() noexcept + { + connection_ptr connection; + { + std::unique_lock l(connections_mtx_, std::try_to_lock); + if (l && !connections_.empty()) + { + connection = HPX_MOVE(connections_.front()); + connections_.pop_front(); + } + } + + bool has_work = false; + if (connection) + { + send_messages(HPX_MOVE(connection)); + has_work = true; + } + next_free_tag(); + return has_work; + } + + private: + tag_provider tag_provider_; + + void next_free_tag() noexcept + { + int next_free = -1; + { + std::unique_lock l(next_free_tag_mtx_, std::try_to_lock); + if (l.owns_lock()) + { + next_free = next_free_tag_locked(); + } + } + + if (next_free != -1) + { + HPX_ASSERT(next_free > 1); + tag_provider_.release(next_free); + } + } + + int next_free_tag_locked() noexcept + { + hpx::util::openshmem_environment::scoped_try_lock l; + if (l.locked) + { + return get_next_free_tag(); + } + return -1; + } + + int get_next_free_tag() noexcept + { + int next_free = next_free_tag_; + + hpx::util::openshmem_environment::scoped_lock l; + std::memcpy(&next_free, + hpx::util::openshmem_environment::segments + [hpx::util::openshmem_environment::rank()] + .addr, + sizeof(int)); + + return next_free; + } + + hpx::spinlock connections_mtx_; + connection_list connections_; + + hpx::spinlock next_free_tag_mtx_; + int next_free_tag_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp new file mode 100644 index 000000000000..eb1047405a61 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -0,0 +1,325 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct sender; + struct sender_connection; + + int acquire_tag(sender*) noexcept; + void add_connection(sender*, std::shared_ptr const&); + + struct sender_connection + : parcelset::parcelport_connection> + { + private: + using sender_type = sender; + + using write_handler_type = + hpx::function; + + using data_type = std::vector; + + enum connection_state + { + initialized, + sent_header, + sent_transmission_chunks, + sent_data, + sent_chunks + }; + + using base_type = + parcelset::parcelport_connection; + + public: + sender_connection(sender_type* s, int dst, parcelset::parcelport* pp) + : state_(initialized) + , sender_(s) + , tag_(-1) + , dst_(dst) + , chunks_idx_(0) + , ack_(0) + , pp_(pp) + , there_(parcelset::locality(locality(dst_))) + { + } + + parcelset::locality const& destination() const noexcept + { + return there_; + } + + constexpr void verify_( + parcelset::locality const& /* parcel_locality_id */) const noexcept + { + } + + template + void async_write( + Handler&& handler, ParcelPostprocess&& parcel_postprocess) + { + HPX_ASSERT(!handler_); + HPX_ASSERT(!postprocess_handler_); + HPX_ASSERT(!buffer_.data_.empty()); + +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + buffer_.data_point_.time_ = + hpx::chrono::high_resolution_clock::now(); +#endif + chunks_idx_ = 0; + tag_ = acquire_tag(sender_); + header_ = header(buffer_, tag_); + header_.assert_valid(); + + state_ = initialized; + + handler_ = HPX_FORWARD(Handler, handler); + + if (!send()) + { + postprocess_handler_ = + HPX_FORWARD(ParcelPostprocess, parcel_postprocess); + add_connection(sender_, shared_from_this()); + } + else + { + HPX_ASSERT(!handler_); + error_code ec; + parcel_postprocess(ec, there_, shared_from_this()); + } + } + + bool send() + { + switch (state_) + { + case initialized: + return send_header(); + + case sent_header: + return send_transmission_chunks(); + + case sent_transmission_chunks: + return send_data(); + + case sent_data: + return send_chunks(); + + case sent_chunks: + return done(); + + default: + HPX_ASSERT(false); + } + return false; + } + + bool send_header() + { + { + hpx::util::openshmem_environment::scoped_lock l; + HPX_ASSERT(state_ == initialized); + + // compute + send the number of OPENSHMEM_PAGEs to send and the + // remainder number of bytes to a OPENSHMEM_PAGE + // + std::size_t chunks[] = {static_cast(header_.data_size_ / + OPENSHMEM_PER_RANK_PAGESIZE), + static_cast( + header_.data_size_ % OPENSHMEM_PER_RANK_PAGESIZE)}; + const std::size_t sizeof_chunks = sizeof(chunks); + + // put from this localities openshmem shared memory segment + // into the remote locality (dst_)'s shared memory segment + // + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(chunks), dst_, + static_cast( + hpx::util::openshmem_environment::segments[dst_].addr), + sizeof_chunks, &hpx::util::openshmem_environment::rcv); + } + + state_ = sent_header; + return send_transmission_chunks(); + } + + bool send_transmission_chunks() + { + HPX_ASSERT(state_ == sent_header); + if (!request_done()) + { + return false; + } + + std::vector& + chunks = buffer_.transmission_chunks_; + if (!chunks.empty()) + { + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(chunks.data()), dst_, + static_cast( + hpx::util::openshmem_environment::segments + [hpx::util::openshmem_environment::rank()] + .addr), + static_cast(chunks.size() * + sizeof(parcel_buffer_type::transmission_chunk_type)), + &hpx::util::openshmem_environment::rcv); + } + + state_ = sent_transmission_chunks; + return send_data(); + } + + bool send_data() + { + HPX_ASSERT(state_ == sent_transmission_chunks); + if (!request_done()) + { + return false; + } + + if (!header_.piggy_back()) + { + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(buffer_.data_.data()), dst_, + static_cast( + hpx::util::openshmem_environment::segments[dst_].addr), + buffer_.data_.size(), + &hpx::util::openshmem_environment::rcv); + } + state_ = sent_data; + + return send_chunks(); + } + + bool send_chunks() + { + HPX_ASSERT(state_ == sent_data); + + while (chunks_idx_ < buffer_.chunks_.size()) + { + serialization::serialization_chunk& c = + buffer_.chunks_[chunks_idx_]; + if (c.type_ == serialization::chunk_type::chunk_type_pointer) + { + if (!request_done()) + { + return false; + } + + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(c.data_.cpos_), + dst_, + static_cast( + hpx::util::openshmem_environment::segments[dst_] + .addr), + static_cast(c.size_), + &hpx::util::openshmem_environment::rcv); + + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::xmt); + hpx::util::openshmem_environment::xmt = 0; + } + + ++chunks_idx_; + } + + state_ = sent_chunks; + + return done(); + } + + bool done() + { + if (!request_done()) + { + return false; + } + + error_code ec(throwmode::lightweight); + handler_(ec); + handler_.reset(); +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + buffer_.data_point_.time_ = + hpx::chrono::high_resolution_clock::now() - + buffer_.data_point_.time_; + pp_->add_sent_data(buffer_.data_point_); +#endif + buffer_.clear(); + + state_ = initialized; + + return true; + } + + bool request_done() + { + hpx::util::openshmem_environment::scoped_try_lock l; + if (!l.locked) + { + return false; + } + + return true; + } + + connection_state state_; + sender_type* sender_; + int tag_; + int dst_; + + using handler_type = hpx::move_only_function; + handler_type handler_; + + using post_handler_type = hpx::move_only_function)>; + post_handler_type postprocess_handler_; + + header header_; + + std::size_t chunks_idx_; + char ack_; + + parcelset::parcelport* pp_; + + parcelset::locality there_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp new file mode 100644 index 000000000000..6e75ac5e5482 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp @@ -0,0 +1,61 @@ +// Copyright (c) 2014-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include + +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct tag_provider + { + tag_provider() noexcept + : next_tag_(2) + { + } + + int acquire() noexcept + { + int tag = -1; + std::lock_guard l(mtx_); + if (free_tags_.empty()) + { + HPX_ASSERT(next_tag_ < (std::numeric_limits::max)()); + tag = next_tag_++; + } + else + { + tag = free_tags_.front(); + free_tags_.pop_front(); + } + HPX_ASSERT(tag > 1); + return tag; + } + + void release(int tag) + { + HPX_ASSERT(tag > 1); + std::lock_guard l(mtx_); + HPX_ASSERT(tag < next_tag_); + + free_tags_.push_back(tag); + } + + hpx::spinlock mtx_; + int next_tag_; + std::deque free_tags_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/src/locality.cpp b/libs/full/parcelport_openshmem/src/locality.cpp new file mode 100644 index 000000000000..e9e22c16acbe --- /dev/null +++ b/libs/full/parcelport_openshmem/src/locality.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2013-2014 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + void locality::save(serialization::output_archive& ar) const + { + ar << rank_; + } + + void locality::load(serialization::input_archive& ar) + { + ar >> rank_; + } + + std::ostream& operator<<(std::ostream& os, locality const& loc) noexcept + { + hpx::util::ios_flags_saver ifs(os); + os << loc.rank_; + return os; + } +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp new file mode 100644 index 000000000000..7915e9d9bb3a --- /dev/null +++ b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp @@ -0,0 +1,313 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2020 Google +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace hpx::parcelset { + + namespace policies::openshmem { + class HPX_EXPORT parcelport; + } // namespace policies::openshmem + + template <> + struct connection_handler_traits + { + using connection_type = policies::openshmem::sender_connection; + using send_early_parcel = std::true_type; + using do_background_work = std::true_type; + using send_immediate_parcels = std::false_type; + using is_connectionless = std::false_type; + + static constexpr const char* type() noexcept + { + return "openshmem"; + } + + static constexpr const char* pool_name() noexcept + { + return "parcel-pool-openshmem"; + } + + static constexpr const char* pool_name_postfix() noexcept + { + return "-openshmem"; + } + }; + + namespace policies::openshmem { + + int acquire_tag(sender* s) noexcept + { + return s->acquire_tag(); + } + + void add_connection( + sender* s, std::shared_ptr const& ptr) + { + s->add(ptr); + } + + class HPX_EXPORT parcelport : public parcelport_impl + { + using base_type = parcelport_impl; + + static parcelset::locality here() + { + return parcelset::locality( + locality(util::openshmem_environment::enabled() ? + util::openshmem_environment::rank() : + -1)); + } + + static std::size_t max_connections( + util::runtime_configuration const& ini) + { + return hpx::util::get_entry_as(ini, + "hpx.parcel.openshmem.max_connections", + HPX_PARCEL_MAX_CONNECTIONS); + } + + static std::size_t background_threads( + [[maybe_unused]] util::runtime_configuration const& ini) + { + /* + return hpx::util::get_entry_as(ini, + "hpx.parcel.openshmem.background_threads", + HPX_HAVE_PARCELPORT_OPENSHMEM_BACKGROUND_THREADS); + */ + return 1UL; + } + + public: + parcelport(util::runtime_configuration const& ini, + threads::policies::callback_notifier const& notifier) + : base_type(ini, here(), notifier) + , stopped_(false) + , receiver_(*this) + , background_threads_(background_threads(ini)) + { + } + + parcelport(parcelport const&) = delete; + parcelport(parcelport&&) = delete; + parcelport& operator=(parcelport const&) = delete; + parcelport& operator=(parcelport&&) = delete; + + ~parcelport() + { + util::openshmem_environment::finalize(); + } + + // Start the handling of connections. + bool do_run() + { + receiver_.run(); + sender_.run(); + + for (std::size_t i = 0; i != io_service_pool_.size(); ++i) + { + io_service_pool_.get_io_service(int(i)).post( + hpx::bind(&parcelport::io_service_work, this)); + } + return true; + } + + // Stop the handling of connections. + void do_stop() + { + while (do_background_work(0, parcelport_background_mode_all)) + { + if (threads::get_self_ptr()) + hpx::this_thread::suspend( + hpx::threads::thread_schedule_state::pending, + "openshmem::parcelport::do_stop"); + } + + bool expected = false; + if (stopped_.compare_exchange_strong(expected, true)) + { + stopped_ = true; + hpx::util::openshmem_environment::global_barrier(); + } + } + + /// Return the name of this locality + std::string get_locality_name() const override + { + return util::openshmem_environment::get_processor_name(); + } + + std::shared_ptr create_connection( + parcelset::locality const& l, error_code&) + { + int dest_rank = l.get().rank(); + return sender_.create_connection(dest_rank, this); + } + + parcelset::locality agas_locality( + util::runtime_configuration const&) const override + { + return parcelset::locality( + locality(util::openshmem_environment::enabled() ? 0 : -1)); + } + + parcelset::locality create_locality() const override + { + return parcelset::locality(locality()); + } + + bool background_work( + std::size_t num_thread, parcelport_background_mode mode) + { + if (stopped_.load(std::memory_order_acquire) || + num_thread >= background_threads_) + { + return false; + } + + bool has_work = false; + if (mode & parcelport_background_mode_send) + { + has_work = sender_.background_work(); + } + if (mode & parcelport_background_mode_receive) + { + has_work = receiver_.background_work() || has_work; + } + return has_work; + } + + private: + std::atomic stopped_; + + sender sender_; + receiver receiver_; + + void io_service_work() + { + std::size_t k = 0; + + // We only execute work on the IO service while HPX is starting + while (hpx::is_starting()) + { + bool has_work = sender_.background_work(); + has_work = receiver_.background_work() || has_work; + if (has_work) + { + k = 0; + } + else + { + ++k; + util::detail::yield_k(k, + "hpx::parcelset::policies::openshmem::parcelport::" + "io_service_work"); + } + } + } + + std::size_t background_threads_; + + void early_write_handler(std::error_code const& ec, parcel const& p) + { + if (ec) + { + // all errors during early parcel handling are fatal + std::exception_ptr exception = hpx::detail::get_exception( + hpx::exception(ec), "openshmem::early_write_handler", + __FILE__, __LINE__, + "error while handling early parcel: " + ec.message() + + "(" + std::to_string(ec.value()) + ")" + + parcelset::dump_parcel(p)); + + hpx::report_error(exception); + } + } + }; + } // namespace policies::openshmem +} // namespace hpx::parcelset + +namespace hpx::traits { + + // Inject additional configuration data into the factory registry for this + // type. This information ends up in the system wide configuration database + // under the plugin specific section: + // + // [hpx.parcel.openshmem] + // ... + // priority = 1 + // + template <> + struct plugin_config_data + { + static constexpr char const* priority() noexcept + { + return "1"; + } + + static void init( + int* argc, char*** argv, util::command_line_handling& cfg) + { + util::openshmem_environment::init(argc, argv, cfg.rtcfg_); + cfg.num_localities_ = + static_cast(util::openshmem_environment::size()); + cfg.node_ = + static_cast(util::openshmem_environment::rank()); + } + + // by default no additional initialization using the resource + // partitioner is required + static constexpr void init(hpx::resource::partitioner&) noexcept {} + + static void destroy() noexcept + { + util::openshmem_environment::finalize(); + } + + static constexpr char const* call() noexcept + { + return ""; + } + }; +} // namespace hpx::traits + +HPX_REGISTER_PARCELPORT( + hpx::parcelset::policies::openshmem::parcelport, openshmem) + +#endif diff --git a/libs/full/parcelport_openshmem/tests/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/CMakeLists.txt new file mode 100644 index 000000000000..5206a56dbe16 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) + +if(HPX_WITH_TESTS) + if(HPX_WITH_TESTS_UNIT) + add_hpx_pseudo_target(tests.unit.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.unit.modules tests.unit.modules.parcelport_openshmem + ) + add_subdirectory(unit) + endif() + + if(HPX_WITH_TESTS_REGRESSIONS) + add_hpx_pseudo_target(tests.regressions.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.regressions.modules tests.regressions.modules.parcelport_openshmem + ) + add_subdirectory(regressions) + endif() + + if(HPX_WITH_TESTS_BENCHMARKS) + add_hpx_pseudo_target(tests.performance.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.performance.modules tests.performance.modules.parcelport_openshmem + ) + add_subdirectory(performance) + endif() + + if(HPX_WITH_TESTS_HEADERS) + add_hpx_header_tests( + modules.parcelport_openshmem + HEADERS ${parcelport_openshmem_headers} + HEADER_ROOT ${PROJECT_SOURCE_DIR}/include + DEPENDENCIES hpx_parcelport_openshmem + ) + endif() +endif() diff --git a/libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)