diff --git a/ompi/mca/coll/base/coll_base_util.c b/ompi/mca/coll/base/coll_base_util.c index 1a1fb8961f4..cb7c2dd12cc 100644 --- a/ompi/mca/coll/base/coll_base_util.c +++ b/ompi/mca/coll/base/coll_base_util.c @@ -9,8 +9,8 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,6 +29,27 @@ #include "ompi/mca/pml/pml.h" #include "coll_base_util.h" +struct retain_op_data { + ompi_request_complete_fn_t req_complete_cb; + void *req_complete_cb_data; + ompi_op_t *op; + ompi_datatype_t *datatype; +}; + +struct retain_datatypes_data { + ompi_request_complete_fn_t req_complete_cb; + void *req_complete_cb_data; + ompi_datatype_t *stype; + ompi_datatype_t *rtype; +}; + +struct retain_datatypes_w_data { + ompi_request_complete_fn_t req_complete_cb; + void *req_complete_cb_data; + int count; + ompi_datatype_t *types[]; +}; + int ompi_coll_base_sendrecv_actual( const void* sendbuf, size_t scount, ompi_datatype_t* sdatatype, int dest, int stag, @@ -103,3 +124,142 @@ int ompi_rounddown(int num, int factor) num /= factor; return num * factor; /* floor(num / factor) * factor */ } + +static int release_op_callback(struct ompi_request_t *request) { + struct retain_op_data * p = (struct retain_op_data *)request->req_complete_cb_data; + int rc = OMPI_SUCCESS; + assert (NULL != p); + if (NULL != p->req_complete_cb) { + request->req_complete_cb = p->req_complete_cb; + request->req_complete_cb_data = p->req_complete_cb_data; + rc = request->req_complete_cb(request); + } + if (NULL != p->op) { + OBJ_RELEASE(p->op); + } + if (NULL != p->datatype) { + OBJ_RELEASE(p->datatype); + } + free(p); + return rc; +} + +int ompi_coll_base_retain_op( ompi_request_t *request, ompi_op_t *op, + ompi_datatype_t *type) { + bool retain = !ompi_op_is_intrinsic(op); + retain |= !ompi_datatype_is_predefined(type); + if (OPAL_UNLIKELY(retain)) { + struct retain_op_data *p = (struct retain_op_data *)calloc(1, sizeof(struct retain_op_data)); + if (OPAL_UNLIKELY(NULL == p)) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + if (!ompi_op_is_intrinsic(op)) { + OBJ_RETAIN(op); + p->op = op; + } + if (!ompi_datatype_is_predefined(type)) { + OBJ_RETAIN(type); + p->datatype = type; + } + p->req_complete_cb = request->req_complete_cb; + p->req_complete_cb_data = request->req_complete_cb_data; + request->req_complete_cb = release_op_callback; + request->req_complete_cb_data = p; + } + return OMPI_SUCCESS; +} + +static int release_datatypes_callback(struct ompi_request_t *request) { + struct retain_datatypes_data * p = (struct retain_datatypes_data *)request->req_complete_cb_data; + int rc = OMPI_SUCCESS; + assert (NULL != p); + if (NULL != p->req_complete_cb) { + request->req_complete_cb = p->req_complete_cb; + request->req_complete_cb_data = p->req_complete_cb_data; + rc = request->req_complete_cb(request); + } + if (NULL != p->stype) { + OBJ_RELEASE(p->stype); + } + if (NULL != p->rtype) { + OBJ_RELEASE(p->rtype); + } + free(p); + return rc; +} + +int ompi_coll_base_retain_datatypes( ompi_request_t *request, ompi_datatype_t *stype, + ompi_datatype_t *rtype) { + bool retain = NULL != stype && !ompi_datatype_is_predefined(stype); + retain |= NULL != rtype && !ompi_datatype_is_predefined(rtype); + if (OPAL_UNLIKELY(retain)) { + struct retain_datatypes_data *p = (struct retain_datatypes_data *)calloc(1, sizeof(struct retain_datatypes_data)); + if (OPAL_UNLIKELY(NULL == p)) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + if (NULL != stype && !ompi_datatype_is_predefined(stype)) { + OBJ_RETAIN(stype); + p->stype = stype; + } + if (NULL != rtype && !ompi_datatype_is_predefined(rtype)) { + OBJ_RETAIN(rtype); + p->rtype = rtype; + } + p->req_complete_cb = request->req_complete_cb; + p->req_complete_cb_data = request->req_complete_cb_data; + request->req_complete_cb = release_datatypes_callback; + request->req_complete_cb_data = p; + } + return OMPI_SUCCESS; +} + +static int release_datatypes_w_callback(struct ompi_request_t *request) { + struct retain_datatypes_w_data * p = (struct retain_datatypes_w_data *)request->req_complete_cb_data; + int rc = OMPI_SUCCESS; + assert (NULL != p); + if (NULL != p->req_complete_cb) { + request->req_complete_cb = p->req_complete_cb; + request->req_complete_cb_data = p->req_complete_cb_data; + rc = request->req_complete_cb(request); + } + for (int i=0; icount; i++) { + OBJ_RELEASE(p->types[i]); + } + free(p); + return rc; +} + +int ompi_coll_base_retain_datatypes_w( ompi_request_t *request, int count, + ompi_datatype_t *const stypes[], ompi_datatype_t *const rtypes[]) { + int datatypes = 0; + for (int i=0; itypes[datatypes++] = stypes[i]; + OBJ_RETAIN(stypes[i]); + } + if (NULL != rtypes[i] && !ompi_datatype_is_predefined(rtypes[i])) { + p->types[datatypes++] = rtypes[i]; + OBJ_RETAIN(rtypes[i]); + } + } + p->req_complete_cb = request->req_complete_cb; + p->req_complete_cb_data = request->req_complete_cb_data; + request->req_complete_cb = release_datatypes_w_callback; + request->req_complete_cb_data = p; + } + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/base/coll_base_util.h b/ompi/mca/coll/base/coll_base_util.h index 8306b8fe83d..21eb6801087 100644 --- a/ompi/mca/coll/base/coll_base_util.h +++ b/ompi/mca/coll/base/coll_base_util.h @@ -9,8 +9,8 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -27,6 +27,7 @@ #include "ompi/mca/mca.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/request/request.h" +#include "ompi/op/op.h" #include "ompi/mca/pml/pml.h" BEGIN_C_DECLS @@ -84,5 +85,15 @@ unsigned int ompi_mirror_perm(unsigned int x, int nbits); */ int ompi_rounddown(int num, int factor); +int ompi_coll_base_retain_op( ompi_request_t *request, ompi_op_t *op, + ompi_datatype_t *type); + +int ompi_coll_base_retain_datatypes( ompi_request_t *request, ompi_datatype_t *stype, + ompi_datatype_t *rtype); + +int ompi_coll_base_retain_datatypes_w( ompi_request_t *request, int count, + ompi_datatype_t *const stypes[], + ompi_datatype_t *const rtypes[]); + END_C_DECLS #endif /* MCA_COLL_BASE_UTIL_EXPORT_H */ diff --git a/ompi/mpi/c/iallgather.c b/ompi/mpi/c/iallgather.c index 7d2740b6512..806cdc14675 100644 --- a/ompi/mpi/c/iallgather.c +++ b/ompi/mpi/c/iallgather.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -102,6 +103,9 @@ int MPI_Iallgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_iallgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request, comm->c_coll->coll_iallgather_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iallgatherv.c b/ompi/mpi/c/iallgatherv.c index 0373a15b1d3..6cd7600f49c 100644 --- a/ompi/mpi/c/iallgatherv.c +++ b/ompi/mpi/c/iallgatherv.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -126,6 +127,9 @@ int MPI_Iallgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, recvbuf, recvcounts, displs, recvtype, comm, request, comm->c_coll->coll_iallgatherv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iallreduce.c b/ompi/mpi/c/iallreduce.c index d0ea511cf84..bfa968c55b4 100644 --- a/ompi/mpi/c/iallreduce.c +++ b/ompi/mpi/c/iallreduce.c @@ -12,8 +12,8 @@ * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -112,10 +113,11 @@ int MPI_Iallreduce(const void *sendbuf, void *recvbuf, int count, /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_iallreduce(sendbuf, recvbuf, count, datatype, op, comm, request, comm->c_coll->coll_iallreduce_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ialltoall.c b/ompi/mpi/c/ialltoall.c index 2d46b76f38f..6aad902071b 100644 --- a/ompi/mpi/c/ialltoall.c +++ b/ompi/mpi/c/ialltoall.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -101,5 +102,8 @@ int MPI_Ialltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_ialltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request, comm->c_coll->coll_ialltoall_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ialltoallv.c b/ompi/mpi/c/ialltoallv.c index 577b3828949..7f74e44631d 100644 --- a/ompi/mpi/c/ialltoallv.c +++ b/ompi/mpi/c/ialltoallv.c @@ -13,8 +13,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -130,6 +131,9 @@ int MPI_Ialltoallv(const void *sendbuf, const int sendcounts[], const int sdispl err = comm->c_coll->coll_ialltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, request, comm->c_coll->coll_ialltoallv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ialltoallw.c b/ompi/mpi/c/ialltoallw.c index b7bc86eaa7d..fd7024207fd 100644 --- a/ompi/mpi/c/ialltoallw.c +++ b/ompi/mpi/c/ialltoallw.c @@ -13,8 +13,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -127,6 +128,12 @@ int MPI_Ialltoallw(const void *sendbuf, const int sendcounts[], const int sdispl sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, request, comm->c_coll->coll_ialltoallw_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes_w(*request, + OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm), + sendtypes, + recvtypes); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ibcast.c b/ompi/mpi/c/ibcast.c index 1f049b4c6de..698e125c7ff 100644 --- a/ompi/mpi/c/ibcast.c +++ b/ompi/mpi/c/ibcast.c @@ -1,7 +1,7 @@ /* * Copyright (c) 2012 Oak Rigde National Laboratory. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017-2018 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. @@ -19,6 +19,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -86,5 +87,8 @@ int MPI_Ibcast(void *buffer, int count, MPI_Datatype datatype, err = comm->c_coll->coll_ibcast(buffer, count, datatype, root, comm, request, comm->c_coll->coll_ibcast_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, datatype, NULL); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iexscan.c b/ompi/mpi/c/iexscan.c index 14cf23c590b..4c56e08f1e4 100644 --- a/ompi/mpi/c/iexscan.c +++ b/ompi/mpi/c/iexscan.c @@ -12,8 +12,8 @@ * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,6 +29,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -82,10 +83,11 @@ int MPI_Iexscan(const void *sendbuf, void *recvbuf, int count, /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_iexscan(sendbuf, recvbuf, count, datatype, op, comm, request, comm->c_coll->coll_iexscan_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/igather.c b/ompi/mpi/c/igather.c index 3fcda7e8069..ea0a97aee64 100644 --- a/ompi/mpi/c/igather.c +++ b/ompi/mpi/c/igather.c @@ -15,8 +15,8 @@ * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -173,5 +174,8 @@ int MPI_Igather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_igather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, request, comm->c_coll->coll_igather_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/igatherv.c b/ompi/mpi/c/igatherv.c index e2deab3cc9f..b6a237d8fbf 100644 --- a/ompi/mpi/c/igatherv.c +++ b/ompi/mpi/c/igatherv.c @@ -13,7 +13,7 @@ * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science + * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -29,6 +29,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -196,5 +197,8 @@ int MPI_Igatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_igatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm, request, comm->c_coll->coll_igatherv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ineighbor_allgather.c b/ompi/mpi/c/ineighbor_allgather.c index 2706ea44d4a..cba5b5d4e36 100644 --- a/ompi/mpi/c/ineighbor_allgather.c +++ b/ompi/mpi/c/ineighbor_allgather.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Rigde National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -124,6 +125,9 @@ int MPI_Ineighbor_allgather(const void *sendbuf, int sendcount, MPI_Datatype sen err = comm->c_coll->coll_ineighbor_allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request, comm->c_coll->coll_ineighbor_allgather_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ineighbor_allgatherv.c b/ompi/mpi/c/ineighbor_allgatherv.c index 2f3c244064c..58dedb61057 100644 --- a/ompi/mpi/c/ineighbor_allgatherv.c +++ b/ompi/mpi/c/ineighbor_allgatherv.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -147,6 +148,9 @@ int MPI_Ineighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype se recvbuf, (int *) recvcounts, (int *) displs, recvtype, comm, request, comm->c_coll->coll_ineighbor_allgatherv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ineighbor_alltoall.c b/ompi/mpi/c/ineighbor_alltoall.c index b3d0846421e..b03b7cc50fa 100644 --- a/ompi/mpi/c/ineighbor_alltoall.c +++ b/ompi/mpi/c/ineighbor_alltoall.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -124,5 +125,8 @@ int MPI_Ineighbor_alltoall(const void *sendbuf, int sendcount, MPI_Datatype send err = comm->c_coll->coll_ineighbor_alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request, comm->c_coll->coll_ineighbor_alltoall_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ineighbor_alltoallv.c b/ompi/mpi/c/ineighbor_alltoallv.c index 9645e15b05d..95518cf1c43 100644 --- a/ompi/mpi/c/ineighbor_alltoallv.c +++ b/ompi/mpi/c/ineighbor_alltoallv.c @@ -13,8 +13,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -147,6 +148,9 @@ int MPI_Ineighbor_alltoallv(const void *sendbuf, const int sendcounts[], const i err = comm->c_coll->coll_ineighbor_alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, request, comm->c_coll->coll_ineighbor_alltoallv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ineighbor_alltoallw.c b/ompi/mpi/c/ineighbor_alltoallw.c index 150f28d7173..0841f632aa7 100644 --- a/ompi/mpi/c/ineighbor_alltoallw.c +++ b/ompi/mpi/c/ineighbor_alltoallw.c @@ -13,8 +13,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -147,6 +148,12 @@ int MPI_Ineighbor_alltoallw(const void *sendbuf, const int sendcounts[], const M err = comm->c_coll->coll_ineighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, request, comm->c_coll->coll_ineighbor_alltoallw_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes_w(*request, + OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm), + sendtypes, + recvtypes); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ireduce.c b/ompi/mpi/c/ireduce.c index 47948887824..be552250fce 100644 --- a/ompi/mpi/c/ireduce.c +++ b/ompi/mpi/c/ireduce.c @@ -13,8 +13,8 @@ * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -136,10 +137,11 @@ int MPI_Ireduce(const void *sendbuf, void *recvbuf, int count, OPAL_CR_ENTER_LIBRARY(); /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_ireduce(sendbuf, recvbuf, count, datatype, op, root, comm, request, comm->c_coll->coll_ireduce_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ireduce_scatter.c b/ompi/mpi/c/ireduce_scatter.c index 211b217971e..56525fa19f7 100644 --- a/ompi/mpi/c/ireduce_scatter.c +++ b/ompi/mpi/c/ireduce_scatter.c @@ -13,8 +13,8 @@ * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -133,10 +134,11 @@ int MPI_Ireduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_ireduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, comm, request, comm->c_coll->coll_ireduce_scatter_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ireduce_scatter_block.c b/ompi/mpi/c/ireduce_scatter_block.c index ded4abf2232..ce43ab3cd4f 100644 --- a/ompi/mpi/c/ireduce_scatter_block.c +++ b/ompi/mpi/c/ireduce_scatter_block.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -100,10 +101,11 @@ int MPI_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount, /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_ireduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, op, comm, request, comm->c_coll->coll_ireduce_scatter_block_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iscan.c b/ompi/mpi/c/iscan.c index 34502b8e366..cfae0ff409a 100644 --- a/ompi/mpi/c/iscan.c +++ b/ompi/mpi/c/iscan.c @@ -13,8 +13,8 @@ * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -96,11 +97,12 @@ int MPI_Iscan(const void *sendbuf, void *recvbuf, int count, /* Call the coll component to actually perform the allgather */ - OBJ_RETAIN(op); err = comm->c_coll->coll_iscan(sendbuf, recvbuf, count, datatype, op, comm, request, comm->c_coll->coll_iscan_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iscatter.c b/ompi/mpi/c/iscatter.c index 79a22d57a52..b3dee5a1e7f 100644 --- a/ompi/mpi/c/iscatter.c +++ b/ompi/mpi/c/iscatter.c @@ -15,8 +15,8 @@ * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -156,5 +157,8 @@ int MPI_Iscatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_iscatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, request, comm->c_coll->coll_iscatter_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iscatterv.c b/ompi/mpi/c/iscatterv.c index 66ae9003caa..782284e8590 100644 --- a/ompi/mpi/c/iscatterv.c +++ b/ompi/mpi/c/iscatterv.c @@ -13,8 +13,8 @@ * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,6 +29,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -196,5 +197,8 @@ int MPI_Iscatterv(const void *sendbuf, const int sendcounts[], const int displs[ err = comm->c_coll->coll_iscatterv(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, request, comm->c_coll->coll_iscatterv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); }