diff --git a/src/tools/perf/libperf.c b/src/tools/perf/libperf.c index 268cb7230e8..1acf10b4ec2 100644 --- a/src/tools/perf/libperf.c +++ b/src/tools/perf/libperf.c @@ -756,6 +756,7 @@ static ucs_status_t ucp_perf_test_alloc_mem(ucx_perf_context_t *perf, ucx_perf_p { ucs_status_t status; ucp_mem_map_params_t mem_map_params; + ucp_mem_attr_t mem_attr; size_t buffer_size; if (params->iov_stride) { @@ -774,13 +775,21 @@ static ucs_status_t ucp_perf_test_alloc_mem(ucx_perf_context_t *perf, ucx_perf_p mem_map_params.length = buffer_size * params->thread_count; mem_map_params.flags = (params->flags & UCX_PERF_TEST_FLAG_MAP_NONBLOCK) ? UCP_MEM_MAP_NONBLOCK : 0; + mem_map_params.flags |= UCP_MEM_MAP_ALLOCATE; status = ucp_mem_map(perf->ucp.context, &mem_map_params, &perf->ucp.send_memh); if (status != UCS_OK) { goto err; } - perf->send_buffer = mem_map_params.address; + + mem_attr.field_mask = UCP_MEM_ATTR_FIELD_ADDRESS; + status = ucp_mem_query(perf->ucp.send_memh, &mem_attr); + if (status != UCS_OK) { + goto err; + } + + perf->send_buffer = mem_attr.address; /* Allocate receive buffer memory */ perf->recv_buffer = NULL; @@ -790,13 +799,20 @@ static ucs_status_t ucp_perf_test_alloc_mem(ucx_perf_context_t *perf, ucx_perf_p UCP_MEM_MAP_PARAM_FIELD_FLAGS; mem_map_params.address = perf->recv_buffer; mem_map_params.length = buffer_size * params->thread_count; - mem_map_params.flags = 0; + mem_map_params.flags = UCP_MEM_MAP_ALLOCATE; status = ucp_mem_map(perf->ucp.context, &mem_map_params, &perf->ucp.recv_memh); if (status != UCS_OK) { goto err_free_send_buffer; } - perf->recv_buffer = mem_map_params.address; + + mem_attr.field_mask = UCP_MEM_ATTR_FIELD_ADDRESS; + status = ucp_mem_query(perf->ucp.recv_memh, &mem_attr); + if (status != UCS_OK) { + goto err_free_send_buffer; + } + + perf->recv_buffer = mem_attr.address; /* Allocate IOV datatype memory */ perf->params.msg_size_cnt = params->msg_size_cnt; @@ -1411,4 +1427,3 @@ static int ucx_perf_thread_spawn(ucx_perf_context_t *perf, return UCS_ERR_INVALID_PARAM; } #endif /* _OPENMP */ - diff --git a/src/ucp/api/ucp.h b/src/ucp/api/ucp.h index 4754ceea085..f6aeed6ab66 100644 --- a/src/ucp/api/ucp.h +++ b/src/ucp/api/ucp.h @@ -175,11 +175,15 @@ enum ucp_ep_params_field { * present. It is used for the enablement of backward compatibility support. */ enum ucp_mem_map_params_field { - UCP_MEM_MAP_PARAM_FIELD_ADDRESS = UCS_BIT(0), /**< Address of remote peer */ + UCP_MEM_MAP_PARAM_FIELD_ADDRESS = UCS_BIT(0), /**< Address of the memory that + would be used in the + @ref ucp_mem_map routine, + see @ref ucp_mem_map_matrix + for details */ UCP_MEM_MAP_PARAM_FIELD_LENGTH = UCS_BIT(1), /**< The size of memory that would be allocated or registered in the - ucp_mem_map routine.*/ + @ref ucp_mem_map routine.*/ UCP_MEM_MAP_PARAM_FIELD_FLAGS = UCS_BIT(2) /**< Allocation flags */ }; @@ -247,11 +251,19 @@ enum ucp_dt_type { * ucp_mem_map() function. */ enum { - UCP_MEM_MAP_NONBLOCK = UCS_BIT(0) /**< Complete the mapping faster, possibly by - not populating the pages in the mapping - up-front, and mapping them later when - they are accessed by communication - routines. */ + UCP_MEM_MAP_NONBLOCK = UCS_BIT(0), /**< Complete the mapping faster, possibly by + not populating the pages in the mapping + up-front, and mapping them later when + they are accessed by communication + routines. */ + UCP_MEM_MAP_ALLOCATE = UCS_BIT(1), /**< Identify requirement for allocation, + if passed address is not a null-pointer + then it will be used as a hint or direct + address for allocation. */ + UCP_MEM_MAP_FIXED = UCS_BIT(2) /**< Don't interpret address as a hint: + place the mapping at exactly that + address. The address must be a multiple + of the page size. */ }; @@ -667,7 +679,7 @@ typedef struct ucp_mem_map_params { * segment and returns its address in this argument. * Therefore, this value is optional. * If it's not set (along with its corresponding bit in the field_mask - - * UCP_MEM_MAP_PARAM_FIELD_ADDRESS), the ucp_mem_map routine will consider + * @ref UCP_MEM_MAP_PARAM_FIELD_ADDRESS), the ucp_mem_map routine will consider * address as set to NULL and will allocate memory. */ void *address; @@ -675,18 +687,18 @@ typedef struct ucp_mem_map_params { /** * Length (in bytes) to allocate or map (register). * This field is mandatory for filling (along with its corresponding bit - * in the field_mask - UCP_MEM_MAP_PARAM_FIELD_LENGTH). - * The ucp_mem_map routine will return with an error if the length isn't + * in the field_mask - @ref UCP_MEM_MAP_PARAM_FIELD_LENGTH). + * The @ref ucp_mem_map routine will return with an error if the length isn't * specified. */ size_t length; /** - * Allocation flags, e.g. @ref UCP_MEM_MAP_NONBLOCK "UCP_MEM_MAP_NONBLOCK". + * Allocation flags, e.g. @ref UCP_MEM_MAP_NONBLOCK. * This value is optional. * If it's not set (along with its corresponding bit in the field_mask - - * UCP_MEM_MAP_PARAM_FIELD_FLAGS), the ucp_mem_map routine will consider - * the flags as set to zero. + * @ref UCP_MEM_MAP_PARAM_FIELD_FLAGS), the @ref ucp_mem_map routine will + * consider the flags as set to zero. */ unsigned flags; } ucp_mem_map_params_t; @@ -1299,6 +1311,39 @@ ucs_status_t ucp_ep_flush(ucp_ep_h ep); * by a user, then the user is responsible for segmentation and subsequent * management. * + * + * + * + * + * + * + * + * + * + * + *
Matrix of behavior
parameter/flag @ref UCP_MEM_MAP_NONBLOCK "NONBLOCK"@ref UCP_MEM_MAP_ALLOCATE "ALLOCATE"@ref UCP_MEM_MAP_FIXED "FIXED"@ref ucp_mem_map_params.address "address"@b result + *
@b value 0/1 - the value\n only affects the\n register/map\n phase0 0 0 @ref anch_err "error" + *
1 0 0 @ref anch_alloc_reg "alloc+register" + *
0 1 0 @ref anch_err "error"
0 0 defined @ref anch_reg "register" + *
1 1 0 @ref anch_err "error"
1 0 defined @ref anch_alloc_hint_reg "alloc+register,hint" + *
0 1 defined @ref anch_err "error"
1 1 defined @ref anch_alloc_fixed_reg "alloc+register,fixed" + *
+ * + * @note + * @li \anchor anch_reg @b register means that the memory will be registered in + * corresponding transports for RMA/AMO operations. This case intends that + * the memory was allocated by user before. + * @li \anchor anch_alloc_reg @b alloc+register means that the memory will be allocated + * in the memory provided by the system and registered in corresponding + * transports for RMA/AMO operations. + * @li \anchor anch_alloc_hint_reg alloc+register,hint means that + * the memory will be allocated with using @ref ucp_mem_map_params.address + * as a hint and registered in corresponding transports for RMA/AMO operations. + * @li \anchor anch_alloc_fixed_reg alloc+register,fixed means that the memory + * will be allocated and registered in corresponding transports for RMA/AMO + * operations. + * @li \anchor anch_err @b error is an erroneous combination of the parameters. + * * @param [in] context Application @ref ucp_context_h "context" to map * (register) and allocate the memory on. * @param [in] params User defined @ref ucp_mem_map_params_t configurations @@ -1308,7 +1353,7 @@ ucs_status_t ucp_ep_flush(ucp_ep_h ep); * * @return Error code as defined by @ref ucs_status_t */ -ucs_status_t ucp_mem_map(ucp_context_h context, ucp_mem_map_params_t *params, +ucs_status_t ucp_mem_map(ucp_context_h context, const ucp_mem_map_params_t *params, ucp_mem_h *memh_p); @@ -1342,6 +1387,22 @@ ucs_status_t ucp_mem_map(ucp_context_h context, ucp_mem_map_params_t *params, ucs_status_t ucp_mem_unmap(ucp_context_h context, ucp_mem_h memh); +/** + * @ingroup UCP_MEM + * @brief query mapped memory segment + * + * This routine returns address and length of memory segment mapped with + * @ref ucp_mem_map "ucp_mem_map()" routine. + * + * @param [in] memh @ref ucp_mem_h "Handle" to memory region. + * @param [out] attr Filled with attributes of the @ref ucp_mem_h + * "UCP memory handle". + * + * @return Error code as defined by @ref ucs_status_t + */ +ucs_status_t ucp_mem_query(const ucp_mem_h memh, ucp_mem_attr_t *attr); + + /** * @ingroup UCP_MEM * @brief list of UCP memory use advice. diff --git a/src/ucp/api/ucp_def.h b/src/ucp/api/ucp_def.h index 63a82e345d1..1d00c44e6c6 100644 --- a/src/ucp/api/ucp_def.h +++ b/src/ucp/api/ucp_def.h @@ -114,6 +114,44 @@ typedef struct ucp_rkey *ucp_rkey_h; typedef struct ucp_mem *ucp_mem_h; +/** + * @ingroup UCP_MEM + * @brief Attributes of the @ref ucp_mem_h "UCP Memory handle", filled by + * @ref ucp_mem_query function. + */ +typedef struct ucp_mem_attr { + /** + * Mask of valid fields in this structure, using bits from @ref ucp_mem_attr_field. + * Fields not specified in this mask would be ignored. + * Provides ABI compatibility with respect to adding new fields. + */ + uint64_t field_mask; + + /** + * Address of the memory segment. + */ + void *address; + + /** + * Size of the memory segment. + */ + size_t length; +} ucp_mem_attr_t; + + +/** + * @ingroup UCP_MEM + * @brief UCP Memory handle attributes field mask. + * + * The enumeration allows specifying which fields in @ref ucp_mem_attr_t are + * present. It is used for the enablement of backward compatibility support. + */ +enum ucp_mem_attr_field { + UCP_MEM_ATTR_FIELD_ADDRESS = UCS_BIT(0), /**< Virtual address */ + UCP_MEM_ATTR_FIELD_LENGTH = UCS_BIT(1) /**< The size of memory region */ +}; + + /** * @ingroup UCP_WORKER * @brief UCP Worker diff --git a/src/ucp/core/ucp_mm.c b/src/ucp/core/ucp_mm.c index 87eda078d70..d7e8e0a3e9a 100644 --- a/src/ucp/core/ucp_mm.c +++ b/src/ucp/core/ucp_mm.c @@ -5,6 +5,7 @@ */ #include "ucp_mm.h" +#include "ucs/sys/sys.h" #include #include @@ -150,9 +151,8 @@ static ucs_status_t ucp_mem_alloc(ucp_context_h context, size_t length, } } - /* TODO: pass addr */ - status = uct_mem_alloc(NULL, length, uct_flags, &method, 1, mds, num_mds, - name, &mem); + status = uct_mem_alloc(memh->address, length, uct_flags, &method, 1, mds, + num_mds, name, &mem); if (status == UCS_OK) { goto allocated; } @@ -178,24 +178,100 @@ static ucs_status_t ucp_mem_alloc(ucp_context_h context, size_t length, } -ucs_status_t ucp_mem_map(ucp_context_h context, ucp_mem_map_params_t *params, +static inline unsigned +ucp_mem_map_params2uct_flags(ucp_mem_map_params_t *params) +{ + unsigned flags = 0; + + if (params->field_mask & UCP_MEM_MAP_PARAM_FIELD_FLAGS) { + if (params->flags & UCP_MEM_MAP_NONBLOCK) { + flags |= UCT_MD_MEM_FLAG_NONBLOCK; + } + + if (params->flags & UCP_MEM_MAP_FIXED) { + flags |= UCT_MD_MEM_FLAG_FIXED; + } + } + + return flags; +} + +/* Matrix of behavior + * |-----------------------------------------------------------------------------| + * | parameter | value | + * |-----------|-----------------------------------------------------------------| + * | ALLOCATE | 0 | 1 | 0 | 0 | 1 | 1 | 0 | 1 | + * | FIXED | 0 | 0 | 1 | 0 | 1 | 0 | 1 | 1 | + * | addr | 0 | 0 | 0 | 1 | 0 | 1 | 1 | 1 | + * |-----------|-----|-----------|-----|-----|-----|-----------|-----|-----------| + * | result | err | alloc/reg | err | reg | err | alloc/reg | err | alloc/reg | + * | | | | | | | (hint) | | (fixed) | + * |-----------------------------------------------------------------------------| + */ +static inline ucs_status_t ucp_mem_map_check_and_adjust_params(ucp_mem_map_params_t *params) +{ + if (!(params->field_mask & UCP_MEM_MAP_PARAM_FIELD_LENGTH)) { + ucs_error("The length value for mapping memory isn't set: %s", + ucs_status_string(UCS_ERR_INVALID_PARAM)); + return UCS_ERR_INVALID_PARAM; + } + + /* First of all, define all fields */ + if (!(params->field_mask & UCP_MEM_MAP_PARAM_FIELD_ADDRESS)) { + params->field_mask |= UCP_MEM_MAP_PARAM_FIELD_ADDRESS; + params->address = NULL; + } + + if (!(params->field_mask & UCP_MEM_MAP_PARAM_FIELD_FLAGS)) { + params->field_mask |= UCP_MEM_MAP_PARAM_FIELD_FLAGS; + params->flags = 0; + } + + if ((params->flags & UCP_MEM_MAP_FIXED) && + (!params->address || + ((uintptr_t)params->address % ucs_get_page_size()))) { + ucs_error("UCP_MEM_MAP_FIXED flag requires page aligned address"); + return UCS_ERR_INVALID_PARAM; + } + + /* Now, lets check the rest of erroneous cases from the matrix */ + if (params->address == NULL) { + if (!(params->flags & UCP_MEM_MAP_ALLOCATE)) { + ucs_error("Undefined address requires UCP_MEM_MAP_ALLOCATE flag"); + return UCS_ERR_INVALID_PARAM; + } + } else if (!(params->flags & UCP_MEM_MAP_ALLOCATE) && + (params->flags & UCP_MEM_MAP_FIXED)) { + ucs_error("Wrong combination of flags when address is defined"); + return UCS_ERR_INVALID_PARAM; + } + + return UCS_OK; +} + +static inline int ucp_mem_map_is_allocate(ucp_mem_map_params_t *params) +{ + return (params->field_mask & UCP_MEM_MAP_PARAM_FIELD_FLAGS) && + (params->flags & UCP_MEM_MAP_ALLOCATE); +} + +ucs_status_t ucp_mem_map(ucp_context_h context, const ucp_mem_map_params_t *params, ucp_mem_h *memh_p) { - ucs_status_t status; - unsigned uct_flags; - ucp_mem_h memh; + ucs_status_t status; + ucp_mem_h memh; + ucp_mem_map_params_t mem_params; /* always acquire context lock */ UCP_THREAD_CS_ENTER(&context->mt_lock); - if (!(params->field_mask & UCP_MEM_MAP_PARAM_FIELD_LENGTH)) { - status = UCS_ERR_INVALID_PARAM; - ucs_error("The length value for mapping memory isn't set: %s", - ucs_status_string(status)); + mem_params = *params; + status = ucp_mem_map_check_and_adjust_params(&mem_params); + if (status != UCS_OK) { goto out; } - if (params->length == 0) { + if (mem_params.length == 0) { ucs_debug("mapping zero length buffer, return dummy memh"); *memh_p = &ucp_mem_dummy_handle; status = UCS_OK; @@ -211,30 +287,26 @@ ucs_status_t ucp_mem_map(ucp_context_h context, ucp_mem_map_params_t *params, goto out; } - uct_flags = 0; - if ((params->field_mask & UCP_MEM_MAP_PARAM_FIELD_FLAGS) && - (params->flags & UCP_MEM_MAP_NONBLOCK)) { - uct_flags |= UCT_MD_MEM_FLAG_NONBLOCK; - } + memh->address = mem_params.address; + memh->length = mem_params.length; - if (!(params->field_mask & UCP_MEM_MAP_PARAM_FIELD_ADDRESS)) { - params->address = NULL; - } - - if (params->address == NULL) { - status = ucp_mem_alloc(context, params->length, uct_flags, "user allocation", memh); + if (ucp_mem_map_is_allocate(&mem_params)) { + ucs_debug("allocation user memory at %p length %zu", mem_params.address, + mem_params.length); + status = ucp_mem_alloc(context, mem_params.length, + ucp_mem_map_params2uct_flags(&mem_params), + "user allocation", memh); if (status != UCS_OK) { goto err_free_memh; } - - params->address = memh->address; } else { - ucs_debug("registering user memory at %p length %zu", params->address, params->length); - memh->address = params->address; - memh->length = params->length; + ucs_debug("registering user memory at %p length %zu", mem_params.address, + mem_params.length); memh->alloc_method = UCT_ALLOC_METHOD_LAST; memh->alloc_md = NULL; - status = ucp_memh_reg_mds(context, memh, uct_flags, UCT_MEM_HANDLE_NULL); + status = ucp_memh_reg_mds(context, memh, + ucp_mem_map_params2uct_flags(&mem_params), + UCT_MEM_HANDLE_NULL); if (status != UCS_OK) { goto err_free_memh; } @@ -296,6 +368,19 @@ ucs_status_t ucp_mem_unmap(ucp_context_h context, ucp_mem_h memh) return status; } +ucs_status_t ucp_mem_query(const ucp_mem_h memh, ucp_mem_attr_t *attr) +{ + if (attr->field_mask & UCP_MEM_ATTR_FIELD_ADDRESS) { + attr->address = memh->address; + } + + if (attr->field_mask & UCP_MEM_ATTR_FIELD_LENGTH) { + attr->length = memh->length; + } + + return UCS_OK; +} + static ucs_status_t ucp_advice2uct(unsigned ucp_advice, unsigned *uct_advice) { switch(ucp_advice) { diff --git a/test/gtest/ucp/test_ucp_fence.cc b/test/gtest/ucp/test_ucp_fence.cc index 7ccfa67ab09..3c0fda2f625 100644 --- a/test/gtest/ucp/test_ucp_fence.cc +++ b/test/gtest/ucp/test_ucp_fence.cc @@ -131,8 +131,9 @@ class test_ucp_fence : public test_ucp_atomic { ucs_status_t status; ucp_mem_map_params_t params; + ucp_mem_attr_t mem_attr; ucp_mem_h memh; - void *memheap; + void *memheap = NULL; void *rkey_buffer; size_t rkey_buffer_size; @@ -145,18 +146,31 @@ class test_ucp_fence : public test_ucp_atomic { receiver().connect(&sender()); } - memheap = malloc(memheap_size); - params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH | UCP_MEM_MAP_PARAM_FIELD_FLAGS; - params.address = memheap; params.length = memheap_size; params.flags = GetParam().variant; + if (params.flags & UCP_MEM_MAP_FIXED) { + params.address = (void *)(uintptr_t)0xFF000000; + params.flags |= UCP_MEM_MAP_ALLOCATE; + } else { + memheap = malloc(memheap_size); + params.address = memheap; + params.flags = params.flags & (~UCP_MEM_MAP_ALLOCATE); + } status = ucp_mem_map(receiver().ucph(), ¶ms, &memh); ASSERT_UCS_OK(status); + mem_attr.field_mask = UCP_MEM_ATTR_FIELD_ADDRESS | + UCP_MEM_ATTR_FIELD_LENGTH; + status = ucp_mem_query(memh, &mem_attr); + ASSERT_UCS_OK(status); + EXPECT_LE(memheap_size, mem_attr.length); + if (!memheap) { + memheap = mem_attr.address; + } memset(memheap, 0, memheap_size); status = ucp_rkey_pack(receiver().ucph(), memh, &rkey_buffer, &rkey_buffer_size); @@ -172,14 +186,15 @@ class test_ucp_fence : public test_ucp_atomic { EXPECT_EQ(error, (uint32_t)0); ucp_rkey_destroy(rkey); + status = ucp_mem_unmap(receiver().ucph(), memh); + ASSERT_UCS_OK(status); disconnect(sender()); disconnect(receiver()); - status = ucp_mem_unmap(receiver().ucph(), memh); - ASSERT_UCS_OK(status); - - free(memheap); + if (!(GetParam().variant & UCP_MEM_MAP_FIXED)) { + free(memheap); + } } static ucp_params_t get_ctx_params() { diff --git a/test/gtest/ucp/test_ucp_memheap.cc b/test/gtest/ucp/test_ucp_memheap.cc index 18952444d8e..8e98f99c877 100644 --- a/test/gtest/ucp/test_ucp_memheap.cc +++ b/test/gtest/ucp/test_ucp_memheap.cc @@ -6,7 +6,7 @@ */ #include "test_ucp_memheap.h" - +#include std::vector test_ucp_memheap::enum_test_params(const ucp_params_t& ctx_params, @@ -30,10 +30,13 @@ void test_ucp_memheap::test_nonblocking_implicit_stream_xfer(nonblocking_send_fu bool malloc_allocate, bool is_ep_flush) { + void *memheap; size_t memheap_size; ucp_mem_map_params_t params; + ucp_mem_attr_t mem_attr; ucs_status_t status; + memheap = NULL; memheap_size = max_iter * size + alignment; if (max_iter == DEFAULT_ITERS) { @@ -50,26 +53,35 @@ void test_ucp_memheap::test_nonblocking_implicit_stream_xfer(nonblocking_send_fu receiver().connect(&sender()); } - ucp_mem_h memh; - void *memheap; - - if (malloc_allocate) { - memheap = malloc(memheap_size); - } else { - memheap = NULL; - } - params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH | UCP_MEM_MAP_PARAM_FIELD_FLAGS; - params.address = memheap; params.length = memheap_size; params.flags = GetParam().variant; + if (malloc_allocate) { + memheap = malloc(memheap_size); + params.address = memheap; + params.flags = params.flags & (~(UCP_MEM_MAP_ALLOCATE|UCP_MEM_MAP_FIXED)); + } else if (params.flags & UCP_MEM_MAP_FIXED) { + params.address = (void *)(uintptr_t)0xFF000000; + } else { + params.address = NULL; + params.flags |= UCP_MEM_MAP_ALLOCATE; + } + ucp_mem_h memh; status = ucp_mem_map(receiver().ucph(), ¶ms, &memh); ASSERT_UCS_OK(status); - memheap = params.address; + mem_attr.field_mask = UCP_MEM_ATTR_FIELD_ADDRESS | + UCP_MEM_ATTR_FIELD_LENGTH; + status = ucp_mem_query(memh, &mem_attr); + ASSERT_UCS_OK(status); + + EXPECT_GE(mem_attr.length, memheap_size); + if (!malloc_allocate) { + memheap = mem_attr.address; + } memset(memheap, 0, memheap_size); void *rkey_buffer; @@ -91,7 +103,8 @@ void test_ucp_memheap::test_nonblocking_implicit_stream_xfer(nonblocking_send_fu ucs_assert(size * i + alignment <= memheap_size); - (this->*send)(&sender(), size, (void*)((uintptr_t)memheap + alignment + i * size), + (this->*send)(&sender(), size, + (void*)((uintptr_t)memheap + alignment + i * size), rkey, expected_data[i]); ASSERT_UCS_OK(status); @@ -106,7 +119,8 @@ void test_ucp_memheap::test_nonblocking_implicit_stream_xfer(nonblocking_send_fu for (int i = 0; i < max_iter; ++i) { EXPECT_EQ(expected_data[i], - std::string((char *)((uintptr_t)memheap + alignment + i * size), expected_data[i].length())); + std::string((char *)((uintptr_t)memheap + alignment + i * size), + expected_data[i].length())); } ucp_rkey_destroy(rkey); @@ -132,6 +146,7 @@ void test_ucp_memheap::test_blocking_xfer(blocking_send_func_t send, bool is_ep_flush) { ucp_mem_map_params_t params; + ucp_mem_attr_t mem_attr; ucs_status_t status; size_t size; int zero_offset = 0; @@ -151,25 +166,36 @@ void test_ucp_memheap::test_blocking_xfer(blocking_send_func_t send, } ucp_mem_h memh; - void *memheap; - - if (malloc_allocate) { - memheap = malloc(memheap_size); - } else { - memheap = NULL; - } + void *memheap = NULL; params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH | UCP_MEM_MAP_PARAM_FIELD_FLAGS; - params.address = memheap; params.length = memheap_size; params.flags = GetParam().variant; + if (malloc_allocate) { + memheap = malloc(memheap_size); + params.address = memheap; + params.flags = params.flags & (~(UCP_MEM_MAP_ALLOCATE|UCP_MEM_MAP_FIXED)); + } else if (params.flags & UCP_MEM_MAP_FIXED) { + params.address = (void *)(uintptr_t)0xFF000000; + params.flags |= UCP_MEM_MAP_ALLOCATE; + } else { + params.address = NULL; + params.flags |= UCP_MEM_MAP_ALLOCATE; + } status = ucp_mem_map(receiver().ucph(), ¶ms, &memh); ASSERT_UCS_OK(status); - memheap = params.address; + mem_attr.field_mask = UCP_MEM_ATTR_FIELD_ADDRESS | + UCP_MEM_ATTR_FIELD_LENGTH; + status = ucp_mem_query(memh, &mem_attr); + ASSERT_UCS_OK(status); + EXPECT_GE(mem_attr.length, memheap_size); + if (!memheap) { + memheap = mem_attr.address; + } memset(memheap, 0, memheap_size); void *rkey_buffer; @@ -222,14 +248,13 @@ void test_ucp_memheap::test_blocking_xfer(blocking_send_func_t send, ucp_rkey_destroy(rkey); receiver().flush_worker(); - disconnect(sender()); - disconnect(receiver()); - status = ucp_mem_unmap(receiver().ucph(), memh); ASSERT_UCS_OK(status); + disconnect(sender()); + disconnect(receiver()); + if (malloc_allocate) { free(memheap); } } - diff --git a/test/gtest/ucp/test_ucp_mmap.cc b/test/gtest/ucp/test_ucp_mmap.cc index bd6ea82ad9f..d3d6ea25576 100644 --- a/test/gtest/ucp/test_ucp_mmap.cc +++ b/test/gtest/ucp/test_ucp_mmap.cc @@ -5,6 +5,7 @@ */ #include "test_ucp_memheap.h" +#include "ucp/core/ucp_mm.h" class test_ucp_mmap : public test_ucp_memheap { @@ -74,7 +75,7 @@ UCS_TEST_P(test_ucp_mmap, alloc) { UCP_MEM_MAP_PARAM_FIELD_FLAGS; params.address = NULL; params.length = size; - params.flags = rand_flags(); + params.flags = rand_flags() | UCP_MEM_MAP_ALLOCATE; status = ucp_mem_map(sender().ucph(), ¶ms, &memh); ASSERT_UCS_OK(status); @@ -141,7 +142,7 @@ UCS_TEST_P(test_ucp_mmap, dummy_mem) { UCP_MEM_MAP_PARAM_FIELD_FLAGS; params.address = NULL; params.length = 0; - params.flags = rand_flags(); + params.flags = rand_flags() | UCP_MEM_MAP_ALLOCATE; status = ucp_mem_map(sender().ucph(), ¶ms, &memh[0]); ASSERT_UCS_OK(status); @@ -167,6 +168,7 @@ UCS_TEST_P(test_ucp_mmap, alloc_advise) { ucp_mem_h memh; ucp_mem_map_params_t params; + ucp_mem_attr_t attr; ucp_mem_advise_params_t advise_params; params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | @@ -174,16 +176,20 @@ UCS_TEST_P(test_ucp_mmap, alloc_advise) { UCP_MEM_MAP_PARAM_FIELD_FLAGS; params.address = NULL; params.length = size; - params.flags = UCP_MEM_MAP_NONBLOCK; + params.flags = UCP_MEM_MAP_NONBLOCK | UCP_MEM_MAP_ALLOCATE; status = ucp_mem_map(sender().ucph(), ¶ms, &memh); ASSERT_UCS_OK(status); - + attr.field_mask = UCP_MEM_ATTR_FIELD_ADDRESS | UCP_MEM_ATTR_FIELD_LENGTH; + status = ucp_mem_query(memh, &attr); + ASSERT_UCS_OK(status); + EXPECT_GE(attr.length, size); + advise_params.field_mask = UCP_MEM_ADVISE_PARAM_FIELD_ADDRESS | UCP_MEM_ADVISE_PARAM_FIELD_LENGTH | UCP_MEM_ADVISE_PARAM_FIELD_ADVICE; - advise_params.address = params.address; + advise_params.address = attr.address; advise_params.length = size; advise_params.advice = UCP_MADV_WILLNEED; status = ucp_mem_advise(sender().ucph(), memh, &advise_params); @@ -207,8 +213,9 @@ UCS_TEST_P(test_ucp_mmap, reg_advise) { void *ptr = malloc(size); - ucp_mem_h memh; - ucp_mem_map_params_t params; + ucp_mem_h memh; + ucp_mem_map_params_t params; + ucp_mem_attr_t mem_attr; ucp_mem_advise_params_t advise_params; params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | @@ -221,10 +228,14 @@ UCS_TEST_P(test_ucp_mmap, reg_advise) { status = ucp_mem_map(sender().ucph(), ¶ms, &memh); ASSERT_UCS_OK(status); + mem_attr.field_mask = UCP_MEM_ATTR_FIELD_ADDRESS; + status = ucp_mem_query(memh, &mem_attr); + ASSERT_UCS_OK(status); + advise_params.field_mask = UCP_MEM_ADVISE_PARAM_FIELD_ADDRESS | UCP_MEM_ADVISE_PARAM_FIELD_LENGTH | UCP_MEM_ADVISE_PARAM_FIELD_ADVICE; - advise_params.address = params.address; + advise_params.address = mem_attr.address; advise_params.length = size; advise_params.advice = UCP_MADV_WILLNEED; status = ucp_mem_advise(sender().ucph(), memh, &advise_params); @@ -238,4 +249,39 @@ UCS_TEST_P(test_ucp_mmap, reg_advise) { free(ptr); } +UCS_TEST_P(test_ucp_mmap, fixed) { + ucs_status_t status; + bool is_dummy; + + sender().connect(&sender()); + + for (int i = 0; i < 1000 / ucs::test_time_multiplier(); ++i) { + size_t size = (i + 1) * ((i % 2) ? 1000 : 1); + uintptr_t uptr = 0xFF000000; + void *ptr = (void *)uptr; + + ucp_mem_h memh; + ucp_mem_map_params_t params; + + params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | + UCP_MEM_MAP_PARAM_FIELD_LENGTH | + UCP_MEM_MAP_PARAM_FIELD_FLAGS; + params.address = ptr; + params.length = size; + params.flags = UCP_MEM_MAP_FIXED | UCP_MEM_MAP_ALLOCATE; + + status = ucp_mem_map(sender().ucph(), ¶ms, &memh); + ASSERT_UCS_OK(status); + EXPECT_EQ(memh->address, ptr); + EXPECT_GE(memh->length, size); + EXPECT_GE(memh->alloc_method, UCT_ALLOC_METHOD_MMAP); + + is_dummy = (size == 0); + test_rkey_management(&sender(), memh, is_dummy); + + status = ucp_mem_unmap(sender().ucph(), memh); + ASSERT_UCS_OK(status); + } +} + UCP_INSTANTIATE_TEST_CASE(test_ucp_mmap)