diff --git a/src/cart/crt_ctl.c b/src/cart/crt_ctl.c index 3ab6f1c0df4..4be89d72e0c 100644 --- a/src/cart/crt_ctl.c +++ b/src/cart/crt_ctl.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2018-2022 Intel Corporation. + * (C) Copyright 2018-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -47,7 +47,7 @@ static int verify_ctl_in_args(struct crt_ctl_ep_ls_in *in_args) static int crt_ctl_fill_buffer_cb(d_list_t *rlink, void *arg) { - crt_phy_addr_t uri; + char *uri; struct crt_uri_cache *uri_cache = arg; struct crt_uri_item *ui; uint32_t *idx; diff --git a/src/cart/crt_group.c b/src/cart/crt_group.c index 89c6c87ad81..edaa0585cf9 100644 --- a/src/cart/crt_group.c +++ b/src/cart/crt_group.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -12,9 +12,8 @@ #include #include "crt_internal.h" -static int crt_group_primary_add_internal(struct crt_grp_priv *grp_priv, - d_rank_t rank, int tag, - char *uri); +static int +crt_group_primary_add_internal(struct crt_grp_priv *grp_priv, d_rank_t rank, int tag, char *uri); /* global CRT group list */ D_LIST_HEAD(crt_grp_list); @@ -377,8 +376,8 @@ grp_li_uri_set(struct crt_lookup_item *li, int tag, const char *uri) struct crt_uri_item *ui; d_list_t *rlink; struct crt_grp_priv *grp_priv; - crt_phy_addr_t nul_str = NULL; - crt_phy_addr_t uri_dup; + char *nul_str = NULL; + char *uri_dup; d_rank_t rank; int rc = 0; int i; @@ -922,9 +921,8 @@ crt_grp_lc_addr_insert(struct crt_grp_priv *passed_grp_priv, * (base_addr == NULL) means the caller only want to lookup the hg_addr. */ void -crt_grp_lc_lookup(struct crt_grp_priv *grp_priv, int ctx_idx, - d_rank_t rank, uint32_t tag, - crt_phy_addr_t *uri, hg_addr_t *hg_addr) +crt_grp_lc_lookup(struct crt_grp_priv *grp_priv, int ctx_idx, d_rank_t rank, uint32_t tag, + char **uri, hg_addr_t *hg_addr) { struct crt_lookup_item *li; d_list_t *rlink; @@ -1004,8 +1002,7 @@ crt_grp_lookup_locked(crt_group_id_t grp_id) bool found = false; d_list_for_each_entry(grp_priv, &crt_grp_list, gp_link) { - if (crt_grp_id_identical(grp_priv->gp_pub.cg_grpid, - grp_id)) { + if (crt_grp_id_identical(grp_priv->gp_pub.cg_grpid, grp_id)) { found = true; break; } @@ -1022,8 +1019,7 @@ crt_grp_lookup_grpid(crt_group_id_t grp_id) D_RWLOCK_RDLOCK(&crt_grp_list_rwlock); d_list_for_each_entry(grp_priv, &crt_grp_list, gp_link) { - if (crt_grp_id_identical(grp_priv->gp_pub.cg_grpid, - grp_id)) { + if (crt_grp_id_identical(grp_priv->gp_pub.cg_grpid, grp_id)) { found = true; break; } @@ -1172,7 +1168,7 @@ crt_grp_priv_destroy(struct crt_grp_priv *grp_priv) d_hash_table_destroy_inplace(&grp_priv->gp_s2p_table, true); } - D_FREE(grp_priv->gp_psr_phy_addr); + D_FREE(grp_priv->gp_psr_uri); D_FREE(grp_priv->gp_pub.cg_grpid); D_RWLOCK_DESTROY(&grp_priv->gp_rwlock); @@ -1907,7 +1903,7 @@ crt_group_config_save(crt_group_t *grp, bool forall) char *tmp_name = NULL; crt_group_id_t grpid; d_rank_t rank; - crt_phy_addr_t addr = NULL; + char *addr = NULL; bool addr_free = false; bool locked = false; d_rank_list_t *membs = NULL; @@ -2086,7 +2082,7 @@ crt_grp_config_psr_load(struct crt_grp_priv *grp_priv, d_rank_t psr_rank) crt_group_id_t grpid = NULL, grpname = NULL; char all_or_self[8] = {'\0'}; char fmt[64] = {'\0'}; - crt_phy_addr_t addr_str = NULL; + char *addr_str = NULL; d_rank_t rank; int grp_size; int rc = 0; @@ -2312,7 +2308,7 @@ int crt_grp_psr_reload(struct crt_grp_priv *grp_priv) { d_rank_t psr_rank; - crt_phy_addr_t uri = NULL; + char *uri = NULL; int rc = 0; psr_rank = grp_priv->gp_psr_rank; @@ -2619,7 +2615,7 @@ int crt_rank_uri_get(crt_group_t *group, d_rank_t rank, int tag, char **uri_str) { struct crt_grp_priv *grp_priv; - crt_phy_addr_t uri; + char *uri; hg_addr_t hg_addr; int rc = 0; @@ -2797,20 +2793,6 @@ crt_group_rank_remove(crt_group_t *group, d_rank_t rank) return rc; } -int -crt_group_info_get(crt_group_t *group, d_iov_t *grp_info) -{ - D_ERROR("API is currently not supported\n"); - return -DER_NOSYS; -} - -int -crt_group_info_set(d_iov_t *grp_info) -{ - D_ERROR("API is currently not supported\n"); - return -DER_NOSYS; -} - int crt_group_ranks_get(crt_group_t *group, d_rank_list_t **list) { @@ -2829,8 +2811,7 @@ crt_group_ranks_get(crt_group_t *group, d_rank_list_t **list) } int -crt_group_view_create(crt_group_id_t srv_grpid, - crt_group_t **ret_grp) +crt_group_view_create(crt_group_id_t srv_grpid, crt_group_t **ret_grp) { struct crt_grp_gdata *grp_gdata; struct crt_grp_priv *grp_priv = NULL; @@ -3031,11 +3012,6 @@ crt_group_secondary_create(crt_group_id_t grp_name, crt_group_t *primary_grp, return rc; } -/* - * TODO: This is a temporary function until switch to non-PMIX - * mode is complete. At that point this function will be - * replaced by the generic crt_group_destroy(). - */ int crt_group_secondary_destroy(crt_group_t *grp) { diff --git a/src/cart/crt_group.h b/src/cart/crt_group.h index f9503526e65..6d99d8739d5 100644 --- a/src/cart/crt_group.h +++ b/src/cart/crt_group.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -93,8 +93,8 @@ struct crt_grp_priv { d_rank_list_t *gp_psr_ranks; /* PSR rank in attached group */ d_rank_t gp_psr_rank; - /* PSR phy addr address in attached group */ - crt_phy_addr_t gp_psr_phy_addr; + /* PSR URI address in attached group */ + char *gp_psr_uri; /* address lookup cache, only valid for primary group */ struct d_hash_table *gp_lookup_cache; @@ -196,8 +196,7 @@ struct crt_uri_item { d_list_t ui_link; /* URI string for each remote tag */ - /* TODO: in phase2 change this to hash table */ - ATOMIC crt_phy_addr_t ui_uri[CRT_SRV_CONTEXT_NUM]; + ATOMIC d_string_t ui_uri[CRT_SRV_CONTEXT_NUM]; /* Primary rank; for secondary groups only */ d_rank_t ui_pri_rank; @@ -239,9 +238,9 @@ struct crt_grp_gdata { void crt_hdlr_uri_lookup(crt_rpc_t *rpc_req); int crt_grp_detach(crt_group_t *attached_grp); -void crt_grp_lc_lookup(struct crt_grp_priv *grp_priv, int ctx_idx, - d_rank_t rank, uint32_t tag, crt_phy_addr_t *base_addr, - hg_addr_t *hg_addr); +void + crt_grp_lc_lookup(struct crt_grp_priv *grp_priv, int ctx_idx, d_rank_t rank, uint32_t tag, + char **base_addr, hg_addr_t *hg_addr); int crt_grp_lc_uri_insert(struct crt_grp_priv *grp_priv, d_rank_t rank, uint32_t tag, const char *uri); int crt_grp_lc_addr_insert(struct crt_grp_priv *grp_priv, @@ -325,25 +324,27 @@ crt_grp_priv_decref(struct crt_grp_priv *grp_priv) } static inline int -crt_grp_psr_set(struct crt_grp_priv *grp_priv, d_rank_t psr_rank, - crt_phy_addr_t psr_addr, bool steal) +crt_grp_psr_set(struct crt_grp_priv *grp_priv, d_rank_t psr_rank, char *psr_uri, bool steal) { int rc = 0; D_RWLOCK_WRLOCK(&grp_priv->gp_rwlock); - D_FREE(grp_priv->gp_psr_phy_addr); + + D_FREE(grp_priv->gp_psr_uri); grp_priv->gp_psr_rank = psr_rank; + if (steal) { - grp_priv->gp_psr_phy_addr = psr_addr; + grp_priv->gp_psr_uri = psr_uri; } else { - D_STRNDUP(grp_priv->gp_psr_phy_addr, psr_addr, - CRT_ADDR_STR_MAX_LEN); - if (grp_priv->gp_psr_phy_addr == NULL) + D_STRNDUP(grp_priv->gp_psr_uri, psr_uri, CRT_ADDR_STR_MAX_LEN); + if (grp_priv->gp_psr_uri == NULL) rc = -DER_NOMEM; } + D_RWLOCK_UNLOCK(&grp_priv->gp_rwlock); - D_DEBUG(DB_TRACE, "group %s, set psr rank %d, uri %s.\n", - grp_priv->gp_pub.cg_grpid, psr_rank, psr_addr); + + D_DEBUG(DB_TRACE, "group %s, set psr rank %d, uri %s.\n", grp_priv->gp_pub.cg_grpid, + psr_rank, psr_uri); return rc; } diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c index f428ae711a4..45062c3f133 100644 --- a/src/cart/crt_init.c +++ b/src/cart/crt_init.c @@ -23,7 +23,6 @@ static const char *crt_env_names[] = { "D_INTERFACE", "D_DOMAIN", "D_PORT", - "CRT_PHY_ADDR_STR", "D_LOG_STDERR_IN_LOG", "D_LOG_SIZE", "D_LOG_FILE", @@ -34,11 +33,7 @@ static const char *crt_env_names[] = { "DD_SUBSYS", "CRT_TIMEOUT", "CRT_ATTACH_INFO_PATH", - "OFI_PORT", - "OFI_INTERFACE", - "OFI_DOMAIN", "CRT_CREDIT_EP_CTX", - "CRT_CTX_SHARE_ADDR", "CRT_CTX_NUM", "D_FI_CONFIG", "FI_UNIVERSE_SIZE", @@ -134,6 +129,18 @@ dump_opt(crt_init_options_t *opt) D_INFO("provider = %s\n", opt->cio_provider); D_INFO("interface = %s\n", opt->cio_interface); D_INFO("domain = %s\n", opt->cio_domain); + D_INFO("port = %s\n", opt->cio_port); + D_INFO("Flags: fi: %d, use_credits: %d, use_esnsors: %d\n", opt->cio_fault_inject, + opt->cio_use_credits, opt->cio_use_sensors); + + if (opt->cio_use_expected_size) + D_INFO("max_expected_size = %d\n", opt->cio_max_expected_size); + if (opt->cio_use_unexpected_size) + D_INFO("max_unexpect_size = %d\n", opt->cio_max_unexpected_size); + + /* Handle similar to D_PROVIDER_AUTH_KEY in dump_envariables() */ + if (opt->cio_auth_key) + D_INFO("auth_key is set\n"); } static int @@ -190,7 +197,6 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, bool primary, crt_init_options_t *opt) { - bool set_sep = false; uint32_t ctx_num = 0; uint32_t max_expect_size = 0; uint32_t max_unexpect_size = 0; @@ -225,13 +231,6 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, D_DEBUG(DB_ALL, "Max number of contexts set to %d\n", max_num_ctx); - d_getenv_bool("CRT_CTX_SHARE_ADDR", &set_sep); - if (set_sep) - D_WARN("Unsupported SEP mode requested. Unset CRT_CTX_SHARE_ADDR\n"); - - if (opt && opt->cio_sep_override && opt->cio_use_sep) - D_WARN("Unsupported SEP mode requested in init options\n"); - if (opt && opt->cio_use_expected_size) max_expect_size = opt->cio_max_expected_size; @@ -241,7 +240,7 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, prov_data->cpg_inited = true; prov_data->cpg_provider = provider; prov_data->cpg_ctx_num = 0; - prov_data->cpg_sep_mode = set_sep; + prov_data->cpg_sep_mode = false; prov_data->cpg_contig_ports = true; prov_data->cpg_ctx_max_num = max_num_ctx; prov_data->cpg_max_exp_size = max_expect_size; @@ -255,15 +254,14 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, prov_data->cpg_num_remote_tags = 1; prov_data->cpg_last_remote_tag = 0; - D_DEBUG(DB_ALL, "prov_idx: %d primary: %d sep_mode: %d sizes: (%d/%d) max_ctx: %d\n", - provider, primary, set_sep, max_expect_size, max_unexpect_size, max_num_ctx); + D_DEBUG(DB_ALL, "prov_idx: %d primary: %d sizes: (%d/%d) max_ctx: %d\n", provider, primary, + max_expect_size, max_unexpect_size, max_num_ctx); D_INIT_LIST_HEAD(&prov_data->cpg_ctx_list); return DER_SUCCESS; } - /* first step init - for initializing crt_gdata */ static int data_init(int server, crt_init_options_t *opt) { @@ -550,14 +548,6 @@ prov_settings_apply(bool primary, crt_provider_t prov, crt_init_options_t *opt) if (g_prov_settings_applied[prov] == true) return; - /* rxm and verbs providers only works with regular EP */ - if (prov != CRT_PROV_OFI_SOCKETS && - crt_provider_is_sep(primary, prov)) { - D_WARN("set CRT_CTX_SHARE_ADDR as 1 is invalid " - "for current provider, ignoring it.\n"); - crt_provider_set_sep(prov, primary, false); - } - if (prov == CRT_PROV_OFI_VERBS_RXM || prov == CRT_PROV_OFI_TCP_RXM) { /* Use shared receive queues to avoid large mem consumption */ @@ -598,12 +588,6 @@ crt_protocol_info_free(struct crt_protocol_info *protocol_info) crt_hg_free_protocol_info((struct na_protocol_info *)protocol_info); } -static inline void -warn_deprecated(const char *old_env, const char *new_env) -{ - D_WARN("Usage of %s is deprecated. Set %s instead\n", old_env, new_env); -} - int crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) { @@ -703,11 +687,6 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) provider = opt->cio_provider; else { d_agetenv_str(&provider_env, "D_PROVIDER"); - if (provider_env == NULL) { - d_agetenv_str(&provider_env, CRT_PHY_ADDR_ENV); - if (provider_env != NULL) - warn_deprecated(CRT_PHY_ADDR_ENV, "D_PROVIDER"); - } provider = provider_env; } @@ -715,11 +694,6 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) interface = opt->cio_interface; else { d_agetenv_str(&interface_env, "D_INTERFACE"); - if (interface_env == NULL) { - d_agetenv_str(&interface_env, "OFI_INTERFACE"); - if (interface_env != NULL) - warn_deprecated("OFI_INTERFACE", "D_INTERFACE"); - } interface = interface_env; } @@ -727,11 +701,6 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) domain = opt->cio_domain; else { d_agetenv_str(&domain_env, "D_DOMAIN"); - if (domain_env == NULL) { - d_agetenv_str(&domain_env, "OFI_DOMAIN"); - if (domain_env != NULL) - warn_deprecated("OFI_DOMAIN", "D_DOMAIN"); - } domain = domain_env; } @@ -739,11 +708,6 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) port = opt->cio_port; else { d_agetenv_str(&port_env, "D_PORT"); - if (port_env == NULL) { - d_agetenv_str(&port_env, "OFI_PORT"); - if (port_env != NULL) - warn_deprecated("OFI_PORT", "D_PORT"); - } port = port_env; } @@ -987,10 +951,10 @@ crt_initialized() int crt_finalize(void) { - int local_rc; - int rc = 0; - int i; struct crt_prov_gdata *prov_data; + int local_rc; + int rc = 0; + int i; D_RWLOCK_WRLOCK(&crt_gdata.cg_rwlock); @@ -1008,16 +972,14 @@ crt_finalize(void) prov_data = &crt_gdata.cg_prov_gdata_primary; if (prov_data->cpg_ctx_num > 0) { - D_ASSERT(!crt_context_empty(crt_gdata.cg_primary_prov, - CRT_LOCKED)); + D_ASSERT(!crt_context_empty(crt_gdata.cg_primary_prov, CRT_LOCKED)); D_ERROR("cannot finalize, current ctx_num(%d).\n", prov_data->cpg_ctx_num); crt_gdata.cg_refcount++; D_RWLOCK_UNLOCK(&crt_gdata.cg_rwlock); D_GOTO(out, rc = -DER_BUSY); } else { - D_ASSERT(crt_context_empty(crt_gdata.cg_primary_prov, - CRT_LOCKED)); + D_ASSERT(crt_context_empty(crt_gdata.cg_primary_prov, CRT_LOCKED)); } if (crt_plugin_gdata.cpg_inited == 1) @@ -1040,7 +1002,6 @@ crt_finalize(void) D_RWLOCK_UNLOCK(&crt_gdata.cg_rwlock); - /* allow the same program to re-initialize */ crt_gdata.cg_refcount = 0; crt_gdata.cg_inited = 0; gdata_init_flag = 0; @@ -1096,8 +1057,8 @@ static inline bool is_integer_str(char *str) static inline int crt_get_port_opx(int *port) { - int rc = 0; - uint16_t pid; + int rc = 0; + uint16_t pid; pid = getpid(); *port = pid; @@ -1226,12 +1187,6 @@ crt_na_config_init(bool primary, crt_provider_t provider, save_ptr++; } - if (crt_provider_is_sep(primary, provider) && count > 1) { - D_ERROR("Multi-interface not available with SEP\n"); - D_GOTO(out, rc = -DER_NOSYS); - } - - D_ALLOC_ARRAY(na_cfg->noc_iface_str, count); if (!na_cfg->noc_iface_str) D_GOTO(out, rc = -DER_NOMEM); @@ -1255,7 +1210,7 @@ crt_na_config_init(bool primary, crt_provider_t provider, if (crt_is_service() && port_str != NULL && strlen(port_str) > 0) { if (!is_integer_str(port_str)) { - D_DEBUG(DB_ALL, "ignoring invalid OFI_PORT %s.", port_str); + D_DEBUG(DB_ALL, "ignoring invalid D_PORT %s.", port_str); } else { port = atoi(port_str); @@ -1273,7 +1228,7 @@ crt_na_config_init(bool primary, crt_provider_t provider, } } - D_DEBUG(DB_ALL, "OFI_PORT %d, using it as service port.\n", port); + D_DEBUG(DB_ALL, "D_PORT %d, using it as service port.\n", port); } } else if (provider == CRT_PROV_OFI_OPX) { rc = crt_get_port_opx(&port); diff --git a/src/cart/crt_rpc.c b/src/cart/crt_rpc.c index ae5dddaa883..45d8430ad57 100644 --- a/src/cart/crt_rpc.c +++ b/src/cart/crt_rpc.c @@ -790,7 +790,7 @@ crt_req_decref(crt_rpc_t *req) } static inline int -crt_req_fill_tgt_uri(struct crt_rpc_priv *rpc_priv, crt_phy_addr_t base_uri) +crt_req_fill_tgt_uri(struct crt_rpc_priv *rpc_priv, char *base_uri) { D_ASSERT(rpc_priv != NULL); D_ASSERT(base_uri != NULL); @@ -1197,9 +1197,9 @@ crt_req_ep_lc_lookup(struct crt_rpc_priv *rpc_priv, bool *uri_exists) crt_rpc_t *req; crt_endpoint_t *tgt_ep; struct crt_context *ctx; - crt_phy_addr_t uri = NULL; + char *uri = NULL; int rc = 0; - crt_phy_addr_t base_addr = NULL; + char *base_addr = NULL; struct crt_prov_gdata *prov_data; int dst_tag; @@ -1252,8 +1252,7 @@ crt_req_ep_lc_lookup(struct crt_rpc_priv *rpc_priv, bool *uri_exists) if (base_addr != NULL && rpc_priv->crp_hg_addr == NULL) { rc = crt_req_fill_tgt_uri(rpc_priv, base_addr); if (rc != 0) - RPC_ERROR(rpc_priv, - "crt_req_fill_tgt_uri() failed, " DF_RC "\n", + RPC_ERROR(rpc_priv, "crt_req_fill_tgt_uri() failed, " DF_RC "\n", DP_RC(rc)); D_GOTO(out, rc); } @@ -1266,27 +1265,23 @@ crt_req_ep_lc_lookup(struct crt_rpc_priv *rpc_priv, bool *uri_exists) */ if (base_addr == NULL && !crt_is_service()) { D_RWLOCK_RDLOCK(&grp_priv->gp_rwlock); - if (tgt_ep->ep_rank == grp_priv->gp_psr_rank && - dst_tag == 0) { - D_STRNDUP(uri, grp_priv->gp_psr_phy_addr, - CRT_ADDR_STR_MAX_LEN); + if (tgt_ep->ep_rank == grp_priv->gp_psr_rank && dst_tag == 0) { + D_STRNDUP(uri, grp_priv->gp_psr_uri, CRT_ADDR_STR_MAX_LEN); D_RWLOCK_UNLOCK(&grp_priv->gp_rwlock); + if (uri == NULL) D_GOTO(out, rc = -DER_NOMEM); base_addr = uri; - rc = crt_grp_lc_uri_insert(grp_priv, - tgt_ep->ep_rank, 0, uri); + rc = crt_grp_lc_uri_insert(grp_priv, tgt_ep->ep_rank, 0, uri); if (rc != 0) { - D_ERROR("crt_grp_lc_uri_insert() failed, " - "rc: %d\n", rc); + D_ERROR("crt_grp_lc_uri_insert() failed rc=%d\n", rc); D_GOTO(out, rc); } rc = crt_req_fill_tgt_uri(rpc_priv, uri); if (rc != 0) { - D_ERROR("crt_req_fill_tgt_uri failed, " - "opc: %#x.\n", req->cr_opc); + RPC_ERROR(rpc_priv, "tgt_uri='%s' fill failed\n", uri); D_GOTO(out, rc); } } else { diff --git a/src/cart/crt_rpc.h b/src/cart/crt_rpc.h index 8f56e5e0c81..20be21232c8 100644 --- a/src/cart/crt_rpc.h +++ b/src/cart/crt_rpc.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -151,7 +151,7 @@ struct crt_rpc_priv { hg_handle_t crp_hg_hdl; /* HG request handle */ hg_addr_t crp_hg_addr; /* target na address */ struct crt_hg_hdl *crp_hdl_reuse; /* reused hg_hdl */ - crt_phy_addr_t crp_tgt_uri; /* target uri address */ + char *crp_tgt_uri; /* target uri address */ crt_rpc_t *crp_ul_req; /* uri lookup request */ uint32_t crp_ul_retry; /* uri lookup retry counter */ @@ -379,10 +379,8 @@ CRT_GEN_STRUCT(crt_grp_cache, CRT_SEQ_GRP_CACHE) ((d_rank_t) (ul_rank) CRT_VAR) \ ((uint32_t) (ul_tag) CRT_VAR) -#define CRT_OSEQ_URI_LOOKUP /* output fields */ \ - ((crt_phy_addr_t) (ul_uri) CRT_VAR) \ - ((uint32_t) (ul_tag) CRT_VAR) \ - ((int32_t) (ul_rc) CRT_VAR) +#define CRT_OSEQ_URI_LOOKUP /* output fields */ \ + ((d_string_t)(ul_uri)CRT_VAR)((uint32_t)(ul_tag)CRT_VAR)((int32_t)(ul_rc)CRT_VAR) CRT_RPC_DECLARE(crt_uri_lookup, CRT_ISEQ_URI_LOOKUP, CRT_OSEQ_URI_LOOKUP) diff --git a/src/include/cart/api.h b/src/include/cart/api.h index 2c82b8f4303..5beaa575e4a 100644 --- a/src/include/cart/api.h +++ b/src/include/cart/api.h @@ -2058,35 +2058,6 @@ int crt_self_uri_get(int tag, char **uri); */ int crt_self_incarnation_get(uint64_t *incarnation); -/** - * Retrieve group information containing ranks and associated uris - * - * This call will allocate memory for buffers in passed \a grp_info. - * User is responsible for freeing the memory once not needed anymore. - * - * Returned data in \a grp_info can be passed to crt_group_info_set - * call in order to setup group on a different node. - * - * \param[in] group Group identifier - * \param[in] grp_info group info to be filled. - * - * \return DER_SUCCESS on success, negative value - * on failure. - */ -int crt_group_info_get(crt_group_t *group, d_iov_t *grp_info); - -/** - * Sets group info (nodes and associated uris) baesd on passed - * grp_info data. \a grp_info is to be retrieved via \a crt_group_info_get - * call. - * - * \param[in] grp_info Group information to set - * - * \return DER_SUCCESS on success, negative value - * on failure. - */ -int crt_group_info_set(d_iov_t *grp_info); - /** * Retrieve list of ranks that belong to the specified group. * @@ -2297,8 +2268,7 @@ void crt_swim_fini(void); #define crt_proc_d_rank_t crt_proc_uint32_t #define crt_proc_int crt_proc_int32_t #define crt_proc_crt_status_t crt_proc_int32_t -#define crt_proc_crt_group_id_t crt_proc_d_string_t -#define crt_proc_crt_phy_addr_t crt_proc_d_string_t +#define crt_proc_crt_group_id_t crt_proc_d_string_t /** * \a err is an error that ought to be logged at a less serious level than ERR. diff --git a/src/include/cart/types.h b/src/include/cart/types.h index 7ae44ceac1a..f44f85b2817 100644 --- a/src/include/cart/types.h +++ b/src/include/cart/types.h @@ -136,10 +136,6 @@ typedef struct { /** CaRT context handle */ typedef void *crt_context_t; -/** Physical address string, e.g., "bmi+tcp://localhost:3344". */ -typedef d_string_t crt_phy_addr_t; -#define CRT_PHY_ADDR_ENV "CRT_PHY_ADDR_STR" - /** * RPC is identified by opcode. All the opcodes with the highest 16 bits as 1 * are reserved for internal usage, such as group maintenance etc. If user diff --git a/src/tests/ftest/cart/utest/utest_portnumber.c b/src/tests/ftest/cart/utest/utest_portnumber.c index 5e15b897ce5..a64537def78 100644 --- a/src/tests/ftest/cart/utest/utest_portnumber.c +++ b/src/tests/ftest/cart/utest/utest_portnumber.c @@ -290,7 +290,7 @@ int main(int argc, char **argv) d_setenv("FI_UNIVERSE_SIZE", "2048", 1); d_setenv("FI_OFI_RXM_USE_SRX", "1", 1); d_setenv("D_LOG_MASK", "CRIT", 1); - d_setenv("OFI_PORT", "34571", 1); + d_setenv("D_PORT", "34571", 1); d_register_alt_assert(mock_assert); diff --git a/src/utils/crt_launch/crt_launch.c b/src/utils/crt_launch/crt_launch.c index fa24335b535..8700913f82d 100644 --- a/src/utils/crt_launch/crt_launch.c +++ b/src/utils/crt_launch/crt_launch.c @@ -1,9 +1,9 @@ /* - * (C) Copyright 2019-2022 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ - /** +/** * MPI-based and cart-based crt_launch application that facilitates launching * cart-based clients and servers when no pmix is used. * @@ -18,7 +18,7 @@ * CRT_L_GRP_CFG - Path to group configuration file generated in /tmp/ having * form of crt_launch-info-XXXXXX where X's are replaced by random string. * - * OFI_PORT - port to use for + * D_PORT - port to use for first context * */ #include @@ -156,7 +156,7 @@ get_self_uri(struct host *h, int rank) if (str_port == NULL) return -DER_NOMEM; - d_setenv("OFI_PORT", str_port, 1); + d_setenv("D_PORT", str_port, 1); rc = crt_init(0, CRT_FLAG_BIT_SERVER | CRT_FLAG_BIT_AUTO_SWIM_DISABLE); if (rc != 0) { @@ -313,7 +313,7 @@ int main(int argc, char **argv) sprintf(str_rank, "%d", hostbuf->my_rank); sprintf(str_port, "%d", hostbuf->ofi_port); - /* Set CRT_L_RANK and OFI_PORT */ + /* Set CRT_L_RANK and D_PORT */ d_setenv("CRT_L_RANK", str_rank, true); d_setenv("D_PORT", str_port, true); diff --git a/src/utils/self_test/self_test.c b/src/utils/self_test/self_test.c index b043f66ac32..6e06af59bd0 100644 --- a/src/utils/self_test/self_test.c +++ b/src/utils/self_test/self_test.c @@ -84,10 +84,9 @@ static void *progress_fn(void *arg) pthread_exit(NULL); } -static int self_test_init(char *dest_name, crt_context_t *crt_ctx, - crt_group_t **srv_grp, pthread_t *tid, - char *attach_info_path, bool listen, - bool use_daos_agent_vars) +static int +self_test_init(char *dest_name, crt_context_t *crt_ctx, crt_group_t **srv_grp, pthread_t *tid, + char *attach_info_path, bool listen, bool use_agent) { uint32_t init_flags = 0; uint32_t grp_size; @@ -100,7 +99,7 @@ static int self_test_init(char *dest_name, crt_context_t *crt_ctx, /* rank, num_attach_retries, is_server, assert_on_error */ crtu_test_init(0, attach_retries, false, false); - if (use_daos_agent_vars) { + if (use_agent) { ret = dc_agent_init(); if (ret != 0) { fprintf(stderr, "dc_agent_init() failed. ret: %d\n", ret); @@ -134,7 +133,7 @@ static int self_test_init(char *dest_name, crt_context_t *crt_ctx, } g_context_created = true; - if (use_daos_agent_vars) { + if (use_agent) { ret = crt_group_view_create(dest_name, srv_grp); if (!*srv_grp || ret != 0) { D_ERROR("Failed to create group view; ret=%d\n", ret); @@ -802,14 +801,11 @@ randomize_endpts(struct st_endpoint *endpts, uint32_t num_endpts) printf("\n"); } -static int run_self_test(struct st_size_params all_params[], - int num_msg_sizes, int rep_count, int max_inflight, - char *dest_name, struct st_endpoint *ms_endpts_in, - uint32_t num_ms_endpts_in, - struct st_endpoint *endpts, uint32_t num_endpts, - int output_megabits, int16_t buf_alignment, - char *attach_info_path, - bool use_daos_agent_vars) +static int +run_self_test(struct st_size_params all_params[], int num_msg_sizes, int rep_count, + int max_inflight, char *dest_name, struct st_endpoint *ms_endpts_in, + uint32_t num_ms_endpts_in, struct st_endpoint *endpts, uint32_t num_endpts, + int output_megabits, int16_t buf_alignment, char *attach_info_path, bool use_agent) { crt_context_t crt_ctx; crt_group_t *srv_grp; @@ -841,9 +837,8 @@ static int run_self_test(struct st_size_params all_params[], if (ms_endpts_in == NULL) listen = true; /* Initialize CART */ - ret = self_test_init(dest_name, &crt_ctx, &srv_grp, &tid, - attach_info_path, listen /* run as server */, - use_daos_agent_vars); + ret = self_test_init(dest_name, &crt_ctx, &srv_grp, &tid, attach_info_path, + listen /* run as server */, use_agent); if (ret != 0) { D_ERROR("self_test_init failed; ret = %d\n", ret); D_GOTO(cleanup_nothread, ret); @@ -1076,183 +1071,183 @@ static int run_self_test(struct st_size_params all_params[], return ret; } -static void print_usage(const char *prog_name, const char *msg_sizes_str, - int rep_count, - int max_inflight) +static void +print_usage(const char *prog_name, const char *msg_sizes_str, int rep_count, int max_inflight) { - /* TODO --randomize-endpoints */ /* TODO --verbose */ - printf("Usage: %s --group-name --endpoint [optional arguments]\n" - "\n" - "Required Arguments\n" - " --group-name \n" - " Short version: -g\n" - " The name of the process set to test against\n" - "\n" - " --endpoint \n" - " Short version: -e\n" - " Describes an endpoint (or range of endpoints) to connect to\n" - " Note: Can be specified multiple times\n" - "\n" - " ranks and tags are comma-separated lists to connect to\n" - " Supports both ranges and lists - for example, \"1-5,3,8\"\n" - "\n" - " Example: --endpoint 1-3,2:0-1\n" - " This would create these endpoints:\n" - " 1:0\n" - " 1:1\n" - " 2:0\n" - " 2:1\n" - " 3:0\n" - " 3:1\n" - " 2:0\n" - " 2:1\n" - "\n" - " By default, self-test will send test messages to these\n" - " endpoints in the order listed above. See --randomize-endpoints\n" - " for more information\n" - "\n" - "Optional Arguments\n" - " --message-sizes <(a b),(c d),...>\n" - " Short version: -s\n" - " List of size tuples (in bytes) to use for the self test.\n" - "\n" - " Note that the ( ) are not strictly necessary\n" - " Providing a single size (a) is interpreted as an alias for (a a)\n" - "\n" - " For each tuple, the first value is the sent size, and the second value is the reply size\n" - " Valid sizes are [0-%d]\n" - " Performance results will be reported individually for each tuple.\n" - "\n" - " Each size integer can be prepended with a single character to specify\n" - " the underlying transport mechanism. Available types are:\n" - " 'e' - Empty (no payload)\n" - " 'i' - I/O vector (IOV)\n" - " 'b' - Bulk transfer\n" - " For example, (b1000) would transfer 1000 bytes via bulk in both directions\n" - " Similarly, (i100 b1000) would use IOV to send and bulk to reply\n" - " Only reasonable combinations are permitted (i.e. e1000 is not allowed)\n" - " If no type specifier is specified, one will be chosen automatically. The simple\n" - " heuristic is that bulk will be used if a specified size is >= %u\n" - " BULK_GET will be used on the service side to 'send' data from client\n" - " to service, and BULK_PUT will be used on the service side to 'reply'\n" - " (assuming bulk transfers specified)\n" - "\n" - " Note that different messages are sent internally via different structures.\n" - " These are enumerated as follows, with x,y > 0:\n" - " (0 0) - Empty payload sent in both directions\n" - " (ix 0) - 8-byte session_id + x-byte iov sent, empty reply\n" - " (0 iy) - 8-byte session_id sent, y-byte iov reply\n" - " (ix iy) - 8-byte session_id + x-byte iov sent, y-byte iov reply\n" - " (0 by) - 8-byte session_id + 8-byte bulk handle sent\n" - " y-byte BULK_PUT, empty reply\n" - " (bx 0) - 8-byte session_id + 8-byte bulk_handle sent\n" - " x-byte BULK_GET, empty reply\n" - " (ix by) - 8-byte session_id + x-byte iov + 8-byte bulk_handle sent\n" - " y-byte BULK_PUT, empty reply\n" - " (bx iy) - 8-byte session_id + 8-byte bulk_handle sent\n" - " x-byte BULK_GET, y-byte iov reply\n" - " (bx by) - 8-byte session_id + 8-byte bulk_handle sent\n" - " x-byte BULK_GET, y-byte BULK_PUT, empty reply\n" - "\n" - " Note also that any message size other than (0 0) will use test sessions.\n" - " A self-test session will be negotiated with the service before sending\n" - " any traffic, and the session will be closed after testing this size completes.\n" - " The time to create and tear down these sessions is NOT measured.\n" - "\n" - " Default: \"%s\"\n" - "\n" - " --master-endpoint \n" - " Short version: -m\n" - " Describes an endpoint (or range of endpoints) that will each run a\n" - " 1:many self-test against the list of endpoints given via the\n" - " --endpoint argument.\n" - "\n" - " Specifying multiple --master-endpoint ranks/tags sets up a many:many\n" - " self-test - the first 'many' is the list of master endpoints, each\n" - " which executes a separate concurrent test against the second\n" - " 'many' (the list of test endpoints)\n" - "\n" - " The argument syntax for this option is identical to that for\n" - " --endpoint. Also, like --endpoint, --master-endpoint can be\n" - " specified multiple times\n" - "\n" - " Unlike --endpoint, the list of master endpoints is sorted and\n" - " any duplicate entries are removed automatically. This is because\n" - " each instance of self-test can only manage one 1:many test at\n" - " a time\n" - "\n" - " If not specified, the default value is to use this command-line\n" - " application itself to run a 1:many test against the test endpoints\n" - "\n" - " This client application sends all of the self-test parameters to\n" - " this master node and instructs it to run a self-test session against\n" - " the other endpoints specified by the --endpoint argument\n" - "\n" - " This allows self-test to be run between any arbitrary CART-enabled\n" - " applications without having to make them self-test aware. These\n" - " other applications can be busy doing something else entirely and\n" - " self-test will have no impact on that workload beyond consuming\n" - " additional network and compute resources\n" - "\n" - " --repetitions-per-size \n" - " Short version: -r\n" - " Number of samples per message size per endpt.\n" - " RPCs for each particular size will be repeated this many times per endpt.\n" - " Default: %d\n" - "\n" - " --max-inflight-rpcs \n" - " Short version: -i\n" - " Maximum number of RPCs allowed to be executing concurrently.\n" - "\n" - " Note that at the beginning of each test run, a buffer of size send_size\n" - " is allocated for each in-flight RPC (total max_inflight * send_size).\n" - " This could be a lot of memory. Also, if the reply uses bulk, the\n" - " size increases to (max_inflight * max(send_size, reply_size))\n" - "\n" - " Default: %d\n" - "\n" - " --align \n" - " Short version: -a\n" - "\n" - " Forces all test buffers to be aligned (or misaligned) as specified.\n" - "\n" - " The argument specifies what the least-significant byte of all test buffer\n" - " addresses should be forced to be. For example, if --align 0 is specified,\n" - " all test buffer addresses will end in 0x00 (thus aligned to 256 bytes).\n" - " To force misalignment, use something like --align 3. For 64-bit (8-byte)\n" - " alignment, use something like --align 8 or --align 24 (0x08 and 0x18)\n" - "\n" - " Alignment should be specified as a decimal value in the range [%d:%d]\n" - "\n" - " If specified, buffers will be allocated with an extra 256 bytes of\n" - " alignment padding and the buffer to transfer will start at the point which\n" - " the least - significant byte of the address matches the requested alignment.\n" - "\n" - " Default is no alignment - whatever is returned by the allocator is used\n" - "\n" - " --Mbits\n" - " Short version: -b\n" - " By default, self-test outputs performance results in MB (#Bytes/1024^2)\n" - " Specifying --Mbits switches the output to megabits (#bits/1000000)\n" - " --path /path/to/attach_info_file/directory/\n" - " Short version: -p prefix\n" - " This option implies --singleton is set.\n" - " If specified, self_test will use the address information in:\n" - " /tmp/group_name.attach_info_tmp, if prefix is specified, self_test will use\n" - " the address information in: prefix/group_name.attach_info_tmp.\n" - " Note the = sign in the option.\n" - "\n" - " --use-daos-agent-env\n" - " Short version: -u\n" - " This option sets the following env vars through a running daos_agent.\n" - " - D_INTERFACE\n" - " - D_PROVIDER\n" - " - D_DOMAIN\n" - " - CRT_TIMEOUT\n", - prog_name, UINT32_MAX, - CRT_SELF_TEST_AUTO_BULK_THRESH, msg_sizes_str, rep_count, - max_inflight, CRT_ST_BUF_ALIGN_MIN, CRT_ST_BUF_ALIGN_MIN); + printf( + "Usage: %s --group-name --endpoint [optional arguments]\n" + "\n" + "Required Arguments\n" + " --group-name \n" + " Short version: -g\n" + " The name of the process set to test against\n" + "\n" + " --endpoint \n" + " Short version: -e\n" + " Describes an endpoint (or range of endpoints) to connect to\n" + " Note: Can be specified multiple times\n" + "\n" + " ranks and tags are comma-separated lists to connect to\n" + " Supports both ranges and lists - for example, \"1-5,3,8\"\n" + "\n" + " Example: --endpoint 1-3,2:0-1\n" + " This would create these endpoints:\n" + " 1:0\n" + " 1:1\n" + " 2:0\n" + " 2:1\n" + " 3:0\n" + " 3:1\n" + " 2:0\n" + " 2:1\n" + "\n" + " By default, self-test will send test messages to these\n" + " endpoints in the order listed above. See --randomize-endpoints\n" + " for more information\n" + "\n" + "Optional Arguments\n" + " --message-sizes <(a b),(c d),...>\n" + " Short version: -s\n" + " List of size tuples (in bytes) to use for the self test.\n" + "\n" + " Note that the ( ) are not strictly necessary\n" + " Providing a single size (a) is interpreted as an alias for (a a)\n" + "\n" + " For each tuple, the first value is the sent size, and the second value is the " + "reply size\n" + " Valid sizes are [0-%d]\n" + " Performance results will be reported individually for each tuple.\n" + "\n" + " Each size integer can be prepended with a single character to specify\n" + " the underlying transport mechanism. Available types are:\n" + " 'e' - Empty (no payload)\n" + " 'i' - I/O vector (IOV)\n" + " 'b' - Bulk transfer\n" + " For example, (b1000) would transfer 1000 bytes via bulk in both directions\n" + " Similarly, (i100 b1000) would use IOV to send and bulk to reply\n" + " Only reasonable combinations are permitted (i.e. e1000 is not allowed)\n" + " If no type specifier is specified, one will be chosen automatically. The " + "simple\n" + " heuristic is that bulk will be used if a specified size is >= %u\n" + " BULK_GET will be used on the service side to 'send' data from client\n" + " to service, and BULK_PUT will be used on the service side to 'reply'\n" + " (assuming bulk transfers specified)\n" + "\n" + " Note that different messages are sent internally via different structures.\n" + " These are enumerated as follows, with x,y > 0:\n" + " (0 0) - Empty payload sent in both directions\n" + " (ix 0) - 8-byte session_id + x-byte iov sent, empty reply\n" + " (0 iy) - 8-byte session_id sent, y-byte iov reply\n" + " (ix iy) - 8-byte session_id + x-byte iov sent, y-byte iov reply\n" + " (0 by) - 8-byte session_id + 8-byte bulk handle sent\n" + " y-byte BULK_PUT, empty reply\n" + " (bx 0) - 8-byte session_id + 8-byte bulk_handle sent\n" + " x-byte BULK_GET, empty reply\n" + " (ix by) - 8-byte session_id + x-byte iov + 8-byte bulk_handle sent\n" + " y-byte BULK_PUT, empty reply\n" + " (bx iy) - 8-byte session_id + 8-byte bulk_handle sent\n" + " x-byte BULK_GET, y-byte iov reply\n" + " (bx by) - 8-byte session_id + 8-byte bulk_handle sent\n" + " x-byte BULK_GET, y-byte BULK_PUT, empty reply\n" + "\n" + " Note also that any message size other than (0 0) will use test sessions.\n" + " A self-test session will be negotiated with the service before sending\n" + " any traffic, and the session will be closed after testing this size " + "completes.\n" + " The time to create and tear down these sessions is NOT measured.\n" + "\n" + " Default: \"%s\"\n" + "\n" + " --master-endpoint \n" + " Short version: -m\n" + " Describes an endpoint (or range of endpoints) that will each run a\n" + " 1:many self-test against the list of endpoints given via the\n" + " --endpoint argument.\n" + "\n" + " Specifying multiple --master-endpoint ranks/tags sets up a many:many\n" + " self-test - the first 'many' is the list of master endpoints, each\n" + " which executes a separate concurrent test against the second\n" + " 'many' (the list of test endpoints)\n" + "\n" + " The argument syntax for this option is identical to that for\n" + " --endpoint. Also, like --endpoint, --master-endpoint can be\n" + " specified multiple times\n" + "\n" + " Unlike --endpoint, the list of master endpoints is sorted and\n" + " any duplicate entries are removed automatically. This is because\n" + " each instance of self-test can only manage one 1:many test at\n" + " a time\n" + "\n" + " If not specified, the default value is to use this command-line\n" + " application itself to run a 1:many test against the test endpoints\n" + "\n" + " This client application sends all of the self-test parameters to\n" + " this master node and instructs it to run a self-test session against\n" + " the other endpoints specified by the --endpoint argument\n" + "\n" + " This allows self-test to be run between any arbitrary CART-enabled\n" + " applications without having to make them self-test aware. These\n" + " other applications can be busy doing something else entirely and\n" + " self-test will have no impact on that workload beyond consuming\n" + " additional network and compute resources\n" + "\n" + " --repetitions-per-size \n" + " Short version: -r\n" + " Number of samples per message size per endpt.\n" + " RPCs for each particular size will be repeated this many times per endpt.\n" + " Default: %d\n" + "\n" + " --max-inflight-rpcs \n" + " Short version: -i\n" + " Maximum number of RPCs allowed to be executing concurrently.\n" + "\n" + " Note that at the beginning of each test run, a buffer of size send_size\n" + " is allocated for each in-flight RPC (total max_inflight * send_size).\n" + " This could be a lot of memory. Also, if the reply uses bulk, the\n" + " size increases to (max_inflight * max(send_size, reply_size))\n" + "\n" + " Default: %d\n" + "\n" + " --align \n" + " Short version: -a\n" + "\n" + " Forces all test buffers to be aligned (or misaligned) as specified.\n" + "\n" + " The argument specifies what the least-significant byte of all test buffer\n" + " addresses should be forced to be. For example, if --align 0 is specified,\n" + " all test buffer addresses will end in 0x00 (thus aligned to 256 bytes).\n" + " To force misalignment, use something like --align 3. For 64-bit (8-byte)\n" + " alignment, use something like --align 8 or --align 24 (0x08 and 0x18)\n" + "\n" + " Alignment should be specified as a decimal value in the range [%d:%d]\n" + "\n" + " If specified, buffers will be allocated with an extra 256 bytes of\n" + " alignment padding and the buffer to transfer will start at the point which\n" + " the least - significant byte of the address matches the requested alignment.\n" + "\n" + " Default is no alignment - whatever is returned by the allocator is used\n" + "\n" + " --Mbits\n" + " Short version: -b\n" + " By default, self-test outputs performance results in MB (#Bytes/1024^2)\n" + " Specifying --Mbits switches the output to megabits (#bits/1000000)\n" + " --path /path/to/attach_info_file/directory/\n" + " Short version: -p prefix\n" + " If specified, self_test will use the address information in:\n" + " /tmp/group_name.attach_info_tmp, if prefix is specified, self_test will use\n" + " the address information in: prefix/group_name.attach_info_tmp.\n" + " Note the = sign in the option.\n" + "\n" + " --use-daos-agent-env\n" + " Short version: -u\n" + " This option sets the following env vars through a running daos_agent.\n" + " - D_INTERFACE\n" + " - D_PROVIDER\n" + " - D_DOMAIN\n" + " - CRT_TIMEOUT\n", + prog_name, UINT32_MAX, CRT_SELF_TEST_AUTO_BULK_THRESH, msg_sizes_str, rep_count, + max_inflight, CRT_ST_BUF_ALIGN_MIN, CRT_ST_BUF_ALIGN_MIN); } #define ST_ENDPT_RANK_IDX 0 @@ -1713,10 +1708,9 @@ int main(int argc, char *argv[]) uint32_t num_endpts = 0; uint32_t num_ms_endpts = 0; int output_megabits = 0; - int16_t buf_alignment = - CRT_ST_BUF_ALIGN_DEFAULT; + int16_t buf_alignment = CRT_ST_BUF_ALIGN_DEFAULT; char *attach_info_path = NULL; - bool use_daos_agent_vars = false; + bool use_agent = false; ret = d_log_init(); if (ret != 0) { @@ -1726,25 +1720,21 @@ int main(int argc, char *argv[]) /********************* Parse user arguments *********************/ while (1) { static struct option long_options[] = { - {"group-name", required_argument, 0, 'g'}, - {"master-endpoint", required_argument, 0, 'm'}, - {"endpoint", required_argument, 0, 'e'}, - {"message-sizes", required_argument, 0, 's'}, - {"repetitions-per-size", required_argument, 0, 'r'}, - {"max-inflight-rpcs", required_argument, 0, 'i'}, - {"align", required_argument, 0, 'a'}, - {"Mbits", no_argument, 0, 'b'}, - {"singleton", no_argument, 0, 't'}, - {"randomize-endpoints", no_argument, 0, 'q'}, - {"path", required_argument, 0, 'p'}, - {"nopmix", no_argument, 0, 'n'}, - {"use-daos-agent-env", no_argument, 0, 'u'}, - {"help", no_argument, 0, 'h'}, - {0, 0, 0, 0} - }; - - c = getopt_long(argc, argv, "g:m:e:s:r:i:a:bthnqp:u", - long_options, NULL); + {"group-name", required_argument, 0, 'g'}, + {"master-endpoint", required_argument, 0, 'm'}, + {"endpoint", required_argument, 0, 'e'}, + {"message-sizes", required_argument, 0, 's'}, + {"repetitions-per-size", required_argument, 0, 'r'}, + {"max-inflight-rpcs", required_argument, 0, 'i'}, + {"align", required_argument, 0, 'a'}, + {"Mbits", no_argument, 0, 'b'}, + {"randomize-endpoints", no_argument, 0, 'q'}, + {"path", required_argument, 0, 'p'}, + {"use-daos-agent-env", no_argument, 0, 'u'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0}}; + + c = getopt_long(argc, argv, "g:m:e:s:r:i:a:bhqp:u", long_options, NULL); if (c == -1) break; @@ -1797,35 +1787,26 @@ int main(int argc, char *argv[]) attach_info_path = optarg; break; case 'u': - use_daos_agent_vars = true; + use_agent = true; break; case 'q': g_randomize_endpoints = true; break; - /* 't' and 'n' options are deprecated */ - case 't': - printf("Warning: 't' argument is deprecated\n"); - break; - case 'n': - printf("Warning: 'n' argument is deprecated\n"); - break; case 'h': case '?': - print_usage(argv[0], default_msg_sizes_str, - default_rep_count, + print_usage(argv[0], default_msg_sizes_str, default_rep_count, default_max_inflight); D_GOTO(cleanup, ret = 0); break; default: - print_usage(argv[0], default_msg_sizes_str, - default_rep_count, + print_usage(argv[0], default_msg_sizes_str, default_rep_count, default_max_inflight); D_GOTO(cleanup, ret = -DER_INVAL); } } - if (use_daos_agent_vars == false) { + if (use_agent == false) { char *attach_path; char *attach_path_env = NULL; @@ -1858,8 +1839,6 @@ int main(int argc, char *argv[]) } /******************** Parse message sizes argument ********************/ - - /* * Count the number of tuple tokens (',') in the user-specified string * This gives an upper limit on the number of arguments the user passed @@ -1898,8 +1877,7 @@ int main(int argc, char *argv[]) while (pch != NULL) { D_ASSERTF(num_msg_sizes <= num_tokens, "Token counting err\n"); - ret = parse_message_sizes_string(pch, - &all_params[num_msg_sizes]); + ret = parse_message_sizes_string(pch, &all_params[num_msg_sizes]); if (ret == 0) num_msg_sizes++; else @@ -1921,8 +1899,7 @@ int main(int argc, char *argv[]) struct st_size_params *realloced_mem; /* This should always succeed since the buffer is shrinking.. */ - D_REALLOC_ARRAY(realloced_mem, all_params, num_tokens + 1, - num_msg_sizes); + D_REALLOC_ARRAY(realloced_mem, all_params, num_tokens + 1, num_msg_sizes); if (realloced_mem == NULL) D_GOTO(cleanup, ret = -DER_NOMEM); all_params = (struct st_size_params *)realloced_mem; @@ -1939,20 +1916,25 @@ int main(int argc, char *argv[]) printf("--group-name argument not specified or is invalid\n"); D_GOTO(cleanup, ret = -DER_INVAL); } + if (ms_endpts == NULL) printf("Warning: No --master-endpoint specified; using this" " command line application as the master endpoint\n"); - if (endpts == NULL || num_endpts == 0) { - printf("Warning: No --endpoint specified; using 0:2 default\n"); + int tag = 0; /* use context 0 as a default one for non-daos case */ + + /* In case of the DAOS engines first 2 contexts are reserved */ + if (use_agent) + tag = 2; + + printf("Warning: No --endpoint specified; using 0:%d default\n", tag); num_endpts = 1; D_ALLOC_ARRAY(endpts, 1); endpts[0].rank = 0; - endpts[0].tag = 2; + endpts[0].tag = tag; } - /* repeat rep_count for each endpoint */ rep_count = rep_count * num_endpts; @@ -1998,28 +1980,19 @@ int main(int argc, char *argv[]) rep_count, max_inflight); /********************* Run the self test *********************/ - ret = run_self_test(all_params, num_msg_sizes, rep_count, - max_inflight, dest_name, ms_endpts, - num_ms_endpts, endpts, num_endpts, - output_megabits, buf_alignment, attach_info_path, - use_daos_agent_vars); + ret = run_self_test(all_params, num_msg_sizes, rep_count, max_inflight, dest_name, + ms_endpts, num_ms_endpts, endpts, num_endpts, output_megabits, + buf_alignment, attach_info_path, use_agent); /********************* Clean up *********************/ cleanup: - if (ms_endpts != NULL) { - D_FREE(ms_endpts); - ms_endpts = NULL; - } - if (endpts != NULL) { - D_FREE(endpts); - endpts = NULL; - } - if (all_params != NULL) - D_FREE(all_params); + D_FREE(ms_endpts); + D_FREE(endpts); + D_FREE(all_params); - if (use_daos_agent_vars) { + if (use_agent) dc_mgmt_fini(); - } + d_log_fini(); return ret;