From fc7fa9c292588caf07d3ba275205cfd5db3988d9 Mon Sep 17 00:00:00 2001 From: Alexander A Oganezov Date: Fri, 9 Aug 2024 06:03:52 +0000 Subject: [PATCH 1/8] - Reduce burst timeout error prints when multiple rpcs fail at the same time - Replace error prints with info during state processing, as error is already logged above Required-githooks: true Signed-off-by: Alexander A Oganezov --- src/cart/crt_context.c | 91 +++++++++++++++++++++++------------ src/cart/crt_init.c | 65 +++++++++++++------------ src/cart/crt_internal.h | 29 ++++++++++- src/cart/crt_internal_fns.h | 4 +- src/cart/crt_internal_types.h | 17 +++---- 5 files changed, 128 insertions(+), 78 deletions(-) diff --git a/src/cart/crt_context.c b/src/cart/crt_context.c index 9c11dc32dcb..e470375b93a 100644 --- a/src/cart/crt_context.c +++ b/src/cart/crt_context.c @@ -1059,9 +1059,7 @@ crt_req_timeout_track(struct crt_rpc_priv *rpc_priv) if (rc == 0) { rpc_priv->crp_in_binheap = 1; } else { - RPC_ERROR(rpc_priv, - "d_binheap_insert failed, rc: %d\n", - rc); + RPC_ERROR(rpc_priv, "d_binheap_insert failed, rc: %d\n", rc); RPC_DECREF(rpc_priv); } @@ -1163,10 +1161,10 @@ crt_req_timeout_hdlr(struct crt_rpc_priv *rpc_priv) switch (rpc_priv->crp_state) { case RPC_STATE_INITED: case RPC_STATE_QUEUED: - RPC_ERROR(rpc_priv, "aborting %s rpc to group %s, tgt %d:%d, tgt_uri %s\n", - rpc_priv->crp_state == RPC_STATE_QUEUED ? "queued" : "inited", - grp_priv->gp_pub.cg_grpid, tgt_ep->ep_rank, tgt_ep->ep_tag, - rpc_priv->crp_tgt_uri); + RPC_INFO(rpc_priv, "aborting %s rpc to group %s, tgt %d:%d, tgt_uri %s\n", + rpc_priv->crp_state == RPC_STATE_QUEUED ? "queued" : "inited", + grp_priv->gp_pub.cg_grpid, tgt_ep->ep_rank, tgt_ep->ep_tag, + rpc_priv->crp_tgt_uri); crt_context_req_untrack(rpc_priv); crt_rpc_complete_and_unlock(rpc_priv, -DER_TIMEDOUT); break; @@ -1174,13 +1172,11 @@ crt_req_timeout_hdlr(struct crt_rpc_priv *rpc_priv) ul_req = rpc_priv->crp_ul_req; D_ASSERT(ul_req != NULL); ul_in = crt_req_get(ul_req); - RPC_ERROR(rpc_priv, - "failed due to URI_LOOKUP(rpc_priv %p) to group %s," - "rank %d through PSR %d timedout\n", - container_of(ul_req, struct crt_rpc_priv, crp_pub), - ul_in->ul_grp_id, - ul_in->ul_rank, - ul_req->cr_ep.ep_rank); + RPC_INFO(rpc_priv, + "failed due to URI_LOOKUP(rpc_priv %p) to group %s," + "rank %d through PSR %d timedout\n", + container_of(ul_req, struct crt_rpc_priv, crp_pub), ul_in->ul_grp_id, + ul_in->ul_rank, ul_req->cr_ep.ep_rank); if (crt_gdata.cg_use_sensors) d_tm_inc_counter(crt_ctx->cc_timedout_uri, 1); @@ -1195,11 +1191,9 @@ crt_req_timeout_hdlr(struct crt_rpc_priv *rpc_priv) crt_rpc_unlock(rpc_priv); break; case RPC_STATE_FWD_UNREACH: - RPC_ERROR(rpc_priv, - "failed due to group %s, rank %d, tgt_uri %s can't reach the target\n", - grp_priv->gp_pub.cg_grpid, - tgt_ep->ep_rank, - rpc_priv->crp_tgt_uri); + RPC_INFO(rpc_priv, + "failed due to group %s, rank %d, tgt_uri %s can't reach the target\n", + grp_priv->gp_pub.cg_grpid, tgt_ep->ep_rank, rpc_priv->crp_tgt_uri); crt_context_req_untrack(rpc_priv); crt_rpc_complete_and_unlock(rpc_priv, -DER_UNREACH); break; @@ -1207,12 +1201,14 @@ crt_req_timeout_hdlr(struct crt_rpc_priv *rpc_priv) /* At this point, RPC should always be completed by * Mercury */ - RPC_ERROR(rpc_priv, "aborting in-flight to group %s, rank %d, tgt_uri %s\n", - grp_priv->gp_pub.cg_grpid, tgt_ep->ep_rank, rpc_priv->crp_tgt_uri); + RPC_INFO(rpc_priv, "aborting in-flight to group %s, rank %d, tgt_uri %s\n", + grp_priv->gp_pub.cg_grpid, tgt_ep->ep_rank, rpc_priv->crp_tgt_uri); rc = crt_hg_req_cancel(rpc_priv); if (rc != 0) { - RPC_ERROR(rpc_priv, "crt_hg_req_cancel failed, rc: %d, " - "opc: %#x.\n", rc, rpc_priv->crp_pub.cr_opc); + RPC_WARN(rpc_priv, + "crt_hg_req_cancel failed, rc: %d, " + "opc: %#x.\n", + rc, rpc_priv->crp_pub.cr_opc); crt_context_req_untrack(rpc_priv); } crt_rpc_unlock(rpc_priv); @@ -1231,6 +1227,8 @@ crt_context_timeout_check(struct crt_context *crt_ctx) struct d_binheap_node *bh_node; d_list_t timeout_list; uint64_t ts_now; + int err_to_print = 0; + int left_to_print = 0; D_ASSERT(crt_ctx != NULL); @@ -1258,17 +1256,48 @@ crt_context_timeout_check(struct crt_context *crt_ctx) }; D_MUTEX_UNLOCK(&crt_ctx->cc_mutex); + /* Limit logging when many rpcs time-out at the same time */ + d_list_for_each_entry(rpc_priv, &timeout_list, crp_tmp_link_timeout) { + left_to_print++; + } + + /* TODO: might expose via env in future */ + err_to_print = 1; + /* handle the timeout RPCs */ while ((rpc_priv = d_list_pop_entry(&timeout_list, struct crt_rpc_priv, crp_tmp_link_timeout))) { - RPC_ERROR(rpc_priv, - "ctx_id %d, (status: %#x) timed out (%d seconds), " - "target (%d:%d)\n", - crt_ctx->cc_idx, - rpc_priv->crp_state, - rpc_priv->crp_timeout_sec, - rpc_priv->crp_pub.cr_ep.ep_rank, - rpc_priv->crp_pub.cr_ep.ep_tag); + /* + * This error is annoying and fills the logs. For now prevent bursts of timeouts + * happening at the same time. + * + * Ideally we would also limit to 1 error per target. Can keep track of it in per + * target cache used for hg_addrs. + * + * Extra lookup cost of cache entry would be ok as this is an error case + **/ + + if (err_to_print > 0) { + RPC_ERROR(rpc_priv, + "ctx_id %d, (status: %#x) timed out (%d seconds), " + "target (%d:%d)\n", + crt_ctx->cc_idx, rpc_priv->crp_state, rpc_priv->crp_timeout_sec, + rpc_priv->crp_pub.cr_ep.ep_rank, rpc_priv->crp_pub.cr_ep.ep_tag); + err_to_print--; + left_to_print--; + + if (err_to_print == 0 && left_to_print > 0) + D_ERROR("[%d more rpcs timed out. rest logged at INFO level]\n", + left_to_print); + + } else { + RPC_INFO(rpc_priv, + "ctx_id %d, (status: %#x) timed out (%d seconds), " + "target (%d:%d)\n", + crt_ctx->cc_idx, rpc_priv->crp_state, rpc_priv->crp_timeout_sec, + rpc_priv->crp_pub.cr_ep.ep_rank, rpc_priv->crp_pub.cr_ep.ep_tag); + left_to_print--; + } crt_req_timeout_hdlr(rpc_priv); RPC_DECREF(rpc_priv); diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c index ad95d9f541c..c1b11ceafda 100644 --- a/src/cart/crt_init.c +++ b/src/cart/crt_init.c @@ -208,6 +208,28 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, return DER_SUCCESS; } +#define DUMP_GDATA_FIELD(format, x) D_INFO("\t%s = " format " \n", #x, crt_gdata.x) + +static void +crt_gdata_dump(void) +{ + D_INFO("settings:\n"); + DUMP_GDATA_FIELD("%d", cg_post_init); + DUMP_GDATA_FIELD("%d", cg_post_incr); + DUMP_GDATA_FIELD("%d", cg_timeout); + DUMP_GDATA_FIELD("%d", cg_swim_ctx_idx); + DUMP_GDATA_FIELD("%d", cg_credit_ep_ctx); + DUMP_GDATA_FIELD("%d", cg_iv_inline_limit); + DUMP_GDATA_FIELD("%d", cg_auto_swim_disable); + DUMP_GDATA_FIELD("%d", cg_server); + DUMP_GDATA_FIELD("%d", cg_use_sensors); + DUMP_GDATA_FIELD("%d", cg_provider_is_primary); + DUMP_GDATA_FIELD("0x%lx", cg_rpcid); + DUMP_GDATA_FIELD("%ld", cg_num_cores); + DUMP_GDATA_FIELD("%d", cg_rpc_quota); + D_INFO("\n"); +} + /* first step init - for initializing crt_gdata */ static int data_init(int server, crt_init_options_t *opt) { @@ -219,10 +241,7 @@ static int data_init(int server, crt_init_options_t *opt) uint32_t post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR; int rc = 0; - D_DEBUG(DB_ALL, "initializing crt_gdata...\n"); crt_env_dump(); - D_DEBUG(DB_ALL, "Starting RPCID %#lx. Num cores: %ld\n", - crt_gdata.cg_rpcid, crt_gdata.cg_num_cores); /* Set context post init / post incr to tune number of pre-posted recvs */ crt_env_get(D_POST_INIT, &post_init); @@ -254,14 +273,8 @@ static int data_init(int server, crt_init_options_t *opt) crt_gdata.cg_timeout = CRT_DEFAULT_TIMEOUT_S; else crt_gdata.cg_timeout = timeout; - - D_DEBUG(DB_ALL, "set the global timeout value as %d second.\n", - crt_gdata.cg_timeout); - crt_gdata.cg_swim_ctx_idx = CRT_DEFAULT_PROGRESS_CTX_IDX; - D_DEBUG(DB_ALL, "SWIM context idx=%d\n", crt_gdata.cg_swim_ctx_idx); - /* Override defaults and environment if option is set */ if (opt && opt->cio_use_credits) { credits = opt->cio_ep_credits; @@ -281,23 +294,12 @@ static int data_init(int server, crt_init_options_t *opt) /* This is a workaround for CART-871 if universe size is not set */ crt_env_get(FI_UNIVERSE_SIZE, &fi_univ_size); if (fi_univ_size == 0) { - D_INFO("FI_UNIVERSE_SIZE was not set; setting to 2048\n"); d_setenv("FI_UNIVERSE_SIZE", "2048", 1); } - if (credits == 0) { - D_DEBUG(DB_ALL, "CRT_CREDIT_EP_CTX set as 0, flow control disabled.\n"); - } else if (credits > CRT_MAX_CREDITS_PER_EP_CTX) { - D_DEBUG(DB_ALL, "ENV CRT_CREDIT_EP_CTX's value %d exceed max " - "allowed value, use %d for flow control.\n", - credits, CRT_MAX_CREDITS_PER_EP_CTX); + if (credits > CRT_MAX_CREDITS_PER_EP_CTX) credits = CRT_MAX_CREDITS_PER_EP_CTX; - } else { - D_DEBUG(DB_ALL, "CRT_CREDIT_EP_CTX set as %d for flow " - "control.\n", credits); - } crt_gdata.cg_credit_ep_ctx = credits; - D_ASSERT(crt_gdata.cg_credit_ep_ctx <= CRT_MAX_CREDITS_PER_EP_CTX); /** Enable statistics only for the server side and if requested */ if (opt && opt->cio_use_sensors && server) { @@ -321,6 +323,8 @@ static int data_init(int server, crt_init_options_t *opt) } gdata_init_flag = 1; + crt_gdata_dump(); + return rc; } @@ -422,15 +426,14 @@ __split_arg(char *s_arg_to_split, const char *delim, char **first_arg, char **se return DER_SUCCESS; } - -int +crt_provider_t crt_str_to_provider(const char *str_provider) { - int provider_idx = CRT_PROV_UNKNOWN; - int i; + crt_provider_t prov = CRT_PROV_UNKNOWN; + int i; if (str_provider == NULL) - return provider_idx; + return prov; for (i = 0; crt_na_dict[i].nad_str != NULL; i++) { @@ -439,12 +442,12 @@ crt_str_to_provider(const char *str_provider) (crt_na_dict[i].nad_alt_str && !strncmp(str_provider, crt_na_dict[i].nad_alt_str, strlen(crt_na_dict[i].nad_alt_str) + 1))) { - provider_idx = crt_na_dict[i].nad_type; + prov = crt_na_dict[i].nad_type; break; } } - return provider_idx; + return prov; } static int @@ -456,14 +459,12 @@ check_grpid(crt_group_id_t grpid) return rc; if (crt_validate_grpid(grpid) != 0) { - D_ERROR("grpid contains invalid characters " - "or is too long\n"); + D_ERROR("grpid contains invalid characters or is too long\n"); D_GOTO(out, rc = -DER_INVAL); } if (strcmp(grpid, CRT_DEFAULT_GRPID) == 0) { - D_ERROR("invalid client grpid (same as " - "CRT_DEFAULT_GRPID).\n"); + D_ERROR("invalid client grpid (same as CRT_DEFAULT_GRPID).\n"); D_GOTO(out, rc = -DER_INVAL); } out: diff --git a/src/cart/crt_internal.h b/src/cart/crt_internal.h index 37b667aa23c..58418d770cd 100644 --- a/src/cart/crt_internal.h +++ b/src/cart/crt_internal.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -42,7 +42,7 @@ ##__VA_ARGS__); \ } while (0) -/* Log an error with a RPC descriptor */ +/* Log an error with an RPC descriptor */ #define RPC_ERROR(rpc, fmt, ...) \ do { \ char *_module; \ @@ -55,6 +55,31 @@ ##__VA_ARGS__); \ } while (0) +/* Log a warning with an RPC descriptor */ +#define RPC_WARN(rpc, fmt, ...) \ + do { \ + char *_module; \ + char *_opc; \ + \ + crt_opc_decode((rpc)->crp_pub.cr_opc, &_module, &_opc); \ + D_TRACE_WARN((rpc), "[opc=%#x (%s:%s) rpcid=%#lx rank:tag=%d:%d] " fmt, \ + (rpc)->crp_pub.cr_opc, _module, _opc, (rpc)->crp_req_hdr.cch_rpcid, \ + (rpc)->crp_pub.cr_ep.ep_rank, (rpc)->crp_pub.cr_ep.ep_tag, \ + ##__VA_ARGS__); \ + } while (0) + +/* Log an info message with an RPC descriptor */ +#define RPC_INFO(rpc, fmt, ...) \ + do { \ + char *_module; \ + char *_opc; \ + \ + crt_opc_decode((rpc)->crp_pub.cr_opc, &_module, &_opc); \ + D_TRACE_INFO((rpc), "[opc=%#x (%s:%s) rpcid=%#lx rank:tag=%d:%d] " fmt, \ + (rpc)->crp_pub.cr_opc, _module, _opc, (rpc)->crp_req_hdr.cch_rpcid, \ + (rpc)->crp_pub.cr_ep.ep_rank, (rpc)->crp_pub.cr_ep.ep_tag, \ + ##__VA_ARGS__); \ + } while (0) /** * If \a cond is false, this is equivalent to an RPC_ERROR (i.e., \a mask is * ignored). If \a cond is true, this is equivalent to an RPC_TRACE. diff --git a/src/cart/crt_internal_fns.h b/src/cart/crt_internal_fns.h index 36ec0ff909c..e3883a5320d 100644 --- a/src/cart/crt_internal_fns.h +++ b/src/cart/crt_internal_fns.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -14,8 +14,6 @@ /** crt_init.c */ bool crt_initialized(void); -int crt_str_to_provider(const char *provider); - /** crt_register.c */ int crt_opc_map_create(void); void crt_opc_map_destroy(struct crt_opc_map *map); diff --git a/src/cart/crt_internal_types.h b/src/cart/crt_internal_types.h index 02b58fec3a5..abd904d921d 100644 --- a/src/cart/crt_internal_types.h +++ b/src/cart/crt_internal_types.h @@ -119,16 +119,13 @@ struct crt_gdata { volatile unsigned int cg_refcount; /** flags to keep track of states */ - unsigned int cg_inited : 1, - cg_grp_inited : 1, - cg_swim_inited : 1, - cg_auto_swim_disable : 1, - /** whether it is a client or server */ - cg_server : 1, - /** whether scalable endpoint is enabled */ - cg_use_sensors : 1, - /** whether we are on a primary provider */ - cg_provider_is_primary : 1; + unsigned int cg_inited : 1, cg_grp_inited : 1, cg_swim_inited : 1, cg_auto_swim_disable : 1, + /** whether it is a client or server */ + cg_server : 1, + /** whether metrics are used */ + cg_use_sensors : 1, + /** whether we are on a primary provider */ + cg_provider_is_primary : 1; ATOMIC uint64_t cg_rpcid; /* rpc id */ From bccb10ac2501ccbace83dc6e508e5134c1dbf126 Mon Sep 17 00:00:00 2001 From: Alexander A Oganezov Date: Fri, 9 Aug 2024 15:45:58 +0000 Subject: [PATCH 2/8] - retrigger Required-githooks: true Signed-off-by: Alexander A Oganezov From 6309fd2484310549d92ecb4ba02428156fa3747b Mon Sep 17 00:00:00 2001 From: Alexander A Oganezov Date: Fri, 9 Aug 2024 16:20:32 +0000 Subject: [PATCH 3/8] - remove []'s from print to not confuse log parser Required-githooks: true Signed-off-by: Alexander A Oganezov --- src/cart/crt_context.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cart/crt_context.c b/src/cart/crt_context.c index e470375b93a..2927ce75150 100644 --- a/src/cart/crt_context.c +++ b/src/cart/crt_context.c @@ -1287,7 +1287,7 @@ crt_context_timeout_check(struct crt_context *crt_ctx) left_to_print--; if (err_to_print == 0 && left_to_print > 0) - D_ERROR("[%d more rpcs timed out. rest logged at INFO level]\n", + D_ERROR(" %d more rpcs timed out. rest logged at INFO level\n", left_to_print); } else { From d4fd0db263d2e7f84598256aa7bda9cb874c45c3 Mon Sep 17 00:00:00 2001 From: Alexander A Oganezov Date: Fri, 9 Aug 2024 16:40:14 +0000 Subject: [PATCH 4/8] - nuke 'No more free indices left' debug message Required-githooks: true Signed-off-by: Alexander A Oganezov --- src/cart/crt_group.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cart/crt_group.c b/src/cart/crt_group.c index 8e97959e315..ac5d74ca130 100644 --- a/src/cart/crt_group.c +++ b/src/cart/crt_group.c @@ -2366,6 +2366,8 @@ crt_grp_psr_reload(struct crt_grp_priv *grp_priv) * * Each time node is removed, an corresponding index is added back * to the free index list + * + * Returns index on success or -DER_NOSPACE if all indecies are used up */ static int grp_get_free_index(struct crt_grp_priv *priv) @@ -2375,11 +2377,9 @@ grp_get_free_index(struct crt_grp_priv *priv) free_index = d_list_pop_entry(&priv->gp_membs.cgm_free_indices, struct free_index, fi_link); - - if (!free_index) { - D_DEBUG(DB_ALL, "No more free indices left\n"); + /* Forces caller to expand the list */ + if (!free_index) return -DER_NOSPACE; - } ret = free_index->fi_index; D_FREE(free_index); From 15378bb60b7719584cbf2bad3b2f8340c764f42f Mon Sep 17 00:00:00 2001 From: Alexander A Oganezov Date: Fri, 9 Aug 2024 17:20:15 +0000 Subject: [PATCH 5/8] codespell Required-githooks: true Signed-off-by: Alexander A Oganezov --- src/cart/crt_group.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cart/crt_group.c b/src/cart/crt_group.c index ac5d74ca130..8b9773a080a 100644 --- a/src/cart/crt_group.c +++ b/src/cart/crt_group.c @@ -2367,7 +2367,7 @@ crt_grp_psr_reload(struct crt_grp_priv *grp_priv) * Each time node is removed, an corresponding index is added back * to the free index list * - * Returns index on success or -DER_NOSPACE if all indecies are used up + * Returns index on success or -DER_NOSPACE if all indices are used up */ static int grp_get_free_index(struct crt_grp_priv *priv) From 6c65123d6230953b7333dad2b4d70adf67225b85 Mon Sep 17 00:00:00 2001 From: Alexander A Oganezov Date: Fri, 9 Aug 2024 23:29:09 +0000 Subject: [PATCH 6/8] - remove empty line Required-githooks: true Signed-off-by: Alexander A Oganezov --- src/cart/crt_init.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c index c1b11ceafda..5552a7f4d62 100644 --- a/src/cart/crt_init.c +++ b/src/cart/crt_init.c @@ -227,7 +227,6 @@ crt_gdata_dump(void) DUMP_GDATA_FIELD("0x%lx", cg_rpcid); DUMP_GDATA_FIELD("%ld", cg_num_cores); DUMP_GDATA_FIELD("%d", cg_rpc_quota); - D_INFO("\n"); } /* first step init - for initializing crt_gdata */ From 34d483e7b7fbd92e5c8121a1aff02f858f1340b2 Mon Sep 17 00:00:00 2001 From: Alexander A Oganezov Date: Sat, 10 Aug 2024 07:05:20 +0000 Subject: [PATCH 7/8] - style Required-githooks: true Signed-off-by: Alexander A Oganezov --- src/cart/crt_internal_types.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/cart/crt_internal_types.h b/src/cart/crt_internal_types.h index abd904d921d..dd0ed31a576 100644 --- a/src/cart/crt_internal_types.h +++ b/src/cart/crt_internal_types.h @@ -119,13 +119,17 @@ struct crt_gdata { volatile unsigned int cg_refcount; /** flags to keep track of states */ - unsigned int cg_inited : 1, cg_grp_inited : 1, cg_swim_inited : 1, cg_auto_swim_disable : 1, - /** whether it is a client or server */ - cg_server : 1, + unsigned int cg_inited : 1; + unsigned int cg_grp_inited : 1; + unsigned int cg_swim_inited : 1; + unsigned int cg_auto_swim_disable : 1; + + /** whether it is a client or server */ + unsigned int cg_server : 1, /** whether metrics are used */ - cg_use_sensors : 1, + unsigned int cg_use_sensors : 1, /** whether we are on a primary provider */ - cg_provider_is_primary : 1; + unsigned int cg_provider_is_primary : 1; ATOMIC uint64_t cg_rpcid; /* rpc id */ From 6a31898c983e64964c5823c55449e0c783f09aa2 Mon Sep 17 00:00:00 2001 From: Alexander A Oganezov Date: Sat, 10 Aug 2024 07:09:25 +0000 Subject: [PATCH 8/8] - style Required-githooks: true Signed-off-by: Alexander A Oganezov --- src/cart/crt_internal_types.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cart/crt_internal_types.h b/src/cart/crt_internal_types.h index dd0ed31a576..3443338ed5e 100644 --- a/src/cart/crt_internal_types.h +++ b/src/cart/crt_internal_types.h @@ -125,11 +125,11 @@ struct crt_gdata { unsigned int cg_auto_swim_disable : 1; /** whether it is a client or server */ - unsigned int cg_server : 1, - /** whether metrics are used */ - unsigned int cg_use_sensors : 1, - /** whether we are on a primary provider */ - unsigned int cg_provider_is_primary : 1; + unsigned int cg_server : 1; + /** whether metrics are used */ + unsigned int cg_use_sensors : 1; + /** whether we are on a primary provider */ + unsigned int cg_provider_is_primary : 1; ATOMIC uint64_t cg_rpcid; /* rpc id */