DAOS-8331 client: add client side metrics

src/cart/README.env

@@ -174,3 +174,15 @@ This file lists the environment variables used in CaRT.
  . CRT_TEST_CONT
    When set to 1, orterun does not automatically shut down other servers when
    one server is shutdown. Used in cart internal testing.
+
+ . D_CLIENT_METRICS_ENABLE
+   When set to 1, client side metrics will be collected on each daos client, which
+   can by retrieved by daos_metrics -j job_id on each client.
+
+ . D_CLIENT_METRICS_RETAIN
+   when set to 1, client side metrics will be retained even after the job exits, i.e.
+   those metrics can be retrieved by daos_metrics even after job exits.
+
+ . D_CLIENT_METRICS_DUMP_PATH
+   Set client side metrics dump path(file) for each client, so these metrics will be
+   dumped to the specified file when the job exits.

src/cart/crt_init.c

@@ -53,7 +53,12 @@ static const char      *crt_env_names[] = {"D_PROVIDER",
 					   "D_QUOTA_RPCS",
 					   "D_POST_INIT",
 					   "D_POST_INCR",
-					   "DAOS_SIGNAL_REGISTER"};
+					   "DAOS_SIGNAL_REGISTER",
+					   "D_CLIENT_METRICS_ENABLE",
+					   "D_CLIENT_METRICS_RETAIN",
+					   "D_CLIENT_METRICS_DUMP_PATH",
+
+};
 
 static void
 crt_lib_init(void) __attribute__((__constructor__));

src/client/api/SConscript

@@ -1,7 +1,7 @@
 """Build DAOS client"""
 
 LIBDAOS_SRC = ['agent.c', 'array.c', 'container.c', 'event.c', 'init.c', 'job.c', 'kv.c', 'mgmt.c',
-               'object.c', 'pool.c', 'rpc.c', 'task.c', 'tx.c', 'pipeline.c']
+               'object.c', 'pool.c', 'rpc.c', 'task.c', 'tx.c', 'pipeline.c', 'metrics.c']
 
 
 def scons():

src/client/api/init.c

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -23,6 +23,7 @@
 #include <daos/btree_class.h>
 #include <daos/placement.h>
 #include <daos/job.h>
+#include <daos/metrics.h>
 #if BUILD_PIPELINE
 #include <daos/pipeline.h>
 #endif
@@ -242,19 +243,25 @@ daos_init(void)
 	if (rc != 0)
 		D_GOTO(out_co, rc);
 
+	rc = dc_tm_init();
+	if (rc)
+		D_GOTO(out_obj, rc);
+
 #if BUILD_PIPELINE
 	/** set up pipeline */
 	rc = dc_pipeline_init();
 	if (rc != 0)
-		D_GOTO(out_obj, rc);
+		D_GOTO(out_tm, rc);
 #endif
 	module_initialized++;
 	D_GOTO(unlock, rc = 0);
 
 #if BUILD_PIPELINE
+out_tm:
+	dc_tm_fini();
+#endif
 out_obj:
 	dc_obj_fini();
-#endif
 out_co:
 	dc_cont_fini();
 out_pool:
@@ -309,6 +316,7 @@ daos_fini(void)
 		D_GOTO(unlock, rc);
 	}
 
+	daos_metrics_fini();
 #if BUILD_PIPELINE
 	dc_pipeline_fini();
 #endif
@@ -322,6 +330,7 @@ daos_fini(void)
 		D_ERROR("failed to disconnect some resources may leak, "
 			DF_RC"\n", DP_RC(rc));
 
+	dc_tm_fini();
 	dc_agent_fini();
 	dc_job_fini();
 

src/client/api/metrics.c

@@ -0,0 +1,157 @@
+/*
+ * (C) Copyright 2020-2024 Intel Corporation.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause-Patent
+ */
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/utsname.h>
+#include <daos/common.h>
+#include <daos/job.h>
+#include <daos/tls.h>
+#include <daos/metrics.h>
+#include <gurt/telemetry_common.h>
+#include <gurt/telemetry_consumer.h>
+#include <gurt/telemetry_producer.h>
+
+#define INIT_JOB_NUM		1024
+bool daos_client_metric;
+bool daos_client_metric_retain;
+
+#define MAX_IDS_SIZE(num) (num * D_TM_METRIC_SIZE)
+/* The client side metrics structure looks like
+ * root/job_id/pid/....
+ */
+int
+dc_tm_init(void)
+{
+	int    metrics_tag;
+	pid_t  pid;
+	int    rc;
+
+	d_getenv_bool(DAOS_CLIENT_METRICS_ENABLE, &daos_client_metric);
+	if (!daos_client_metric)
+		return 0;
+
+	rc = dc_tls_key_create();
+	if (rc)
+		D_GOTO(out, rc);
+
+	metrics_tag = D_TM_OPEN_OR_CREATE | D_TM_MULTIPLE_WRITER_LOCK;
+	d_getenv_bool(DAOS_CLIENT_METRICS_RETAIN, &daos_client_metric_retain);
+	if (daos_client_metric_retain)
+		metrics_tag |= D_TM_RETAIN_SHMEM;
+	else
+		metrics_tag |= D_TM_RETAIN_SHMEM_IF_NON_EMPTY;
+
+	rc = d_tm_init(DC_TM_JOB_ROOT_ID, MAX_IDS_SIZE(INIT_JOB_NUM), metrics_tag);
+	if (rc != 0) {
+		DL_ERROR(rc, "init job root id.");
+		return rc;
+	}
+
+	D_INFO("INIT %s metrics\n", dc_jobid);
+	rc = d_tm_add_ephemeral_dir(NULL, MAX_IDS_SIZE(INIT_JOB_NUM), "%s", dc_jobid);
+	if (rc != 0 && rc != -DER_EXIST) {
+		DL_ERROR(rc, "add metric %s failed", dc_jobid);
+		D_GOTO(out, rc);
+	}
+
+	pid = getpid();
+	D_INFO("INIT %s/%u metrics\n", dc_jobid, pid);
+	rc = d_tm_add_ephemeral_dir(NULL, MAX_IDS_SIZE(INIT_JOB_NUM), "%s/%u",
+				    dc_jobid, pid);
+	if (rc != 0) {
+		DL_ERROR(rc, "add metric %s/%u failed.\n", dc_jobid, pid);
+		D_GOTO(out, rc);
+	}
+
+out:
+	if (rc)
+		d_tm_fini();
+
+	return rc;
+}
+
+static void
+iter_dump(struct d_tm_context *ctx, struct d_tm_node_t *node, int level,
+	  char *path, int format, int opt_fields, void *arg)
+{
+	d_tm_print_node(ctx, node, level, path, format, opt_fields, (FILE *)arg);
+}
+
+static int
+dump_tm_file(const char *dump_path)
+{
+	struct d_tm_context *ctx;
+	struct d_tm_node_t  *root;
+	char		    dirname[D_TM_MAX_NAME_LEN] = {0};
+	uint32_t	    filter;
+	FILE		    *dump_file;
+	int		    rc = 0;
+
+	dump_file = fopen(dump_path, "w+");
+	if (dump_file == NULL) {
+		D_INFO("cannot open %s", dump_path);
+		return -DER_INVAL;
+	}
+
+	filter = D_TM_COUNTER | D_TM_DURATION | D_TM_TIMESTAMP | D_TM_MEMINFO |
+		 D_TM_TIMER_SNAPSHOT | D_TM_GAUGE | D_TM_STATS_GAUGE;
+
+	ctx = d_tm_open(DC_TM_JOB_ROOT_ID);
+	if (ctx == NULL)
+		D_GOTO(close, rc = -DER_NOMEM);
+
+	snprintf(dirname, sizeof(dirname), "%s/%u", dc_jobid, getpid());
+	root = d_tm_find_metric(ctx, dirname);
+	if (root == NULL) {
+		printf("No metrics found at: '%s'\n", dirname);
+		D_GOTO(close_ctx, rc = -DER_NONEXIST);
+	}
+
+	d_tm_print_field_descriptors(0, dump_file);
+
+	d_tm_iterate(ctx, root, 0, filter, NULL, D_TM_CSV, 0, iter_dump, dump_file);
+
+close_ctx:
+	d_tm_close(&ctx);
+close:
+	fclose(dump_file);
+	return rc;
+}
+
+void
+dc_tm_fini()
+{
+	pid_t pid = getpid();
+	char *dump_path;
+	int rc;
+
+	if (!daos_client_metric)
+		return;
+
+	dump_path = getenv(DAOS_CLIENT_METRICS_DUMP_PATH);
+	if (dump_path != NULL) {
+		D_INFO("dump path is %s\n", dump_path);
+		dump_tm_file(dump_path);
+	}
+
+	dc_tls_fini();
+	dc_tls_key_delete();
+
+	if (!daos_client_metric_retain) {
+		rc = d_tm_del_ephemeral_dir("%s/%d", dc_jobid, pid);
+		if (rc != 0)
+			DL_ERROR(rc, "delete tm directory %s/%d.", dc_jobid, pid);
+
+		rc = d_tm_try_del_ephemeral_dir("%s", dc_jobid);
+		if (rc != 0)
+			DL_ERROR(rc, "delete tm directory %s/%d.", dc_jobid, pid);
+	}
+
+	D_INFO("delete pid %s/%u\n", dc_jobid, pid);
+	d_tm_fini();
+}

src/common/SConscript

@@ -8,10 +8,10 @@
                 'acl_api.c', 'acl_util.c', 'acl_principal.c', 'cont_props.c',
                 'dedup.c', 'profile.c', 'compression.c', 'compression_isal.c',
                 'compression_qat.c', 'multihash.c', 'multihash_isal.c',
-                'cipher.c', 'cipher_isal.c', 'qat.c', 'fault_domain.c']
-
+                'cipher.c', 'cipher_isal.c', 'qat.c', 'fault_domain.c',
+                'tls.c', 'metrics.c']
 
 def build_daos_common(denv, client):
     """ Building non-pmem version for client's common lib"""
     benv = denv.Clone()
     stack_mmap_files = []