diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 800171c6b86..5fce435168a 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -70,7 +70,7 @@ jobs: - name: Checkout code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Check DAOS ftest tags. - run: \[ ! -x src/tests/ftest/tags.py \] || ./src/tests/ftest/tags.py lint + run: \[ ! -x src/tests/ftest/tags.py \] || ./src/tests/ftest/tags.py lint --verbose flake8-lint: runs-on: ubuntu-22.04 @@ -105,7 +105,7 @@ jobs: max-line-length: '100' args: '--filename */SConscript, SConstruct' - Doxygen: + doxygen: name: Doxygen runs-on: ubuntu-22.04 steps: @@ -146,7 +146,7 @@ jobs: - name: Run pylint check. run: ./utils/cq/daos_pylint.py --git --output-format github - Codespell: + codespell: name: Codespell runs-on: ubuntu-22.04 steps: @@ -160,3 +160,18 @@ jobs: skip: ./src/control/vendor,./src/control/go.sum,./.git ignore_words_file: ci/codespell.ignores builtin: clear,rare,informal,names,en-GB_to_en-US + + linting-summary: + name: Linting Summary + runs-on: ubuntu-22.04 + needs: [isort, shell-check, log-check, ftest-tags, flake8-lint, doxygen, pylint, codespell] + if: (!cancelled()) + steps: + - name: Check if any job failed + run: | + if [[ -z "$(echo "${{ join(needs.*.result, '') }}" | sed -e 's/success//g')" ]]; then + echo "All jobs succeeded" + else + echo "One or more jobs did not succeed" + exit 1 + fi diff --git a/Jenkinsfile b/Jenkinsfile index 19524e6b873..45fc7f5b6fd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -273,22 +273,22 @@ pipeline { booleanParam(name: 'CI_medium_TEST', defaultValue: true, description: 'Run the Functional Hardware Medium test stage') - booleanParam(name: 'CI_medium-md-on-ssd_TEST', + booleanParam(name: 'CI_medium_md_on_ssd_TEST', defaultValue: true, description: 'Run the Functional Hardware Medium MD on SSD test stage') - booleanParam(name: 'CI_medium-verbs-provider_TEST', + booleanParam(name: 'CI_medium_verbs_provider_TEST', defaultValue: true, description: 'Run the Functional Hardware Medium Verbs Provider test stage') - booleanParam(name: 'CI_medium-verbs-provider-md-on-ssd_TEST', + booleanParam(name: 'CI_medium_verbs_provider_md_on_ssd_TEST', defaultValue: true, description: 'Run the Functional Hardware Medium Verbs Provider MD on SSD test stage') - booleanParam(name: 'CI_medium-ucx-provider_TEST', + booleanParam(name: 'CI_medium_ucx_provider_TEST', defaultValue: true, description: 'Run the Functional Hardware Medium UCX Provider test stage') booleanParam(name: 'CI_large_TEST', defaultValue: true, description: 'Run the Functional Hardware Large test stage') - booleanParam(name: 'CI_large-md-on-ssd_TEST', + booleanParam(name: 'CI_large_md_on_ssd_TEST', defaultValue: true, description: 'Run the Functional Hardware Large MD on SSD test stage') string(name: 'CI_UNIT_VM1_LABEL', diff --git a/TAG b/TAG index 6b69ce8fe11..4887290e7e6 100644 --- a/TAG +++ b/TAG @@ -1 +1 @@ -2.5.101-tb +2.7.100-tb diff --git a/VERSION b/VERSION index d76ea973811..b16311e2e18 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5.101 +2.7.100 diff --git a/ci/jira_query.py b/ci/jira_query.py index 06de4db7b7f..124a0ec07b2 100755 --- a/ci/jira_query.py +++ b/ci/jira_query.py @@ -29,7 +29,7 @@ # valid. We've never checked/enforced these before so there have been a lot of values used in the # past. VALID_COMPONENTS = ('agent', 'build', 'ci', 'csum', 'doc', 'gha', 'il', 'md', 'mercury', - 'packaging', 'pil4dfs', 'swim', 'test', 'tools') + 'packaging', 'pil4dfs', 'swim', 'test', 'tools', 'ddb') # Expected ticket prefix. VALID_TICKET_PREFIX = ('DAOS', 'CORCI', 'SRE') diff --git a/debian/changelog b/debian/changelog index 8057eb6a77a..001134122cb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +daos (2.7.100-1) unstable; urgency=medium + [ Phillip Henderson ] + * Bump version to 2.7.100 + + -- Phillip Henderson <phillip.henderson@intel.com> Mon, 20 May 2024 12:37:00 -0500 + daos (2.5.101-5) unstable; urgency=medium [ Lei Huang ] * Add libaio as a dependent package diff --git a/docs/admin/common_tasks.md b/docs/admin/common_tasks.md index fda88976cf6..065edd13d11 100644 --- a/docs/admin/common_tasks.md +++ b/docs/admin/common_tasks.md @@ -10,7 +10,7 @@ This section describes some of the common tasks handled by admins at a high leve 4. Generate certificate files. 5. Copy one of the example configs from `utils/config/examples` to `/etc/daos` and adjust it based on the environment. E.g., `access_points`, -`bdev_class`. +`class`. 6. Check that the directory where the log files will be created exists. E.g., `control_log_file`, `log_file` field in `engines` section. 7. Start `daos_server`. @@ -38,7 +38,7 @@ to server hosts and `daos-client` to client hosts. 4. Generate certificate files and distribute them to all the hosts. 5. Copy one of the example configs from `utils/config/examples` to `/etc/daos` of one of the server hosts and adjust it based on the environment. -E.g., `access_points`, `bdev_class`. +E.g., `access_points`, `class`. 6. Check that the directory where the log files will be created exists. E.g., `control_log_file`, `log_file` field in `engines` section. 7. Start `daos_server`. diff --git a/docs/admin/deployment.md b/docs/admin/deployment.md index 796ae85995e..7044c07a52f 100644 --- a/docs/admin/deployment.md +++ b/docs/admin/deployment.md @@ -1409,18 +1409,18 @@ When the command is run, the pmem kernel devices created on SCM/PMem regions are formatted and mounted based on the parameters provided in the server config file. - `scm_mount` specifies the location of the mountpoint to create. -- `scm_class` can be set to `ram` to use a tmpfs in the situation that no SCM/PMem +- `class` can be set to `ram` to use a tmpfs in the situation that no SCM/PMem is available (`scm_size` dictates the size of tmpfs in GB), when set to `dcpm` the device specified under `scm_list` will be mounted at `scm_mount` path. ### NVMe Format When the command is run, NVMe SSDs are formatted and set up to be used by DAOS -based on the parameters provided in the server config file. +based on the parameters provided in the server config file engine storage sections. -`bdev_class` can be set to `nvme` to use actual NVMe devices with SPDK for DAOS +`class` can be set to `nvme` to use actual NVMe devices with SPDK for DAOS storage. -Other `bdev_class` values can be used for emulation of NVMe storage as specified +Other `class` values can be used for emulation of NVMe storage as specified in the server config file. `bdev_list` identifies devices to use with a list of PCI addresses (this can be populated after viewing results from `storage scan` command). diff --git a/docs/index.md b/docs/index.md index 2e9137fca07..c03d977f9f4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,4 @@ -# Welcome to DAOS Version 2.5! +# Welcome to DAOS Version 2.7! The Distributed Asynchronous Object Storage (DAOS) is an open-source object store designed from the ground up for massively distributed Non @@ -15,11 +15,11 @@ In addition to on-prem deployments, DAOS can be deployed in cloud environments. More information on cloud deployments is available in the [DAOS in the Cloud](./cloud/) section. -The included document versions are associated with DAOS v2.5. +The included document versions are associated with DAOS v2.7. They may also describe features that are currently under development for a future DAOS release. Those features will be clearly marked as such. !!! warning - DAOS v2.5 is the (unstable) master branch for DAOS v2.6 development. + DAOS v2.7 is the (unstable) master branch for DAOS v2.8 development. Use at your own risk. Please consider the [latest](../latest/) stable DAOS release for production deployments. diff --git a/docs/release/release_notes.md b/docs/release/release_notes.md index 16d32eeea7e..7472246e690 100644 --- a/docs/release/release_notes.md +++ b/docs/release/release_notes.md @@ -1,6 +1,6 @@ -# DAOS Version 2.6 Release Notes +# DAOS Version 2.8 Release Notes -DAOS 2.6 is under active development and has not been released yet. +DAOS 2.8 is under active development and has not been released yet. The release is planned for late 2023. In the meantime, please refer to the support document for the [latest](https://docs.daos.io/latest/release/release_notes/) diff --git a/docs/release/support_matrix.md b/docs/release/support_matrix.md index 38326085587..1768c51ee11 100644 --- a/docs/release/support_matrix.md +++ b/docs/release/support_matrix.md @@ -1,6 +1,6 @@ -# DAOS Version 2.6 Support +# DAOS Version 2.8 Support -DAOS 2.6 is under active development and has not been released yet. +DAOS 2.8 is under active development and has not been released yet. The release is planned for late 2023. In the meantime, please refer to the support document for the [latest](https://docs.daos.io/latest/release/support_matrix/) diff --git a/docs/release/upgrading.md b/docs/release/upgrading.md index 0ced1491a20..196d7356fb3 100644 --- a/docs/release/upgrading.md +++ b/docs/release/upgrading.md @@ -1,6 +1,6 @@ -# Upgrading to DAOS Version 2.6 +# Upgrading to DAOS Version 2.8 -DAOS 2.6 is under active development and has not been released yet. +DAOS 2.8 is under active development and has not been released yet. The release is planned for the second half of 2023. In the meantime, please refer to the upgrading information for the [latest](https://docs.daos.io/latest/release/upgrading/) DAOS release. diff --git a/mkdocs.yml b/mkdocs.yml index ccec0e94b38..80fcf2f1d24 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ # Project Information -site_name: DAOS v2.5 +site_name: DAOS v2.7 site_description: Distributed Asynchronous Object Storage site_author: DAOS Project @@ -57,9 +57,9 @@ markdown_extensions: nav: - Release Information: - 'index.md' - - 'Release Notes v2.5': 'release/release_notes.md' - - 'Support Matrix v2.5': 'release/support_matrix.md' - - 'Upgrading to v2.5': 'release/upgrading.md' + - 'Release Notes v2.7': 'release/release_notes.md' + - 'Support Matrix v2.7': 'release/support_matrix.md' + - 'Upgrading to v2.7': 'release/upgrading.md' - Overview: - 'Architecture': 'overview/architecture.md' - 'Storage Model': 'overview/storage.md' @@ -108,5 +108,5 @@ nav: - 'Dev Environment': 'dev/development.md' - 'Contributing': 'dev/contributing.md' - 'DAOS Internals': 'https://github.com/daos-stack/daos/blob/master/src/README.md' - - 'DAOS API Documentation': 'https://docs.daos.io/v2.5/doxygen/html/index.html' + - 'DAOS API Documentation': 'https://docs.daos.io/v2.7/doxygen/html/index.html' # Attention: Don't change that doxygen path to a relative path, or mkdocs will stumble... diff --git a/src/cart/crt_register.c b/src/cart/crt_register.c index a18350986a1..e4d9bfecd79 100644 --- a/src/cart/crt_register.c +++ b/src/cart/crt_register.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -572,13 +572,14 @@ proto_query_cb(const struct crt_cb_info *cb_info) int crt_proto_query_int(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, uint32_t *ver, int count, - crt_proto_query_cb_t cb, void *arg, crt_context_t ctx) + uint32_t timeout, crt_proto_query_cb_t cb, void *arg, crt_context_t ctx) { crt_rpc_t *rpc_req = NULL; crt_context_t crt_ctx; struct crt_proto_query_in *rpc_req_input; struct proto_query_t *proto_query = NULL; uint32_t *tmp_array = NULL; + uint32_t default_timeout; int rc = DER_SUCCESS; if (ver == NULL) { @@ -629,6 +630,21 @@ crt_proto_query_int(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, uint32_t *ver proto_query->pq_user_arg = arg; proto_query->pq_coq->coq_base = base_opc; + if (timeout != 0) { + /** The global timeout may be overwritten by the per context timeout + * so let's use the API to get the actual setting. + */ + rc = crt_req_get_timeout(rpc_req, &default_timeout); + /** Should only fail if invalid parameter */ + D_ASSERT(rc == 0); + + if (timeout < default_timeout) { + rc = crt_req_set_timeout(rpc_req, timeout); + /** Should only fail if invalid parameter */ + D_ASSERT(rc == 0); + } + } + rc = crt_req_send(rpc_req, proto_query_cb, proto_query); if (rc != 0) D_ERROR("crt_req_send() failed: "DF_RC"\n", DP_RC(rc)); @@ -650,17 +666,17 @@ crt_proto_query_int(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, uint32_t *ver } int -crt_proto_query(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, - uint32_t *ver, int count, crt_proto_query_cb_t cb, void *arg) +crt_proto_query(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, uint32_t *ver, int count, + uint32_t timeout, crt_proto_query_cb_t cb, void *arg) { - return crt_proto_query_int(tgt_ep, base_opc, ver, count, cb, arg, NULL); + return crt_proto_query_int(tgt_ep, base_opc, ver, count, timeout, cb, arg, NULL); } int crt_proto_query_with_ctx(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, uint32_t *ver, int count, - crt_proto_query_cb_t cb, void *arg, crt_context_t ctx) + uint32_t timeout, crt_proto_query_cb_t cb, void *arg, crt_context_t ctx) { - return crt_proto_query_int(tgt_ep, base_opc, ver, count, cb, arg, ctx); + return crt_proto_query_int(tgt_ep, base_opc, ver, count, timeout, cb, arg, ctx); } /* local operation, query if base_opc with version number ver is registered. */ diff --git a/src/cart/crt_rpc.c b/src/cart/crt_rpc.c index d48b0f91cf6..ae5dddaa883 100644 --- a/src/cart/crt_rpc.c +++ b/src/cart/crt_rpc.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -441,8 +441,7 @@ crt_register_proto_fi(crt_endpoint_t *ep) if (rc != 0) return -DER_MISC; - rc = crt_proto_query(ep, cpf.cpf_base, &cpf.cpf_ver, - 1, crt_pfi_cb, &pfi); + rc = crt_proto_query(ep, cpf.cpf_base, &cpf.cpf_ver, 1, 0, crt_pfi_cb, &pfi); if (rc != -DER_SUCCESS) D_GOTO(out, rc); @@ -481,8 +480,7 @@ crt_register_proto_ctl(crt_endpoint_t *ep) if (rc != 0) return -DER_MISC; - rc = crt_proto_query(ep, cpf.cpf_base, &cpf.cpf_ver, - 1, crt_pfi_cb, &pfi); + rc = crt_proto_query(ep, cpf.cpf_base, &cpf.cpf_ver, 1, 0, crt_pfi_cb, &pfi); if (rc != -DER_SUCCESS) D_GOTO(out, rc); diff --git a/src/chk/chk_common.c b/src/chk/chk_common.c index c5b0d044c7a..ace1c736791 100644 --- a/src/chk/chk_common.c +++ b/src/chk/chk_common.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -391,39 +391,30 @@ chk_pool_remove_nowait(struct chk_pool_rec *cpr) D_WARN("Failed to delete pool record: "DF_RC"\n", DP_RC(rc)); } -void -chk_pool_start_svc(struct chk_pool_rec *cpr, int *ret) +int +chk_pool_restart_svc(struct chk_pool_rec *cpr) { int rc = 0; - ABT_mutex_lock(cpr->cpr_mutex); + /* Stop the pool, then restart it with full pool service. */ - if (!cpr->cpr_started) { - rc = ds_pool_start_with_svc(cpr->cpr_uuid); - if (rc == 0) - cpr->cpr_started = 1; - else - D_WARN("Cannot start (1) the pool for "DF_UUIDF" after check: "DF_RC"\n", - DP_UUID(cpr->cpr_uuid), DP_RC(rc)); - } + ABT_mutex_lock(cpr->cpr_mutex); + if (!cpr->cpr_start_post) { + if (cpr->cpr_started) + chk_pool_shutdown(cpr, true); - if (cpr->cpr_started && !cpr->cpr_start_post) { - rc = ds_pool_chk_post(cpr->cpr_uuid); + rc = ds_pool_start_after_check(cpr->cpr_uuid); if (rc != 0) { - D_WARN("Cannot post handle (1) pool start for " - DF_UUIDF" after check: "DF_RC"\n", + D_WARN("Cannot start full PS for "DF_UUIDF" after CR check: "DF_RC"\n", DP_UUID(cpr->cpr_uuid), DP_RC(rc)); - /* Failed to post handle pool start, have to stop it. */ - chk_pool_shutdown(cpr, true); } else { + cpr->cpr_started = 1; cpr->cpr_start_post = 1; } } - ABT_mutex_unlock(cpr->cpr_mutex); - if (ret != NULL) - *ret = rc; + return rc; } static void @@ -460,7 +451,7 @@ chk_pool_wait(struct chk_pool_rec *cpr) } void -chk_pool_stop_one(struct chk_instance *ins, uuid_t uuid, int status, uint32_t phase, int *ret) +chk_pool_stop_one(struct chk_instance *ins, uuid_t uuid, uint32_t status, uint32_t phase, int *ret) { struct chk_bookmark *cbk; struct chk_pool_rec *cpr; @@ -805,10 +796,6 @@ chk_pool_handle_notify(struct chk_instance *ins, struct chk_iv *iv) if (iv->ci_pool_status == CHK__CHECK_POOL_STATUS__CPS_CHECKED) { cpr->cpr_done = 1; - if (iv->ci_pool_destroyed) { - cpr->cpr_destroyed = 1; - cpr->cpr_not_export_ps = 1; - } } else if (iv->ci_pool_status == CHK__CHECK_POOL_STATUS__CPS_FAILED || iv->ci_pool_status == CHK__CHECK_POOL_STATUS__CPS_IMPLICATED) { cpr->cpr_skip = 1; @@ -818,24 +805,23 @@ chk_pool_handle_notify(struct chk_instance *ins, struct chk_iv *iv) D_GOTO(out, rc = -DER_NOTAPPLICABLE); } - if (!ins->ci_is_leader && !cpr->cpr_destroyed && cpr->cpr_done) { + if (iv->ci_phase != cbk->cb_phase || iv->ci_pool_status != cbk->cb_pool_status) { + cbk->cb_phase = iv->ci_phase; + cbk->cb_pool_status = iv->ci_pool_status; + uuid_unparse_lower(cpr->cpr_uuid, uuid_str); + rc = chk_bk_update_pool(cbk, uuid_str); + } + + if (rc == 0 && !ins->ci_is_leader && cpr->cpr_done) { if (iv->ci_pool_status == CHK__CHECK_POOL_STATUS__CPS_CHECKED && !cpr->cpr_not_export_ps) { - chk_pool_start_svc(cpr, NULL); + rc = chk_pool_restart_svc(cpr); } else if (ins->ci_sched_running && !ins->ci_sched_exiting) { chk_pool_get(cpr); d_list_add_tail(&cpr->cpr_shutdown_link, &ins->ci_pool_shutdown_list); } } - if (iv->ci_phase != cbk->cb_phase || iv->ci_pool_status != cbk->cb_pool_status || - cpr->cpr_destroyed) { - cbk->cb_phase = iv->ci_phase; - cbk->cb_pool_status = iv->ci_pool_status; - uuid_unparse_lower(cpr->cpr_uuid, uuid_str); - rc = chk_bk_update_pool(cbk, uuid_str); - } - out: if (cpr != NULL) chk_pool_put(cpr); diff --git a/src/chk/chk_engine.c b/src/chk/chk_engine.c index f9e9fad2a31..9113ca22531 100644 --- a/src/chk/chk_engine.c +++ b/src/chk/chk_engine.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -827,7 +827,13 @@ chk_engine_find_dangling_pm(struct chk_pool_rec *cpr, struct pool_map *map) t_comp->co_flags |= PO_COMPF_CHK_DONE; } - /* dangling parent domain. */ + /* + * dangling parent domain. + * + * NOTE: When we arrive here, all the pool membership information have already been + * scanned via chk_engine_pool_mbs_one(). No service for this pool is started + * on the rank (dangling parent domain). So it is safe to "DOWN{OUT}" it. + */ rc = chk_engine_pm_dangling(cpr, map, r_comp, down ? PO_COMP_ST_DOWN : PO_COMP_ST_DOWNOUT); if (rc != 0) @@ -1672,6 +1678,7 @@ chk_engine_pool_ult(void *args) int rc1 = 0; int rc2 = 0; bool update = true; + bool svc_ref = true; D_ASSERT(svc != NULL); D_ASSERT(cpr != NULL); @@ -1821,27 +1828,42 @@ chk_engine_pool_ult(void *args) } chk_engine_pool_notify(cpr); cbk->cb_time.ct_stop_time = time(NULL); - if (likely(update)) + if (likely(update)) { rc1 = chk_bk_update_pool(cbk, uuid_str); + if (unlikely(rc1 != 0)) + goto log; + } + /* + * The pool may has been marked as non-connectable before corruption, re-enable + * it to allow new connection. + * + * NOTE: After chk_pool_restart_svc(), current rank may be not PS leader again. + * To simplify the logic, we enable such flag on current PS leader before + * chk_pool_restart_svc(). If some client tries to connect the pool after + * that (mark connectable) but before chk_pool_restart_svc(), it will get + * -DER_BUSY temporarily until the rank is ready for full pool service. + */ if (cbk->cb_pool_status == CHK__CHECK_POOL_STATUS__CPS_CHECKED && !cpr->cpr_not_export_ps) { - chk_pool_start_svc(cpr, &rc2); - if (cpr->cpr_started && cpr->cpr_start_post) - /* - * The pool may has been marked as non-connectable before - * corruption, re-enable it to allow new connection. - */ - rc2 = ds_pool_mark_connectable(svc); + rc1 = ds_pool_mark_connectable(svc); + if (rc1 == 0) { + svc_ref = false; + ds_pool_svc_put_leader(svc); + rc2 = chk_pool_restart_svc(cpr); + } } } +log: D_CDEBUG(rc != 0 || rc1 != 0 || rc2 != 0, DLOG_ERR, DLOG_INFO, DF_ENGINE" on rank %u exit pool ULT for "DF_UUIDF" with %s stop: %d/%d/%d\n", DP_ENGINE(ins), dss_self_rank(), DP_UUID(cpr->cpr_uuid), cpr->cpr_stop ? "external" : "self", rc, rc1, rc2); - ds_pool_svc_put_leader(svc); + if (svc_ref) + ds_pool_svc_put_leader(svc); + cpr->cpr_done = 1; if (ins->ci_sched_running && !ins->ci_sched_exiting && (cbk->cb_pool_status != CHK__CHECK_POOL_STATUS__CPS_CHECKED || cpr->cpr_not_export_ps)) @@ -2928,7 +2950,7 @@ chk_engine_pool_start(uint64_t gen, uuid_t uuid, uint32_t phase, uint32_t flags) cbk = &cpr->cpr_bk; chk_pool_get(cpr); - rc = ds_pool_start(uuid); + rc = ds_pool_start(uuid, false); if (rc != 0) D_GOTO(put, rc = (rc == -DER_NONEXIST ? 1 : rc)); @@ -3514,3 +3536,26 @@ chk_engine_fini(void) { chk_ins_fini(&chk_engine); } + +int +chk_engine_pool_stop(uuid_t pool_uuid, bool destroy) +{ + uint32_t status; + uint32_t phase; + int rc = 0; + + if (destroy) { + status = CHK__CHECK_POOL_STATUS__CPS_CHECKED; + phase = CHK__CHECK_SCAN_PHASE__CSP_DONE; + } else { + status = CHK__CHECK_POOL_STATUS__CPS_PAUSED; + phase = CHK_INVAL_PHASE; + } + + chk_pool_stop_one(chk_engine, pool_uuid, status, phase, &rc); + + D_INFO(DF_ENGINE" stop pool "DF_UUIDF" with %s: "DF_RC"\n", DP_ENGINE(chk_engine), + DP_UUID(pool_uuid), destroy ? "destroy" : "non-destroy", DP_RC(rc)); + + return rc; +} diff --git a/src/chk/chk_internal.h b/src/chk/chk_internal.h index a11fa5518ef..86868e305ee 100644 --- a/src/chk/chk_internal.h +++ b/src/chk/chk_internal.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -568,7 +568,6 @@ struct chk_iv { uint32_t ci_ins_status; uint32_t ci_pool_status; uint32_t ci_to_leader:1, /* To check leader. */ - ci_pool_destroyed:1, /* Pool has been destroyed. */ ci_from_psl:1; /* From pool service leader. */ }; @@ -696,9 +695,10 @@ void chk_pools_dump(d_list_t *head, int pool_nr, uuid_t pools[]); void chk_pool_remove_nowait(struct chk_pool_rec *cpr); -void chk_pool_start_svc(struct chk_pool_rec *cpr, int *ret); +int chk_pool_restart_svc(struct chk_pool_rec *cpr); -void chk_pool_stop_one(struct chk_instance *ins, uuid_t uuid, int status, uint32_t phase, int *ret); +void chk_pool_stop_one(struct chk_instance *ins, uuid_t uuid, uint32_t status, uint32_t phase, + int *ret); void chk_pool_stop_all(struct chk_instance *ins, uint32_t status, int *ret); @@ -834,7 +834,7 @@ int chk_pool_start_remote(d_rank_list_t *rank_list, uint64_t gen, uuid_t uuid, u int chk_pool_mbs_remote(d_rank_t rank, uint32_t phase, uint64_t gen, uuid_t uuid, char *label, uint64_t seq, uint32_t flags, uint32_t mbs_nr, - struct chk_pool_mbs *mbs_array, struct rsvc_hint *hint); + struct chk_pool_mbs *mbs_array, int *svc_rc, struct rsvc_hint *svc_hint); int chk_report_remote(d_rank_t leader, uint64_t gen, uint32_t cla, uint32_t act, int result, d_rank_t rank, uint32_t target, uuid_t *pool, char *pool_label, diff --git a/src/chk/chk_iv.c b/src/chk/chk_iv.c index eabef148a23..299c1554856 100644 --- a/src/chk/chk_iv.c +++ b/src/chk/chk_iv.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -213,10 +213,10 @@ chk_iv_update(void *ns, struct chk_iv *iv, uint32_t shortcut, uint32_t sync_mode D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, "CHK iv "DF_X64"/"DF_X64" on rank %u, phase %u, ins_status %u, " - "pool_status %u, to_leader %s, from_psl %s, destroyed %s: rc = %d\n", + "pool_status %u, to_leader %s, from_psl %s: rc = %d\n", iv->ci_gen, iv->ci_seq, iv->ci_rank, iv->ci_phase, iv->ci_ins_status, iv->ci_pool_status, iv->ci_to_leader ? "yes" : "no", - iv->ci_from_psl ? "yes" : "no", iv->ci_pool_destroyed ? "yes" : "no", rc); + iv->ci_from_psl ? "yes" : "no", rc); return rc; } diff --git a/src/chk/chk_leader.c b/src/chk/chk_leader.c index f29fbe70f76..c8a28e2eeb7 100644 --- a/src/chk/chk_leader.c +++ b/src/chk/chk_leader.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -315,8 +315,6 @@ chk_leader_post_repair(struct chk_instance *ins, struct chk_pool_rec *cpr, iv.ci_ins_status = ins->ci_bk.cb_ins_status; iv.ci_phase = cbk->cb_phase; iv.ci_pool_status = cbk->cb_pool_status; - if (cpr->cpr_destroyed) - iv.ci_pool_destroyed = 1; /* Synchronously notify the engines that check on the pool got failure. */ rc = chk_iv_update(ins->ci_iv_ns, &iv, CRT_IV_SHORTCUT_NONE, @@ -945,7 +943,8 @@ chk_leader_orphan_pool(struct chk_pool_rec *cpr) * to fix related inconsistency), then notify check engines to remove related * pool record and bookmark. */ - chk_leader_post_repair(ins, cpr, &result, rc <= 0, cpr->cpr_skip ? true : false); + chk_leader_post_repair(ins, cpr, &result, rc <= 0, + cpr->cpr_skip && !cpr->cpr_destroyed ? true : false); return result; } @@ -1867,13 +1866,14 @@ chk_leader_pool_mbs_one(struct chk_pool_rec *cpr) { struct rsvc_client client = { 0 }; crt_endpoint_t ep = { 0 }; - struct rsvc_hint hint = { 0 }; + struct rsvc_hint svc_hint = { 0 }; struct chk_instance *ins = cpr->cpr_ins; struct chk_bookmark *cbk = &ins->ci_bk; d_rank_list_t *ps_ranks = NULL; struct chk_pool_shard *cps; struct ds_pool_clue *clue; uint32_t interval; + int svc_rc = 0; int rc = 0; int rc1; int i = 0; @@ -1925,9 +1925,9 @@ chk_leader_pool_mbs_one(struct chk_pool_rec *cpr) rc = chk_pool_mbs_remote(ep.ep_rank, CHK__CHECK_SCAN_PHASE__CSP_POOL_MBS, cbk->cb_gen, cpr->cpr_uuid, cpr->cpr_label, cpr->cpr_label_seq, cpr->cpr_delay_label ? CMF_REPAIR_LABEL : 0, - cpr->cpr_shard_nr, cpr->cpr_mbs, &hint); + cpr->cpr_shard_nr, cpr->cpr_mbs, &svc_rc, &svc_hint); - rc1 = rsvc_client_complete_rpc(&client, &ep, rc, rc, &hint); + rc1 = rsvc_client_complete_rpc(&client, &ep, rc, svc_rc, &svc_hint); if (rc1 == RSVC_CLIENT_RECHOOSE || (rc1 == RSVC_CLIENT_PROCEED && daos_rpc_retryable_rc(rc))) { dss_sleep(interval); diff --git a/src/chk/chk_rpc.c b/src/chk/chk_rpc.c index 71bcee72e13..84e6ae33857 100644 --- a/src/chk/chk_rpc.c +++ b/src/chk/chk_rpc.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -861,7 +861,7 @@ chk_pool_start_remote(d_rank_list_t *rank_list, uint64_t gen, uuid_t uuid, uint3 int chk_pool_mbs_remote(d_rank_t rank, uint32_t phase, uint64_t gen, uuid_t uuid, char *label, uint64_t seq, uint32_t flags, uint32_t mbs_nr, struct chk_pool_mbs *mbs_array, - struct rsvc_hint *hint) + int *svc_rc, struct rsvc_hint *svc_hint) { crt_rpc_t *req; struct chk_pool_mbs_in *cpmi; @@ -887,17 +887,17 @@ chk_pool_mbs_remote(d_rank_t rank, uint32_t phase, uint64_t gen, uuid_t uuid, ch goto out; cpmo = crt_reply_get(req); - rc = cpmo->cpmo_status; - *hint = cpmo->cpmo_hint; + *svc_rc = cpmo->cpmo_status; + *svc_hint = cpmo->cpmo_hint; out: if (req != NULL) crt_req_decref(req); - D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, + D_CDEBUG(rc != 0 || *svc_rc != 0, DLOG_ERR, DLOG_INFO, "Sent pool ("DF_UUIDF") members and label %s (" - DF_X64") to rank %u with phase %d gen "DF_X64": "DF_RC"\n", - DP_UUID(uuid), label != NULL ? label : "(null)", seq, rank, phase, gen, DP_RC(rc)); + DF_X64") to rank %u with phase %d gen "DF_X64": %d/%d\n", DP_UUID(uuid), + label != NULL ? label : "(null)", seq, rank, phase, gen, rc, *svc_rc); return rc; } diff --git a/src/client/api/container.c b/src/client/api/container.c index 7e87e713779..ba15e1ff56f 100644 --- a/src/client/api/container.c +++ b/src/client/api/container.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2015-2023 Intel Corporation. + * (C) Copyright 2015-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -444,9 +444,9 @@ daos_cont_delete_acl(daos_handle_t coh, enum daos_acl_principal_type type, return dc_task_schedule(task, true); } -int -daos_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group, - daos_event_t *ev) +static int +cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group, bool check_exists, + daos_event_t *ev) { daos_prop_t *prop; uint32_t nr = 0; @@ -459,6 +459,16 @@ daos_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group, return -DER_INVAL; } + if (check_exists) { + uid_t uid; + + rc = daos_acl_principal_to_uid(user, &uid); + if (rc != 0) { + DL_ERROR(rc, "unable to determine local ID for user %s", user); + return rc; + } + } + nr++; } @@ -468,6 +478,16 @@ daos_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group, return -DER_INVAL; } + if (check_exists) { + gid_t gid; + + rc = daos_acl_principal_to_gid(group, &gid); + if (rc != 0) { + DL_ERROR(rc, "unable to determine local ID for group %s", group); + return rc; + } + } + nr++; } @@ -500,6 +520,18 @@ daos_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group, return rc; } +int +daos_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group, daos_event_t *ev) +{ + return cont_set_owner(coh, user, group, true, ev); +} + +int +daos_cont_set_owner_no_check(daos_handle_t coh, d_string_t user, d_string_t group, daos_event_t *ev) +{ + return cont_set_owner(coh, user, group, false, ev); +} + int daos_cont_aggregate(daos_handle_t coh, daos_epoch_t epoch, daos_event_t *ev) { diff --git a/src/client/api/event.c b/src/client/api/event.c index 5a536550fbb..2bbd63653e7 100644 --- a/src/client/api/event.c +++ b/src/client/api/event.c @@ -14,6 +14,7 @@ #define D_LOGFAC DD_FAC(client) #include "client_internal.h" +#include <daos/mgmt.h> #include <daos/rpc.h> /** thread-private event */ @@ -66,7 +67,7 @@ static unsigned int eq_ref; static tse_sched_t daos_sched_g; int -daos_eq_lib_init() +daos_eq_lib_init(crt_init_options_t *crt_info) { int rc; @@ -76,7 +77,7 @@ daos_eq_lib_init() D_GOTO(unlock, rc = 0); } - rc = crt_init_opt(NULL, 0, daos_crt_init_opt_get(false, 1)); + rc = crt_init_opt(NULL, 0, crt_info); if (rc != 0) { D_ERROR("failed to initialize crt: "DF_RC"\n", DP_RC(rc)); D_GOTO(unlock, rc); @@ -110,9 +111,19 @@ daos_eq_lib_init() int daos_eq_lib_reset_after_fork(void) { + crt_init_options_t *crt_info; + int rc; + eq_ref = 0; ev_thpriv_is_init = false; - return daos_eq_lib_init(); + crt_info = daos_crt_init_opt_get(false, 1); + rc = dc_mgmt_net_cfg(NULL, crt_info); + if (rc == 0) + rc = daos_eq_lib_init(crt_info); + D_FREE(crt_info->cio_provider); + D_FREE(crt_info->cio_interface); + D_FREE(crt_info->cio_domain); + return rc; } int diff --git a/src/client/api/init.c b/src/client/api/init.c index 6c3074ad926..01c40e47789 100644 --- a/src/client/api/init.c +++ b/src/client/api/init.c @@ -148,6 +148,7 @@ daos_init(void) struct d_fault_attr_t *d_fault_init; struct d_fault_attr_t *d_fault_mem = NULL; struct d_fault_attr_t d_fault_mem_saved; + crt_init_options_t *crt_info; int rc; D_MUTEX_LOCK(&module_lock); @@ -201,16 +202,20 @@ daos_init(void) if (rc != 0) D_GOTO(out_job, rc); + crt_info = daos_crt_init_opt_get(false, 1); /** * get CaRT configuration (see mgmtModule.handleGetAttachInfo for the * handling of NULL system names) */ - rc = dc_mgmt_net_cfg(NULL); + rc = dc_mgmt_net_cfg(NULL, crt_info); if (rc != 0) D_GOTO(out_attach, rc); /** set up event queue */ - rc = daos_eq_lib_init(); + rc = daos_eq_lib_init(crt_info); + D_FREE(crt_info->cio_provider); + D_FREE(crt_info->cio_interface); + D_FREE(crt_info->cio_domain); if (rc != 0) { D_ERROR("failed to initialize eq_lib: "DF_RC"\n", DP_RC(rc)); D_GOTO(out_attach, rc); diff --git a/src/client/api/rpc.c b/src/client/api/rpc.c index 612ff4d2898..7de6122b710 100644 --- a/src/client/api/rpc.c +++ b/src/client/api/rpc.c @@ -104,6 +104,7 @@ struct rpc_proto { crt_opcode_t base_opc; uint32_t *ver_array; uint32_t array_size; + uint32_t timeout; }; static void @@ -114,8 +115,9 @@ query_cb(struct crt_proto_query_cb_info *cb_info) if (daos_rpc_retryable_rc(cb_info->pq_rc)) { rproto->ep.ep_rank = (rproto->ep.ep_rank + 1) % rproto->nr_ranks; + rproto->timeout += 3; rc = crt_proto_query_with_ctx(&rproto->ep, rproto->base_opc, rproto->ver_array, - rproto->array_size, query_cb, rproto, + rproto->array_size, rproto->timeout, query_cb, rproto, daos_get_crt_ctx()); if (rc) { D_ERROR("crt_proto_query_with_ctx() failed: "DF_RC"\n", DP_RC(rc)); @@ -156,9 +158,10 @@ daos_rpc_proto_query(crt_opcode_t base_opc, uint32_t *ver_array, int count, int rproto->array_size = count; rproto->ep.ep_grp = sys->sy_group; rproto->base_opc = base_opc; + rproto->timeout = 3; - rc = crt_proto_query_with_ctx(&rproto->ep, base_opc, - ver_array, count, query_cb, rproto, ctx); + rc = crt_proto_query_with_ctx(&rproto->ep, base_opc, ver_array, count, rproto->timeout, + query_cb, rproto, ctx); if (rc) { D_ERROR("crt_proto_query_with_ctx() failed: "DF_RC"\n", DP_RC(rc)); D_GOTO(out_free, rc); diff --git a/src/client/api/tests/eq_tests.c b/src/client/api/tests/eq_tests.c index bc94e81f600..b01e86a4872 100644 --- a/src/client/api/tests/eq_tests.c +++ b/src/client/api/tests/eq_tests.c @@ -1257,7 +1257,7 @@ eq_ut_setup(void **state) return rc; } - rc = daos_eq_lib_init(); + rc = daos_eq_lib_init(daos_crt_init_opt_get(false, 1)); if (rc != 0) { print_error("Failed daos_eq_lib_init: %d\n", rc); return rc; diff --git a/src/client/dfs/cont.c b/src/client/dfs/cont.c index fbeed10dc53..43318f31927 100644 --- a/src/client/dfs/cont.c +++ b/src/client/dfs/cont.c @@ -1256,11 +1256,21 @@ dfs_get_size_by_oid(dfs_t *dfs, daos_obj_id_t oid, daos_size_t chunk_size, daos_ return daos_der2errno(rc); } +inline static bool +is_uid_invalid(uid_t uid) +{ + return uid == (uid_t)-1; +} + +inline static bool +is_gid_invalid(gid_t gid) +{ + return gid == (gid_t)-1; +} + int -dfs_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group) +dfs_cont_set_owner(daos_handle_t coh, d_string_t user, uid_t uid, d_string_t group, gid_t gid) { - uid_t uid; - gid_t gid; daos_key_t dkey; d_sg_list_t sgl; d_iov_t sg_iovs[4]; @@ -1325,10 +1335,13 @@ dfs_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group) i++; if (user != NULL) { - rc = daos_acl_principal_to_uid(user, &uid); - if (rc) { - D_ERROR("daos_acl_principal_to_uid() failed: " DF_RC "\n", DP_RC(rc)); - D_GOTO(out_prop, rc = daos_der2errno(rc)); + if (is_uid_invalid(uid)) { + rc = daos_acl_principal_to_uid(user, &uid); + if (rc) { + D_ERROR("daos_acl_principal_to_uid() failed: " DF_RC "\n", + DP_RC(rc)); + D_GOTO(out_prop, rc = EINVAL); + } } d_iov_set(&sg_iovs[i], &uid, sizeof(uid_t)); recxs[i].rx_idx = UID_IDX; @@ -1337,10 +1350,13 @@ dfs_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group) } if (group != NULL) { - rc = daos_acl_principal_to_gid(group, &gid); - if (rc) { - D_ERROR("daos_acl_principal_to_gid() failed: " DF_RC "\n", DP_RC(rc)); - D_GOTO(out_prop, rc = daos_der2errno(rc)); + if (is_gid_invalid(gid)) { + rc = daos_acl_principal_to_gid(group, &gid); + if (rc) { + D_ERROR("daos_acl_principal_to_gid() failed: " DF_RC "\n", + DP_RC(rc)); + D_GOTO(out_prop, rc = EINVAL); + } } d_iov_set(&sg_iovs[i], &gid, sizeof(gid_t)); recxs[i].rx_idx = GID_IDX; @@ -1348,8 +1364,8 @@ dfs_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group) i++; } - /** set the owner ACL */ - rc = daos_cont_set_owner(coh, user, group, NULL); + /* set the owner ACL - already checked user/group are real above, if needed */ + rc = daos_cont_set_owner_no_check(coh, user, group, NULL); if (rc) { D_ERROR("daos_cont_set_owner() failed, " DF_RC "\n", DP_RC(rc)); D_GOTO(out_prop, rc = daos_der2errno(rc)); diff --git a/src/client/dfs/xattr.c b/src/client/dfs/xattr.c index 946165b0eba..6bfe8b04b6b 100644 --- a/src/client/dfs/xattr.c +++ b/src/client/dfs/xattr.c @@ -62,7 +62,10 @@ dfs_setxattr(dfs_t *dfs, dfs_obj_t *obj, const char *name, const void *value, da iods[0].iod_recxs = NULL; iods[0].iod_type = DAOS_IOD_SINGLE; iods[0].iod_size = size; - d_iov_set(&sg_iovs[0], (void *)value, size); + if (value == NULL) + d_iov_set(&sg_iovs[0], NULL, 0); + else + d_iov_set(&sg_iovs[0], (void *)value, size); sgls[0].sg_nr = 1; sgls[0].sg_nr_out = 0; sgls[0].sg_iovs = &sg_iovs[0]; @@ -168,7 +171,10 @@ dfs_getxattr(dfs_t *dfs, dfs_obj_t *obj, const char *name, void *value, daos_siz iod.iod_size = *size; /** set sgl for fetch */ - d_iov_set(&sg_iov, value, *size); + if (value == NULL) + d_iov_set(&sg_iov, NULL, 0); + else + d_iov_set(&sg_iov, value, *size); sgl.sg_nr = 1; sgl.sg_nr_out = 0; sgl.sg_iovs = &sg_iov; diff --git a/src/client/dfuse/pil4dfs/aio.c b/src/client/dfuse/pil4dfs/aio.c index bb44aa04bf5..7573392a246 100644 --- a/src/client/dfuse/pil4dfs/aio.c +++ b/src/client/dfuse/pil4dfs/aio.c @@ -20,41 +20,19 @@ #include "pil4dfs_int.h" -/* the number of EQs for aio contexts. (d_aio_eq_count_g + d_eq_count) <= d_eq_count_max */ -uint16_t d_aio_eq_count_g; - extern _Atomic bool d_daos_inited; extern bool d_compatible_mode; -extern pthread_mutex_t d_lock_aio_eqs_g; -extern uint16_t d_eq_count_max; -extern uint16_t d_eq_count; extern bool d_hook_enabled; extern struct file_obj *d_file_list[MAX_OPENED_FILE]; extern int d_get_fd_redirected(int fd); -struct d_aio_eq { - daos_handle_t eq; - pthread_mutex_t lock; - _Atomic uint64_t n_op_queued; - _Atomic uint64_t n_op_done; -}; - typedef struct d_aio_req_args { d_iov_t iov; d_sg_list_t sgl; }d_aio_req_args_t; -/* The max number of event queues dedicated to AIO */ -#define MAX_NUM_EQ_AIO (16) -/* list of EQs dedicated for aio contexts. */ -static struct d_aio_eq aio_eq_list[MAX_NUM_EQ_AIO]; -/* The accumulated sum of all created aio contexts. It is used to assign an event queue to a newly - * created aio context. - */ -static long int aio_ctx_total_g; - struct d_aio_ev { daos_event_t ev; struct iocb *piocb; @@ -63,22 +41,15 @@ struct d_aio_ev { struct d_aio_ctx { /* the real io_context_t used in libaio */ - io_context_t ctx; + io_context_t ctx; /* the depth of context set by io_setup */ - int depth; - /* the index of the event queue assigned to current aio context */ - int idx_eq; - bool inited; + int depth; + daos_handle_t eq; + bool inited; /* DFS is involved or not for current context */ - bool on_dfs; - _Atomic uint64_t num_op_submitted; - _Atomic uint64_t num_op_done; - pthread_mutex_t lock; - /* The array of finished ev. ev is removed from the array by io_getevents(). */ - struct d_aio_ev **ev_done_array; - int ev_done_h; - int ev_done_t; - int ev_queue_len; + bool on_dfs; + uint64_t n_op_queued; + uint64_t n_op_done; }; typedef struct d_aio_ctx d_aio_ctx_t; @@ -90,24 +61,8 @@ static int (*next_io_cancel)(io_context_t ctx, struct iocb *iocb, struct io_even static int (*next_io_getevents)(io_context_t ctx, long min_nr, long nr, struct io_event *events, struct timespec *timeout); -/* free all EQs allocated for aio */ - -void -d_free_aio_ctx(void) -{ - int i; - int rc; - - for (i = 0; i < d_aio_eq_count_g; i++) { - rc = daos_eq_destroy(aio_eq_list[i].eq, 0); - if (rc) - DL_ERROR(rc, "daos_eq_destroy() failed"); - D_MUTEX_DESTROY(&aio_eq_list[i].lock); - } - d_aio_eq_count_g = 0; -} - /* aio functions return negative errno in case of failure */ + static int create_ev_eq_for_aio(d_aio_ctx_t *aio_ctx); @@ -131,24 +86,14 @@ io_setup(int maxevents, io_context_t *ctxp) io_destroy(*ctxp); return -ENOMEM; } - aio_ctx_obj->ctx = *ctxp; - aio_ctx_obj->depth = maxevents; - atomic_init(&aio_ctx_obj->num_op_submitted, 0); - atomic_init(&aio_ctx_obj->num_op_done, 0); - aio_ctx_obj->inited = false; - aio_ctx_obj->on_dfs = false; - aio_ctx_obj->ev_done_array = NULL; - aio_ctx_obj->ev_done_h = 0; - aio_ctx_obj->ev_done_t = 0; - aio_ctx_obj->ev_queue_len = 0; - *ctxp = (io_context_t)aio_ctx_obj; - - rc = D_MUTEX_INIT(&aio_ctx_obj->lock, NULL); - if (rc) { - io_destroy(*ctxp); - D_FREE(aio_ctx_obj); - return -daos_der2errno(rc); - } + aio_ctx_obj->ctx = *ctxp; + aio_ctx_obj->depth = maxevents; + aio_ctx_obj->eq.cookie = 0; + aio_ctx_obj->n_op_queued = 0; + aio_ctx_obj->n_op_done = 0; + aio_ctx_obj->inited = false; + aio_ctx_obj->on_dfs = false; + *ctxp = (io_context_t)aio_ctx_obj; if (!d_daos_inited) /* daos_init() is not called yet. Call create_ev_eq_for_aio() inside io_submit() */ @@ -166,6 +111,7 @@ io_setup(int maxevents, io_context_t *ctxp) int io_destroy(io_context_t ctx) { + int rc; d_aio_ctx_t *aio_ctx_obj = (d_aio_ctx_t *)ctx; io_context_t ctx_save = aio_ctx_obj->ctx; @@ -173,64 +119,32 @@ io_destroy(io_context_t ctx) next_io_destroy = dlsym(RTLD_NEXT, "io_destroy"); D_ASSERT(next_io_destroy != NULL); } - D_MUTEX_DESTROY(&aio_ctx_obj->lock); - D_FREE(aio_ctx_obj->ev_done_array); + rc = daos_eq_destroy(aio_ctx_obj->eq, 0); D_FREE(aio_ctx_obj); + if (rc) + return (-daos_der2errno(rc)); + return next_io_destroy(ctx_save); } static int create_ev_eq_for_aio(d_aio_ctx_t *aio_ctx) { - int rc; - int num_aio_eq_free, num_aio_eq_create; - - D_MUTEX_LOCK(&d_lock_aio_eqs_g); - num_aio_eq_free = (int)d_eq_count_max - (int)d_eq_count - (int)d_aio_eq_count_g; - num_aio_eq_create = min(MAX_NUM_EQ_AIO - d_aio_eq_count_g, num_aio_eq_free); - /* only one EQ for an aio context*/ - num_aio_eq_create = min(num_aio_eq_create, 1); - - if (num_aio_eq_create > 0) { - /* allocate EQs for aio context*/ - rc = daos_eq_create(&aio_eq_list[d_aio_eq_count_g].eq); - if (rc) - goto err; - D_MUTEX_INIT(&aio_eq_list[d_aio_eq_count_g].lock, NULL); - atomic_store_relaxed(&aio_eq_list[d_aio_eq_count_g].n_op_queued, 0); - atomic_store_relaxed(&aio_eq_list[d_aio_eq_count_g].n_op_done, 0); - d_aio_eq_count_g += num_aio_eq_create; - } - aio_ctx->idx_eq = aio_ctx_total_g % d_aio_eq_count_g; - aio_ctx_total_g ++; - D_MUTEX_UNLOCK(&d_lock_aio_eqs_g); + int rc; - if (d_aio_eq_count_g == 0) { - DS_ERROR(EBUSY, "no EQs created for AIO contexts"); - return EBUSY; - } + if (aio_ctx->inited) + return 0; - D_MUTEX_LOCK(&aio_ctx->lock); - if (aio_ctx->inited == false) { - D_ALLOC_ARRAY(aio_ctx->ev_done_array, aio_ctx->depth); - if (aio_ctx->ev_done_array == NULL) { - D_MUTEX_UNLOCK(&aio_ctx->lock); - return ENOMEM; - } - /* empty queue */ - aio_ctx->ev_done_h = 0; - aio_ctx->ev_done_t = 0; - aio_ctx->ev_done_array[0] = NULL; - aio_ctx->ev_queue_len = 0; - aio_ctx->inited = true; - } + /* allocate EQs for aio context*/ + rc = daos_eq_create(&aio_ctx->eq); + if (rc) + goto err; aio_ctx->on_dfs = true; - D_MUTEX_UNLOCK(&aio_ctx->lock); + aio_ctx->inited = true; return 0; err: - D_MUTEX_UNLOCK(&d_lock_aio_eqs_g); return daos_der2errno(rc); } @@ -250,7 +164,6 @@ io_submit(io_context_t ctx, long nr, struct iocb *ios[]) struct d_aio_ev *ctx_ev = NULL; d_aio_req_args_t *req_args; int i, n_op_dfs, fd, io_depth; - int idx_aio_eq; int rc, rc2; short op; int *fd_directed = NULL; @@ -301,7 +214,6 @@ io_submit(io_context_t ctx, long nr, struct iocb *ios[]) D_GOTO(err, rc); } - idx_aio_eq = aio_ctx_obj->idx_eq; for (i = 0; i < nr; i++) { op = ios[i]->aio_lio_opcode; fd = fd_directed[i] - FD_FILE_BASE; @@ -310,7 +222,7 @@ io_submit(io_context_t ctx, long nr, struct iocb *ios[]) if (ctx_ev == NULL) D_GOTO(err_loop, rc = ENOMEM); - rc = daos_event_init(&ctx_ev->ev, aio_eq_list[idx_aio_eq].eq, NULL); + rc = daos_event_init(&ctx_ev->ev, aio_ctx_obj->eq, NULL); if (rc) { DL_ERROR(rc, "daos_event_init() failed"); D_GOTO(err_loop, rc = daos_der2errno(rc)); @@ -355,8 +267,7 @@ io_submit(io_context_t ctx, long nr, struct iocb *ios[]) D_GOTO(err_loop, rc); } } - atomic_fetch_add_relaxed(&aio_ctx_obj->num_op_submitted, 1); - atomic_fetch_add_relaxed(&aio_eq_list[idx_aio_eq].n_op_queued, 1); + aio_ctx_obj->n_op_queued++; } D_FREE(fd_directed); @@ -398,72 +309,21 @@ io_cancel(io_context_t ctx, struct iocb *iocb, struct io_event *evt) return (-ENOSYS); } -static int -ev_enqueue(struct d_aio_ctx *ctx, struct d_aio_ev *ev) -{ - D_MUTEX_LOCK(&ctx->lock); - D_ASSERT(ctx->depth > ctx->ev_queue_len); - ctx->ev_done_array[ctx->ev_done_t] = ev; - ctx->ev_done_t++; - if (ctx->ev_done_t >= ctx->depth) - ctx->ev_done_t = 0; - ctx->ev_queue_len++; - D_MUTEX_UNLOCK(&ctx->lock); - return 0; -} - -static void -ev_dequeue_batch(struct d_aio_ctx *ctx, long min_nr, long nr, struct io_event *events, int *num_ev) -{ - int rc; - struct d_aio_ev *ev; - - D_MUTEX_LOCK(&ctx->lock); - while (ctx->ev_queue_len > 0 && *num_ev < min_nr) { - /* dequeue one ev record */ - ev = ctx->ev_done_array[ctx->ev_done_h]; - ctx->ev_done_h++; - if (ctx->ev_done_h >= ctx->depth) - ctx->ev_done_h = 0; - ctx->ev_queue_len--; - - events[*num_ev].obj = ev->piocb; - events[*num_ev].res = ev->piocb->u.c.nbytes; - - rc = daos_event_fini(&ev->ev); - if (rc) - DL_ERROR(rc, "daos_event_fini() failed"); - D_FREE(ev); - (*num_ev)++; - } - D_MUTEX_UNLOCK(&ctx->lock); -} - #define AIO_EQ_DEPTH MAX_EQ -/* poll all EQs in our list */ +/* poll the event queue of current aio context */ static void -aio_poll_eqs(struct d_aio_ctx *ctx, long min_nr, long nr, struct io_event *events, int *num_ev) +aio_poll_eq(struct d_aio_ctx *ctx, long min_nr, long nr, struct io_event *events, int *num_ev) { - int j, idx_eq; + int j; int rc, rc2; struct daos_event *eps[AIO_EQ_DEPTH + 1] = {0}; struct d_aio_ev *p_aio_ev; - /* If there exist large number of aio context (larger than the number of EQs), other aio - * contexs could poll the same EQ and append to ev_done_array. ev_dequeue_batch() and - * ev_dequeue_batch() are needed to handle such situations. - */ - ev_dequeue_batch(ctx, min_nr, nr, events, num_ev); - if (*num_ev >= min_nr) + if (ctx->n_op_queued == 0) return; - /* poll the assigned EQ */ - idx_eq = ctx->idx_eq; - if (atomic_load_relaxed(&aio_eq_list[idx_eq].n_op_queued) == 0) - goto after_poll; - - rc = daos_eq_poll(aio_eq_list[idx_eq].eq, 0, DAOS_EQ_NOWAIT, AIO_EQ_DEPTH, eps); + rc = daos_eq_poll(ctx->eq, 0, DAOS_EQ_NOWAIT, min(AIO_EQ_DEPTH, nr - (*num_ev)), eps); if (rc < 0) DL_ERROR(rc, "daos_eq_poll() failed"); @@ -471,33 +331,22 @@ aio_poll_eqs(struct d_aio_ctx *ctx, long min_nr, long nr, struct io_event *event if (eps[j]->ev_error) { DS_ERROR(eps[j]->ev_error, "daos_eq_poll() error"); } else { - atomic_fetch_add_relaxed(&aio_eq_list[idx_eq].n_op_queued, -1); - atomic_fetch_add_relaxed(&aio_eq_list[idx_eq].n_op_done, 1); + ctx->n_op_queued--; + ctx->n_op_done++; p_aio_ev = container_of(eps[j], struct d_aio_ev, ev); - if (p_aio_ev->ctx == ctx) { - /* append to event list */ - D_MUTEX_LOCK(&ctx->lock); - events[*num_ev].obj = p_aio_ev->piocb; - events[*num_ev].res = p_aio_ev->piocb->u.c.nbytes; - - rc2 = daos_event_fini(&p_aio_ev->ev); - if (rc2) - DL_ERROR(rc, "daos_event_fini() failed"); - (*num_ev)++; - D_MUTEX_UNLOCK(&ctx->lock); - D_FREE(p_aio_ev); - } else { - /* need to append context's finished queue */ - ev_enqueue(p_aio_ev->ctx, p_aio_ev); - } + /* append to event list */ + events[*num_ev].obj = p_aio_ev->piocb; + events[*num_ev].res = p_aio_ev->piocb->u.c.nbytes; + + rc2 = daos_event_fini(&p_aio_ev->ev); + if (rc2) + DL_ERROR(rc2, "daos_event_fini() failed"); + (*num_ev)++; + D_FREE(p_aio_ev); } } -after_poll: - /* try again after polling */ - ev_dequeue_batch(ctx, min_nr, nr, events, num_ev); - if (*num_ev >= min_nr) - return; + return; } static int @@ -539,7 +388,7 @@ io_getevents(io_context_t ctx, long min_nr, long nr, struct io_event *events, min_nr = AIO_EQ_DEPTH; while (1) { - aio_poll_eqs(aio_ctx_obj, min_nr, nr, events, &op_done); + aio_poll_eq(aio_ctx_obj, min_nr, nr, events, &op_done); if (op_done >= min_nr) return op_done; if (timeout) { diff --git a/src/client/dfuse/pil4dfs/dfs_dcache.c b/src/client/dfuse/pil4dfs/dfs_dcache.c index 165c4a07746..96bd96b668b 100644 --- a/src/client/dfuse/pil4dfs/dfs_dcache.c +++ b/src/client/dfuse/pil4dfs/dfs_dcache.c @@ -684,7 +684,7 @@ dcache_find_insert_act(dfs_dcache_t *dcache, char *path, size_t path_len, dcache rc_gc = gc_reclaim(dcache); if (rc_gc != 0) - DS_WARN(rc_gc, "Garbage collection of dir cache failed"); + DS_WARN(daos_der2errno(rc_gc), "Garbage collection of dir cache failed"); out: D_FREE(key); diff --git a/src/client/dfuse/pil4dfs/int_dfs.c b/src/client/dfuse/pil4dfs/int_dfs.c index 7178ab03170..e6fdae98ad4 100644 --- a/src/client/dfuse/pil4dfs/int_dfs.c +++ b/src/client/dfuse/pil4dfs/int_dfs.c @@ -110,7 +110,6 @@ static daos_handle_t eq_list[MAX_EQ]; uint16_t d_eq_count_max; uint16_t d_eq_count; static uint16_t eq_idx; -extern uint16_t d_aio_eq_count_g; /* Configuration of the Garbage Collector */ static uint32_t dcache_size_bits; @@ -237,8 +236,6 @@ static pthread_mutex_t lock_mmap; static pthread_mutex_t lock_fd_dup2ed; static pthread_mutex_t lock_eqh; -pthread_mutex_t d_lock_aio_eqs_g; - /* store ! umask to apply on mode when creating file to honor system umask */ static mode_t mode_not_umask; @@ -505,9 +502,6 @@ register_handler(int sig, struct sigaction *old_handler); static void print_summary(void); -extern void -d_free_aio_ctx(void); - static int num_fd_dup2ed; struct fd_dup2 fd_dup2_list[MAX_FD_DUP2ED]; @@ -3762,6 +3756,17 @@ closedir(DIR *dirp) } if (!d_hook_enabled) return next_closedir(dirp); + + _Pragma("GCC diagnostic push") + _Pragma("GCC diagnostic ignored \"-Wnonnull-compare\"") + /* Check whether dirp is NULL or not since application provides dirp */ + if (!dirp) { + D_DEBUG(DB_ANY, "dirp is NULL in closedir(): %d (%s)\n", EINVAL, strerror(EINVAL)); + errno = EINVAL; + return (-1); + } + _Pragma("GCC diagnostic pop") + fd = dirfd(dirp); if (d_compatible_mode && fd < FD_FILE_BASE) { @@ -3776,16 +3781,6 @@ closedir(DIR *dirp) } } - _Pragma("GCC diagnostic push") - _Pragma("GCC diagnostic ignored \"-Wnonnull-compare\"") - /* Check whether dirp is NULL or not since application provides dirp */ - if (!dirp) { - D_DEBUG(DB_ANY, "dirp is NULL in closedir(): %d (%s)\n", EINVAL, strerror(EINVAL)); - errno = EINVAL; - return (-1); - } - _Pragma("GCC diagnostic pop") - if (fd >= FD_DIR_BASE) { free_dirfd(fd - FD_DIR_BASE); return 0; @@ -4255,6 +4250,7 @@ setup_fd_0_1_2(void) if (num_fd_dup2ed == 0) return 0; + D_MUTEX_LOCK(&lock_fd_dup2ed); for (i = 0; i < MAX_FD_DUP2ED; i++) { /* only check fd 0, 1, and 2 */ if (fd_dup2_list[i].fd_src >= 0 && fd_dup2_list[i].fd_src <= 2) { @@ -4266,9 +4262,10 @@ setup_fd_0_1_2(void) /* get a real fd from kernel */ fd_tmp = libc_open(d_file_list[idx]->path, open_flag); if (fd_tmp < 0) { + error_save = errno; fprintf(stderr, "Error: open %s failed. %d (%s)\n", d_file_list[idx]->path, errno, strerror(errno)); - return errno; + D_GOTO(err, error_save); } /* using dup2() to make sure we get desired fd */ fd_new = dup2(fd_tmp, fd); @@ -4277,7 +4274,7 @@ setup_fd_0_1_2(void) fprintf(stderr, "Error: dup2 failed. %d (%s)\n", errno, strerror(errno)); libc_close(fd_tmp); - return error_save; + D_GOTO(err, error_save); } libc_close(fd_tmp); if (libc_lseek(fd, offset, SEEK_SET) == -1) { @@ -4285,11 +4282,16 @@ setup_fd_0_1_2(void) fprintf(stderr, "Error: lseek failed to set offset. %d (%s)\n", errno, strerror(errno)); libc_close(fd); - return error_save; + D_GOTO(err, error_save); } } } + D_MUTEX_UNLOCK(&lock_fd_dup2ed); return 0; + +err: + D_MUTEX_UNLOCK(&lock_fd_dup2ed); + return error_save; } static int @@ -4318,6 +4320,8 @@ reset_daos_env_before_exec(void) d_daos_inited = false; daos_debug_inited = false; context_reset = false; + /* all IO requests go through dfuse */ + d_hook_enabled = false; } return 0; } @@ -6711,9 +6715,6 @@ init_myhook(void) if (rc) return; - rc = D_MUTEX_INIT(&d_lock_aio_eqs_g, NULL); - if (rc) - return; rc = D_MUTEX_INIT(&lock_eqh, NULL); if (rc) return; @@ -6732,22 +6733,26 @@ init_myhook(void) dcache_size_bits = DCACHE_SIZE_BITS; rc = d_getenv_uint32_t("D_IL_DCACHE_SIZE_BITS", &dcache_size_bits); if (rc != -DER_SUCCESS && rc != -DER_NONEXIST) - DS_WARN(rc, "'D_IL_DCACHE_SIZE_BITS' env variable could not be used"); + DS_WARN(daos_der2errno(rc), + "'D_IL_DCACHE_SIZE_BITS' env variable could not be used"); dcache_rec_timeout = DCACHE_REC_TIMEOUT; rc = d_getenv_uint32_t("D_IL_DCACHE_REC_TIMEOUT", &dcache_rec_timeout); if (rc != -DER_SUCCESS && rc != -DER_NONEXIST) - DS_WARN(rc, "'D_IL_DCACHE_REC_TIMEOUT' env variable could not be used"); + DS_WARN(daos_der2errno(rc), + "'D_IL_DCACHE_REC_TIMEOUT' env variable could not be used"); dcache_gc_period = DCACHE_GC_PERIOD; rc = d_getenv_uint32_t("D_IL_DCACHE_GC_PERIOD", &dcache_gc_period); if (rc != -DER_SUCCESS && rc != -DER_NONEXIST) - DS_WARN(rc, "'D_IL_DCACHE_GC_PERIOD' env variable could not be used"); + DS_WARN(daos_der2errno(rc), + "'D_IL_DCACHE_GC_PERIOD' env variable could not be used"); dcache_gc_reclaim_max = DCACHE_GC_RECLAIM_MAX; rc = d_getenv_uint32_t("D_IL_DCACHE_GC_RECLAIM_MAX", &dcache_gc_reclaim_max); if (rc != -DER_SUCCESS && rc != -DER_NONEXIST) - DS_WARN(rc, "'D_IL_DCACHE_GC_RECLAIM_MAX' env variable could not be used"); + DS_WARN(daos_der2errno(rc), + "'D_IL_DCACHE_GC_RECLAIM_MAX' env variable could not be used"); if (dcache_gc_reclaim_max == 0) { D_WARN("'D_IL_DCACHE_GC_RECLAIM_MAX' env variable could not be used: value == 0."); dcache_gc_reclaim_max = DCACHE_GC_RECLAIM_MAX; @@ -6884,9 +6889,6 @@ destroy_all_eqs(void) { int i, rc; - /** destroy EQs created for aio */ - d_free_aio_ctx(); - /** destroy EQs created by threads */ for (i = 0; i < d_eq_count; i++) { rc = daos_eq_destroy(eq_list[i], 0); @@ -6939,7 +6941,6 @@ finalize_myhook(void) finalize_dfs(); - D_MUTEX_DESTROY(&d_lock_aio_eqs_g); D_MUTEX_DESTROY(&lock_eqh); D_MUTEX_DESTROY(&lock_reserve_fd); D_MUTEX_DESTROY(&lock_dfs); @@ -7103,9 +7104,9 @@ get_eqh(daos_handle_t *eqh) rc = pthread_mutex_lock(&lock_eqh); /** create a new EQ if the EQ pool is not full; otherwise round robin EQ use from pool */ - if (d_eq_count >= (d_eq_count_max - d_aio_eq_count_g)) { + if (d_eq_count >= d_eq_count_max) { td_eqh = eq_list[eq_idx++]; - if (eq_idx == (d_eq_count_max - d_aio_eq_count_g)) + if (eq_idx == d_eq_count_max) eq_idx = 0; } else { rc = daos_eq_create(&td_eqh); diff --git a/src/common/tests/SConscript b/src/common/tests/SConscript index 2310bea6215..a99dc6d4801 100644 --- a/src/common/tests/SConscript +++ b/src/common/tests/SConscript @@ -40,6 +40,8 @@ def scons(): LIBS=['daos_common_pmem', 'gurt', 'cmocka']) tenv.d_test_program('checksum_timing', 'checksum_timing.c', LIBS=['daos_common', 'gurt']) tenv.d_test_program('compress_timing', 'compress_timing.c', LIBS=['daos_common', 'gurt']) + tenv.d_test_program('rsvc_tests', ['rsvc_tests.c', '../rsvc.c'], + LIBS=['daos_common', 'gurt', 'cmocka']) unit_env = tenv.Clone() unit_env.AppendUnique(OBJPREFIX='utest_') @@ -78,9 +80,6 @@ def scons(): ['drpc_tests.c', '../drpc.c', '../drpc.pb-c.c', mock_test_utils], LIBS=['protobuf-c', 'daos_common', 'gurt', 'cmocka']) - unit_env.d_test_program('rsvc_tests', ['rsvc_tests.c', '../rsvc.c'], - LIBS=['daos_common', 'gurt', 'cmocka']) - if __name__ == "SCons.Script": scons() diff --git a/src/container/srv_target.c b/src/container/srv_target.c index eb1da2534b9..62369d14f5f 100644 --- a/src/container/srv_target.c +++ b/src/container/srv_target.c @@ -174,9 +174,8 @@ cont_aggregate_runnable(struct ds_cont_child *cont, struct sched_request *req, * see pool_iv_pre_sync(), the IV fetch from the following * ds_cont_csummer_init() will fail anyway. */ - D_DEBUG(DB_EPC, DF_CONT": skip aggregation " - "No pool map yet or stopping %d\n", - DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), + D_DEBUG(DB_EPC, DF_CONT ": skip %s aggregation: No pool map yet or stopping %d\n", + DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), vos_agg ? "VOS" : "EC", pool->sp_stopping); return false; } @@ -207,15 +206,17 @@ cont_aggregate_runnable(struct ds_cont_child *cont, struct sched_request *req, if (cont->sc_props.dcp_dedup_enabled || cont->sc_props.dcp_compress_enabled || cont->sc_props.dcp_encrypt_enabled) { - D_DEBUG(DB_EPC, DF_CONT": skip aggregation for " - "deduped/compressed/encrypted container\n", - DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid)); + D_DEBUG(DB_EPC, + DF_CONT ": skip %s aggregation for deduped/compressed/encrypted" + " container\n", + DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), vos_agg ? "VOS" : "EC"); return false; } /* snapshot list isn't fetched yet */ if (cont->sc_aggregation_max == 0) { - D_DEBUG(DB_EPC, "No aggregation before snapshots fetched\n"); + D_DEBUG(DB_EPC, "No %s aggregation before snapshots fetched\n", + vos_agg ? "VOS" : "EC"); /* fetch snapshot list */ if (dss_get_module_info()->dmi_tgt_id == 0) ds_cont_tgt_snapshots_refresh(cont->sc_pool->spc_uuid, @@ -238,8 +239,8 @@ cont_aggregate_runnable(struct ds_cont_child *cont, struct sched_request *req, if (pool->sp_reclaim == DAOS_RECLAIM_LAZY && dss_xstream_is_busy() && sched_req_space_check(req) == SCHED_SPACE_PRESS_NONE) { - D_DEBUG(DB_EPC, "Pool reclaim strategy is lazy, service is " - "busy and no space pressure\n"); + D_DEBUG(DB_EPC, "Pool reclaim strategy is lazy, service is busy and no space" + " pressure\n"); return false; } @@ -450,9 +451,9 @@ cont_aggregate_interval(struct ds_cont_child *cont, cont_aggregate_cb_t cb, struct sched_request *req = cont2req(cont, param->ap_vos_agg); int rc = 0; - D_DEBUG(DB_EPC, DF_CONT"[%d]: Aggregation ULT started\n", - DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), - dmi->dmi_tgt_id); + D_DEBUG(DB_EPC, DF_CONT "[%d]: %s Aggregation ULT started\n", + DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), dmi->dmi_tgt_id, + param->ap_vos_agg ? "VOS" : "EC"); if (req == NULL) goto out; @@ -474,8 +475,9 @@ cont_aggregate_interval(struct ds_cont_child *cont, cont_aggregate_cb_t cb, break; /* pool destroyed */ } else if (rc < 0) { DL_CDEBUG(rc == -DER_BUSY, DB_EPC, DLOG_ERR, rc, - DF_CONT ": VOS aggregate failed", - DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid)); + DF_CONT ": %s aggregate failed", + DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), + param->ap_vos_agg ? "VOS" : "EC"); } else if (sched_req_space_check(req) != SCHED_SPACE_PRESS_NONE) { /* Don't sleep too long when there is space pressure */ msecs = 2ULL * 100; @@ -487,9 +489,9 @@ cont_aggregate_interval(struct ds_cont_child *cont, cont_aggregate_cb_t cb, sched_req_sleep(req, msecs); } out: - D_DEBUG(DB_EPC, DF_CONT"[%d]: Aggregation ULT stopped\n", - DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), - dmi->dmi_tgt_id); + D_DEBUG(DB_EPC, DF_CONT "[%d]: %s Aggregation ULT stopped\n", + DP_CONT(cont->sc_pool->spc_uuid, cont->sc_uuid), dmi->dmi_tgt_id, + param->ap_vos_agg ? "VOS" : "EC"); } static int @@ -657,6 +659,10 @@ cont_child_alloc_ref(void *co_uuid, unsigned int ksize, void *po_uuid, cont->sc_dtx_cos_hdl = DAOS_HDL_INVAL; D_INIT_LIST_HEAD(&cont->sc_link); D_INIT_LIST_HEAD(&cont->sc_open_hdls); + cont->sc_dtx_committable_count = 0; + cont->sc_dtx_committable_coll_count = 0; + D_INIT_LIST_HEAD(&cont->sc_dtx_cos_list); + D_INIT_LIST_HEAD(&cont->sc_dtx_coll_list); *link = &cont->sc_list; return 0; @@ -815,6 +821,10 @@ cont_child_stop(struct ds_cont_child *cont_child) * never be started at all */ cont_child->sc_stopping = 1; + + /* Stop DTX reindex by force. */ + stop_dtx_reindex_ult(cont_child, true); + if (cont_child_started(cont_child)) { D_DEBUG(DB_MD, DF_CONT"[%d]: Stopping container\n", DP_CONT(cont_child->sc_pool->spc_uuid, @@ -895,7 +905,7 @@ cont_child_start(struct ds_pool_child *pool_child, const uuid_t co_uuid, DP_CONT(pool_child->spc_uuid, co_uuid), tgt_id); rc = -DER_SHUTDOWN; } else if (!cont_child_started(cont_child)) { - if (!engine_in_check()) { + if (!ds_pool_skip_for_check(pool_child->spc_pool)) { rc = cont_start_agg(cont_child); if (rc != 0) goto out; @@ -952,58 +962,6 @@ ds_cont_child_start_all(struct ds_pool_child *pool_child) return rc; } -static int -cont_child_chk_post_cb(daos_handle_t ih, vos_iter_entry_t *entry, vos_iter_type_t type, - vos_iter_param_t *iter_param, void *data, unsigned *acts) -{ - struct dsm_tls *tls = dsm_tls_get(); - struct ds_pool_child *pool_child = data; - struct ds_cont_child *cont_child = NULL; - int rc = 0; - - /* The container shard must has been opened. */ - rc = cont_child_lookup(tls->dt_cont_cache, entry->ie_couuid, - pool_child->spc_uuid, false /* create */, &cont_child); - if (rc != 0) - goto out; - - if (cont_child->sc_stopping || !cont_child_started(cont_child)) - D_GOTO(out, rc = -DER_SHUTDOWN); - - rc = cont_start_agg(cont_child); - if (rc != 0) - goto out; - - rc = dtx_cont_register(cont_child); - -out: - if (cont_child != NULL) { - if (rc != 0) - cont_stop_agg(cont_child); - - ds_cont_child_put(cont_child); - } - - D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, - "[%d]: Post handle container "DF_CONTF" start after DAOS check: "DF_RC"\n", - dss_get_module_info()->dmi_tgt_id, - DP_CONT(pool_child->spc_uuid, entry->ie_couuid), DP_RC(rc)); - - return rc; -} - -int -ds_cont_chk_post(struct ds_pool_child *pool_child) -{ - vos_iter_param_t iter_param = { 0 }; - struct vos_iter_anchors anchors = { 0 }; - - iter_param.ip_hdl = pool_child->spc_hdl; - - return vos_iterate(&iter_param, VOS_ITER_COUUID, false, &anchors, - cont_child_chk_post_cb, NULL, (void *)pool_child, NULL); -} - /* ds_cont_hdl ****************************************************************/ static inline struct ds_cont_hdl * @@ -1251,10 +1209,6 @@ cont_child_destroy_one(void *vin) ABT_cond_wait(cont->sc_dtx_resync_cond, cont->sc_mutex); ABT_mutex_unlock(cont->sc_mutex); - /* Give chance to DTX reindex ULT for exit. */ - while (unlikely(cont->sc_dtx_reindex)) - ABT_thread_yield(); - /* Make sure checksum scrubbing has stopped */ ABT_mutex_lock(cont->sc_mutex); if (cont->sc_scrubbing) { @@ -1658,7 +1612,7 @@ ds_cont_local_open(uuid_t pool_uuid, uuid_t cont_hdl_uuid, uuid_t cont_uuid, DF_UUID": %d\n", DP_UUID(cont_uuid), hdl->sch_cont->sc_open); hdl->sch_cont->sc_open--; - dtx_cont_close(hdl->sch_cont); + dtx_cont_close(hdl->sch_cont, true); err_cont: if (daos_handle_is_valid(poh)) { @@ -1792,7 +1746,7 @@ cont_close_hdl(uuid_t cont_hdl_uuid) D_ASSERT(cont_child->sc_open > 0); cont_child->sc_open--; if (cont_child->sc_open == 0) - dtx_cont_close(cont_child); + dtx_cont_close(cont_child, false); D_DEBUG(DB_MD, DF_CONT": closed (%d): hdl="DF_UUID"\n", DP_CONT(cont_child->sc_pool->spc_uuid, cont_child->sc_uuid), diff --git a/src/control/cmd/daos/acl.go b/src/control/cmd/daos/acl.go index 507150ce50a..7cbd3f98d0e 100644 --- a/src/control/cmd/daos/acl.go +++ b/src/control/cmd/daos/acl.go @@ -18,6 +18,18 @@ free_ace_list(char **str, size_t str_count) D_FREE(str[i]); D_FREE(str); } + +uid_t +invalid_uid(void) +{ + return (uid_t)-1; +} + +gid_t +invalid_gid(void) +{ + return (gid_t)-1; +} */ import "C" import ( @@ -360,8 +372,11 @@ func (cmd *containerGetACLCmd) Execute(args []string) error { type containerSetOwnerCmd struct { existingContainerCmd - User string `long:"user" short:"u" description:"user who will own the container"` - Group string `long:"group" short:"g" description:"group who will own the container"` + User string `long:"user" short:"u" description:"user who will own the container"` + Uid *int `long:"uid" description:"with --no-check, specify a uid for POSIX container ownership"` + Group string `long:"group" short:"g" description:"group who will own the container"` + Gid *int `long:"gid" description:"with --no-check, specify a gid for POSIX container ownership"` + NoCheck bool `long:"no-check" description:"don't check whether the user/group exists on the local system"` } func (cmd *containerSetOwnerCmd) Execute(args []string) error { @@ -413,11 +428,41 @@ func (cmd *containerSetOwnerCmd) Execute(args []string) error { lType := C.get_dpe_val(&entries[0]) if lType == C.DAOS_PROP_CO_LAYOUT_POSIX { - if err := dfsError(C.dfs_cont_set_owner(ap.cont, user, group)); err != nil { - return errors.Wrapf(err, "%s failed", fsOpString((ap.fs_op))) + uid := C.invalid_uid() + gid := C.invalid_gid() + if cmd.NoCheck { + if cmd.User != "" { + if cmd.Uid == nil { + return errors.New("POSIX container requires --uid to use --no-check") + } + uid = C.uid_t(*cmd.Uid) + } + + if cmd.Group != "" { + if cmd.Gid == nil { + return errors.New("POSIX container requires --gid to use --no-check") + } + gid = C.gid_t(*cmd.Gid) + } + } else if cmd.Uid != nil || cmd.Gid != nil { + return errors.New("--no-check is required to use the --uid and --gid options") + } + + if err := dfsError(C.dfs_cont_set_owner(ap.cont, user, uid, group, gid)); err != nil { + return errors.Wrapf(err, "failed to set owner for POSIX container %s", + cmd.ContainerID()) } } else { - rc := C.daos_cont_set_owner(ap.cont, user, group, nil) + if cmd.Uid != nil || cmd.Gid != nil { + return errors.New("--uid and --gid options apply for POSIX containers only") + } + + var rc C.int + if cmd.NoCheck { + rc = C.daos_cont_set_owner_no_check(ap.cont, user, group, nil) + } else { + rc = C.daos_cont_set_owner(ap.cont, user, group, nil) + } if err := daosError(rc); err != nil { return errors.Wrapf(err, "failed to set owner for container %s", cmd.ContainerID()) diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 88098815972..1d3b8ce9182 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2021-2023 Intel Corporation. +// (C) Copyright 2021-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -15,11 +15,9 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" - "github.com/daos-stack/daos/src/control/cmd/dmg/pretty" + "github.com/daos-stack/daos/src/control/cmd/daos/pretty" "github.com/daos-stack/daos/src/control/common" - "github.com/daos-stack/daos/src/control/common/proto/convert" - mgmtpb "github.com/daos-stack/daos/src/control/common/proto/mgmt" - "github.com/daos-stack/daos/src/control/lib/control" + "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/ui" ) @@ -196,29 +194,30 @@ type poolQueryCmd struct { poolBaseCmd ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"` ShowDisabledRanks bool `short:"b" long:"show-disabled" description:"Show engine unique identifiers (ranks) which are disabled"` + HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"` } -func convertPoolSpaceInfo(in *C.struct_daos_pool_space, mt C.uint) *mgmtpb.StorageUsageStats { +func convertPoolSpaceInfo(in *C.struct_daos_pool_space, mt C.uint) *daos.StorageUsageStats { if in == nil { return nil } - return &mgmtpb.StorageUsageStats{ + return &daos.StorageUsageStats{ Total: uint64(in.ps_space.s_total[mt]), Free: uint64(in.ps_space.s_free[mt]), Min: uint64(in.ps_free_min[mt]), Max: uint64(in.ps_free_max[mt]), Mean: uint64(in.ps_free_mean[mt]), - MediaType: mgmtpb.StorageMediaType(mt), + MediaType: daos.StorageMediaType(mt), } } -func convertPoolRebuildStatus(in *C.struct_daos_rebuild_status) *mgmtpb.PoolRebuildStatus { +func convertPoolRebuildStatus(in *C.struct_daos_rebuild_status) *daos.PoolRebuildStatus { if in == nil { return nil } - out := &mgmtpb.PoolRebuildStatus{ + out := &daos.PoolRebuildStatus{ Status: int32(in.rs_errno), } if out.Status == 0 { @@ -226,53 +225,43 @@ func convertPoolRebuildStatus(in *C.struct_daos_rebuild_status) *mgmtpb.PoolRebu out.Records = uint64(in.rs_rec_nr) switch { case in.rs_version == 0: - out.State = mgmtpb.PoolRebuildStatus_IDLE + out.State = daos.PoolRebuildStateIdle case C.get_rebuild_state(in) == C.DRS_COMPLETED: - out.State = mgmtpb.PoolRebuildStatus_DONE + out.State = daos.PoolRebuildStateDone default: - out.State = mgmtpb.PoolRebuildStatus_BUSY + out.State = daos.PoolRebuildStateBusy } } return out } -// This is not great... But it allows us to leverage the existing -// pretty printer that dmg uses for this info. Better to find some -// way to unify all of this and remove redundancy/manual conversion. -// -// We're basically doing the same thing as ds_mgmt_drpc_pool_query() -// to stuff the info into a protobuf message and then using the -// automatic conversion from proto to control. Kind of ugly but -// gets the job done. We could potentially create some function -// that's shared between this code and the drpc handlers to deal -// with stuffing the protobuf message but it's probably overkill. -func convertPoolInfo(pinfo *C.daos_pool_info_t) (*control.PoolQueryResp, error) { - pqp := new(mgmtpb.PoolQueryResp) - - pqp.Uuid = uuid.Must(uuidFromC(pinfo.pi_uuid)).String() - pqp.TotalTargets = uint32(pinfo.pi_ntargets) - pqp.DisabledTargets = uint32(pinfo.pi_ndisabled) - pqp.ActiveTargets = uint32(pinfo.pi_space.ps_ntargets) - pqp.TotalEngines = uint32(pinfo.pi_nnodes) - pqp.Leader = uint32(pinfo.pi_leader) - pqp.Version = uint32(pinfo.pi_map_ver) - - pqp.TierStats = []*mgmtpb.StorageUsageStats{ - convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_SCM), - convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_NVME), - } - pqp.Rebuild = convertPoolRebuildStatus(&pinfo.pi_rebuild_st) - - pqr := new(control.PoolQueryResp) - return pqr, convert.Types(pqp, pqr) -} - -const ( - dpiQuerySpace = C.DPI_SPACE - dpiQueryRebuild = C.DPI_REBUILD_STATUS - dpiQueryAll = C.uint64_t(^uint64(0)) // DPI_ALL is -1 -) +func convertPoolInfo(pinfo *C.daos_pool_info_t) (*daos.PoolInfo, error) { + poolInfo := new(daos.PoolInfo) + + poolInfo.QueryMask = daos.PoolQueryMask(pinfo.pi_bits) + poolInfo.UUID = uuid.Must(uuidFromC(pinfo.pi_uuid)) + poolInfo.TotalTargets = uint32(pinfo.pi_ntargets) + poolInfo.DisabledTargets = uint32(pinfo.pi_ndisabled) + poolInfo.ActiveTargets = uint32(pinfo.pi_space.ps_ntargets) + poolInfo.TotalEngines = uint32(pinfo.pi_nnodes) + poolInfo.ServiceLeader = uint32(pinfo.pi_leader) + poolInfo.Version = uint32(pinfo.pi_map_ver) + poolInfo.State = daos.PoolServiceStateReady + if poolInfo.DisabledTargets > 0 { + poolInfo.State = daos.PoolServiceStateDegraded + } + + poolInfo.Rebuild = convertPoolRebuildStatus(&pinfo.pi_rebuild_st) + if poolInfo.QueryMask.HasOption(daos.PoolQueryOptionSpace) { + poolInfo.TierStats = []*daos.StorageUsageStats{ + convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_SCM), + convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_NVME), + } + } + + return poolInfo, nil +} func generateRankSet(ranklist *C.d_rank_list_t) string { if ranklist.rl_nr == 0 { @@ -292,9 +281,20 @@ func generateRankSet(ranklist *C.d_rank_list_t) string { } func (cmd *poolQueryCmd) Execute(_ []string) error { + queryMask := daos.DefaultPoolQueryMask + if cmd.HealthOnly { + queryMask = daos.HealthOnlyPoolQueryMask + } if cmd.ShowEnabledRanks && cmd.ShowDisabledRanks { return errors.New("show-enabled and show-disabled can't be used at the same time.") } + if cmd.ShowEnabledRanks { + queryMask.SetOptions(daos.PoolQueryOptionEnabledEngines) + } + if cmd.ShowDisabledRanks { + queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) + } + var rlPtr **C.d_rank_list_t = nil var rl *C.d_rank_list_t = nil @@ -308,49 +308,41 @@ func (cmd *poolQueryCmd) Execute(_ []string) error { } defer cleanup() - pinfo := C.daos_pool_info_t{ - pi_bits: dpiQueryAll, - } - if cmd.ShowDisabledRanks { - pinfo.pi_bits &= C.uint64_t(^(uint64(C.DPI_ENGINES_ENABLED))) + cPoolInfo := C.daos_pool_info_t{ + pi_bits: C.uint64_t(queryMask), } - rc := C.daos_pool_query(cmd.cPoolHandle, rlPtr, &pinfo, nil, nil) + rc := C.daos_pool_query(cmd.cPoolHandle, rlPtr, &cPoolInfo, nil, nil) defer C.d_rank_list_free(rl) if err := daosError(rc); err != nil { return errors.Wrapf(err, "failed to query pool %s", cmd.poolUUID) } - pqr, err := convertPoolInfo(&pinfo) + poolInfo, err := convertPoolInfo(&cPoolInfo) if err != nil { return err } if rlPtr != nil { if cmd.ShowEnabledRanks { - pqr.EnabledRanks = ranklist.MustCreateRankSet(generateRankSet(rl)) + poolInfo.EnabledRanks = ranklist.MustCreateRankSet(generateRankSet(rl)) } if cmd.ShowDisabledRanks { - pqr.DisabledRanks = ranklist.MustCreateRankSet(generateRankSet(rl)) + poolInfo.DisabledRanks = ranklist.MustCreateRankSet(generateRankSet(rl)) } } - // Update the Pool Query State based on response - err = pqr.UpdateState() - if err != nil { - return err - } - if cmd.JSONOutputEnabled() { - return cmd.OutputJSON(pqr, nil) + return cmd.OutputJSON(poolInfo, nil) } var bld strings.Builder - if err := pretty.PrintPoolQueryResponse(pqr, &bld); err != nil { + if err := pretty.PrintPoolInfo(poolInfo, &bld); err != nil { return err } + cmd.Debugf("Pool query options: %s", poolInfo.QueryMask) cmd.Info(bld.String()) return nil @@ -364,11 +356,11 @@ type poolQueryTargetsCmd struct { } // For using the pretty printer that dmg uses for this target info. -func convertPoolTargetInfo(ptinfo *C.daos_target_info_t) (*control.PoolQueryTargetInfo, error) { - pqti := new(control.PoolQueryTargetInfo) - pqti.Type = control.PoolQueryTargetType(ptinfo.ta_type) - pqti.State = control.PoolQueryTargetState(ptinfo.ta_state) - pqti.Space = []*control.StorageTargetUsage{ +func convertPoolTargetInfo(ptinfo *C.daos_target_info_t) (*daos.PoolQueryTargetInfo, error) { + pqti := new(daos.PoolQueryTargetInfo) + pqti.Type = daos.PoolQueryTargetType(ptinfo.ta_type) + pqti.State = daos.PoolQueryTargetState(ptinfo.ta_state) + pqti.Space = []*daos.StorageUsageStats{ { Total: uint64(ptinfo.ta_space.s_total[C.DAOS_MEDIA_SCM]), Free: uint64(ptinfo.ta_space.s_free[C.DAOS_MEDIA_SCM]), @@ -396,10 +388,10 @@ func (cmd *poolQueryTargetsCmd) Execute(_ []string) error { return errors.WithMessage(err, "parsing target list") } - infoResp := new(control.PoolQueryTargetResp) ptInfo := new(C.daos_target_info_t) var rc C.int + infos := make([]*daos.PoolQueryTargetInfo, 0, len(idxList)) for tgt := 0; tgt < len(idxList); tgt++ { rc = C.daos_pool_query_target(cmd.cPoolHandle, C.uint32_t(idxList[tgt]), C.uint32_t(cmd.Rank), ptInfo, nil) if err := daosError(rc); err != nil { @@ -408,19 +400,21 @@ func (cmd *poolQueryTargetsCmd) Execute(_ []string) error { } tgtInfo, err := convertPoolTargetInfo(ptInfo) - infoResp.Infos = append(infoResp.Infos, tgtInfo) if err != nil { return err } + infos = append(infos, tgtInfo) } if cmd.JSONOutputEnabled() { - return cmd.OutputJSON(infoResp, nil) + return cmd.OutputJSON(infos, nil) } var bld strings.Builder - if err := pretty.PrintPoolQueryTargetResponse(infoResp, &bld); err != nil { - return err + for _, info := range infos { + if err := pretty.PrintPoolQueryTargetInfo(info, &bld); err != nil { + return err + } } cmd.Info(bld.String()) diff --git a/src/control/cmd/daos/pretty/pool.go b/src/control/cmd/daos/pretty/pool.go new file mode 100644 index 00000000000..032a4e72319 --- /dev/null +++ b/src/control/cmd/daos/pretty/pool.go @@ -0,0 +1,100 @@ +// +// (C) Copyright 2020-2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package pretty + +import ( + "fmt" + "io" + + "github.com/dustin/go-humanize" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/txtfmt" +) + +const msgNoPools = "No pools in system" + +func getTierNameText(tierIdx int) string { + switch tierIdx { + case int(daos.StorageMediaTypeScm): + return fmt.Sprintf("- Storage tier %d (SCM):", tierIdx) + case int(daos.StorageMediaTypeNvme): + return fmt.Sprintf("- Storage tier %d (NVMe):", tierIdx) + default: + return fmt.Sprintf("- Storage tier %d (unknown):", tierIdx) + } +} + +// PrintPoolInfo generates a human-readable representation of the supplied +// PoolQueryResp struct and writes it to the supplied io.Writer. +func PrintPoolInfo(pi *daos.PoolInfo, out io.Writer) error { + if pi == nil { + return errors.Errorf("nil %T", pi) + } + w := txtfmt.NewErrWriter(out) + + // Maintain output compatibility with the `daos pool query` output. + fmt.Fprintf(w, "Pool %s, ntarget=%d, disabled=%d, leader=%d, version=%d, state=%s\n", + pi.UUID, pi.TotalTargets, pi.DisabledTargets, pi.ServiceLeader, pi.Version, pi.State) + + if pi.PoolLayoutVer != pi.UpgradeLayoutVer { + fmt.Fprintf(w, "Pool layout out of date (%d < %d) -- see `dmg pool upgrade` for details.\n", + pi.PoolLayoutVer, pi.UpgradeLayoutVer) + } + fmt.Fprintln(w, "Pool health info:") + if pi.EnabledRanks != nil && pi.EnabledRanks.Count() > 0 { + fmt.Fprintf(w, "- Enabled ranks: %s\n", pi.EnabledRanks) + } + if pi.DisabledRanks != nil && pi.DisabledRanks.Count() > 0 { + fmt.Fprintf(w, "- Disabled ranks: %s\n", pi.DisabledRanks) + } + if pi.Rebuild != nil { + if pi.Rebuild.Status == 0 { + fmt.Fprintf(w, "- Rebuild %s, %d objs, %d recs\n", + pi.Rebuild.State, pi.Rebuild.Objects, pi.Rebuild.Records) + } else { + fmt.Fprintf(w, "- Rebuild failed, status=%d\n", pi.Rebuild.Status) + } + } else { + fmt.Fprintln(w, "- No rebuild status available.") + } + + if pi.QueryMask.HasOption(daos.PoolQueryOptionSpace) && pi.TierStats != nil { + fmt.Fprintln(w, "Pool space info:") + fmt.Fprintf(w, "- Target(VOS) count:%d\n", pi.ActiveTargets) + for tierIdx, tierStats := range pi.TierStats { + fmt.Fprintln(w, getTierNameText(tierIdx)) + fmt.Fprintf(w, " Total size: %s\n", humanize.Bytes(tierStats.Total)) + fmt.Fprintf(w, " Free: %s, min:%s, max:%s, mean:%s\n", + humanize.Bytes(tierStats.Free), humanize.Bytes(tierStats.Min), + humanize.Bytes(tierStats.Max), humanize.Bytes(tierStats.Mean)) + } + } + return w.Err +} + +// PrintPoolQueryTargetInfo generates a human-readable representation of the supplied +// PoolQueryTargetResp struct and writes it to the supplied io.Writer. +func PrintPoolQueryTargetInfo(pqti *daos.PoolQueryTargetInfo, out io.Writer) error { + if pqti == nil { + return errors.Errorf("nil %T", pqti) + } + w := txtfmt.NewErrWriter(out) + + // Maintain output compatibility with the `daos pool query-targets` output. + fmt.Fprintf(w, "Target: type %s, state %s\n", pqti.Type, pqti.State) + if pqti.Space != nil { + for tierIdx, tierUsage := range pqti.Space { + fmt.Fprintln(w, getTierNameText(tierIdx)) + fmt.Fprintf(w, " Total size: %s\n", humanize.Bytes(tierUsage.Total)) + fmt.Fprintf(w, " Free: %s\n", humanize.Bytes(tierUsage.Free)) + } + } + + return w.Err +} diff --git a/src/control/cmd/daos/pretty/pool_test.go b/src/control/cmd/daos/pretty/pool_test.go new file mode 100644 index 00000000000..cd3580e047d --- /dev/null +++ b/src/control/cmd/daos/pretty/pool_test.go @@ -0,0 +1,447 @@ +// +// (C) Copyright 2020-2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package pretty + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + + "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" +) + +func TestPretty_PrintPoolInfo(t *testing.T) { + poolUUID := test.MockPoolUUID() + backtickStr := "`" + "dmg pool upgrade" + "`" + for name, tc := range map[string]struct { + pi *daos.PoolInfo + expPrintStr string + }{ + "empty response": { + pi: &daos.PoolInfo{}, + expPrintStr: fmt.Sprintf(` +Pool %s, ntarget=0, disabled=0, leader=0, version=0, state=Creating +Pool health info: +- No rebuild status available. +`, uuid.Nil.String()), + }, + "normal response": { + pi: &daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateDegraded, + UUID: poolUUID, + TotalTargets: 2, + DisabledTargets: 1, + ActiveTargets: 1, + ServiceLeader: 42, + Version: 100, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, + Objects: 42, + Records: 21, + }, + TierStats: []*daos.StorageUsageStats{ + { + Total: 2, + Free: 1, + }, + { + Total: 2, + Free: 1, + }, + }, + }, + expPrintStr: fmt.Sprintf(` +Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded +Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. +Pool health info: +- Rebuild busy, 42 objs, 21 recs +Pool space info: +- Target(VOS) count:1 +- Storage tier 0 (SCM): + Total size: 2 B + Free: 1 B, min:0 B, max:0 B, mean:0 B +- Storage tier 1 (NVMe): + Total size: 2 B + Free: 1 B, min:0 B, max:0 B, mean:0 B +`, poolUUID.String()), + }, + "normal response; enabled ranks": { + pi: &daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateDegraded, + UUID: poolUUID, + TotalTargets: 2, + DisabledTargets: 1, + ActiveTargets: 1, + ServiceLeader: 42, + Version: 100, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + EnabledRanks: ranklist.MustCreateRankSet("[0,1,2]"), + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, + Objects: 42, + Records: 21, + }, + TierStats: []*daos.StorageUsageStats{ + { + Total: 2, + Free: 1, + }, + { + Total: 2, + Free: 1, + }, + }, + }, + expPrintStr: fmt.Sprintf(` +Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded +Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. +Pool health info: +- Enabled ranks: 0-2 +- Rebuild busy, 42 objs, 21 recs +Pool space info: +- Target(VOS) count:1 +- Storage tier 0 (SCM): + Total size: 2 B + Free: 1 B, min:0 B, max:0 B, mean:0 B +- Storage tier 1 (NVMe): + Total size: 2 B + Free: 1 B, min:0 B, max:0 B, mean:0 B +`, poolUUID.String()), + }, + "normal response; disabled ranks": { + pi: &daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateDegraded, + UUID: poolUUID, + TotalTargets: 2, + DisabledTargets: 1, + ActiveTargets: 1, + ServiceLeader: 42, + Version: 100, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + DisabledRanks: ranklist.MustCreateRankSet("[0,1,3]"), + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, + Objects: 42, + Records: 21, + }, + TierStats: []*daos.StorageUsageStats{ + { + Total: 2, + Free: 1, + }, + { + Total: 2, + Free: 1, + }, + }, + }, + expPrintStr: fmt.Sprintf(` +Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded +Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. +Pool health info: +- Disabled ranks: 0-1,3 +- Rebuild busy, 42 objs, 21 recs +Pool space info: +- Target(VOS) count:1 +- Storage tier 0 (SCM): + Total size: 2 B + Free: 1 B, min:0 B, max:0 B, mean:0 B +- Storage tier 1 (NVMe): + Total size: 2 B + Free: 1 B, min:0 B, max:0 B, mean:0 B +`, poolUUID.String()), + }, + "unknown/invalid rebuild state response": { + pi: &daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateDegraded, + UUID: poolUUID, + TotalTargets: 2, + DisabledTargets: 1, + ActiveTargets: 1, + ServiceLeader: 42, + Version: 100, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + DisabledRanks: ranklist.MustCreateRankSet("[0,1,3]"), + Rebuild: &daos.PoolRebuildStatus{ + State: 42, + Objects: 42, + Records: 21, + }, + TierStats: []*daos.StorageUsageStats{ + { + Total: 2, + Free: 1, + }, + { + Total: 2, + Free: 1, + }, + }, + }, + expPrintStr: fmt.Sprintf(` +Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded +Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. +Pool health info: +- Disabled ranks: 0-1,3 +- Rebuild unknown, 42 objs, 21 recs +Pool space info: +- Target(VOS) count:1 +- Storage tier 0 (SCM): + Total size: 2 B + Free: 1 B, min:0 B, max:0 B, mean:0 B +- Storage tier 1 (NVMe): + Total size: 2 B + Free: 1 B, min:0 B, max:0 B, mean:0 B +`, poolUUID.String()), + }, + "rebuild failed": { + pi: &daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateDegraded, + UUID: poolUUID, + TotalTargets: 2, + DisabledTargets: 1, + ActiveTargets: 1, + ServiceLeader: 42, + Version: 100, + PoolLayoutVer: 1, + UpgradeLayoutVer: 2, + Rebuild: &daos.PoolRebuildStatus{ + Status: 2, + State: daos.PoolRebuildStateBusy, + Objects: 42, + Records: 21, + }, + TierStats: []*daos.StorageUsageStats{ + { + Total: 2, + Free: 1, + }, + { + Total: 2, + Free: 1, + }, + }, + }, + expPrintStr: fmt.Sprintf(` +Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded +Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. +Pool health info: +- Rebuild failed, status=2 +Pool space info: +- Target(VOS) count:1 +- Storage tier 0 (SCM): + Total size: 2 B + Free: 1 B, min:0 B, max:0 B, mean:0 B +- Storage tier 1 (NVMe): + Total size: 2 B + Free: 1 B, min:0 B, max:0 B, mean:0 B +`, poolUUID.String()), + }, + } { + t.Run(name, func(t *testing.T) { + var bld strings.Builder + if err := PrintPoolInfo(tc.pi, &bld); err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(strings.TrimLeft(tc.expPrintStr, "\n"), bld.String()); diff != "" { + t.Fatalf("unexpected format string (-want, +got):\n%s\n", diff) + } + }) + } +} + +func TestPretty_PrintPoolQueryTarget(t *testing.T) { + for name, tc := range map[string]struct { + pqti *daos.PoolQueryTargetInfo + expErr error + expPrintStr string + }{ + "nil info": { + expErr: errors.New("nil"), + }, + "valid: single target (unknown, down_out)": { + pqti: &daos.PoolQueryTargetInfo{ + Type: 0, + State: daos.PoolTargetStateDownOut, + Space: []*daos.StorageUsageStats{ + { + Total: 6000000000, + Free: 5000000000, + }, + { + Total: 100000000000, + Free: 90000000000, + }, + }, + }, + expPrintStr: ` +Target: type unknown, state down_out +- Storage tier 0 (SCM): + Total size: 6.0 GB + Free: 5.0 GB +- Storage tier 1 (NVMe): + Total size: 100 GB + Free: 90 GB +`, + }, + "valid: single target (unknown, down)": { + pqti: &daos.PoolQueryTargetInfo{ + Type: 0, + State: daos.PoolTargetStateDown, + Space: []*daos.StorageUsageStats{ + { + Total: 6000000000, + Free: 5000000000, + }, + { + Total: 100000000000, + Free: 90000000000, + }, + }, + }, + expPrintStr: ` +Target: type unknown, state down +- Storage tier 0 (SCM): + Total size: 6.0 GB + Free: 5.0 GB +- Storage tier 1 (NVMe): + Total size: 100 GB + Free: 90 GB +`, + }, + "valid: single target (unknown, up)": { + pqti: &daos.PoolQueryTargetInfo{ + Type: 0, + State: daos.PoolTargetStateUp, + Space: []*daos.StorageUsageStats{ + { + Total: 6000000000, + Free: 5000000000, + }, + { + Total: 100000000000, + Free: 90000000000, + }, + }, + }, + expPrintStr: ` +Target: type unknown, state up +- Storage tier 0 (SCM): + Total size: 6.0 GB + Free: 5.0 GB +- Storage tier 1 (NVMe): + Total size: 100 GB + Free: 90 GB +`, + }, + "valid: single target (unknown, up_in)": { + pqti: &daos.PoolQueryTargetInfo{ + Type: 0, + State: daos.PoolTargetStateUpIn, + Space: []*daos.StorageUsageStats{ + { + Total: 6000000000, + Free: 5000000000, + }, + { + Total: 100000000000, + Free: 90000000000, + }, + }, + }, + expPrintStr: ` +Target: type unknown, state up_in +- Storage tier 0 (SCM): + Total size: 6.0 GB + Free: 5.0 GB +- Storage tier 1 (NVMe): + Total size: 100 GB + Free: 90 GB +`, + }, + "valid: single target (unknown, new)": { + pqti: &daos.PoolQueryTargetInfo{ + Type: 0, + State: daos.PoolTargetStateNew, + Space: []*daos.StorageUsageStats{ + { + Total: 6000000000, + Free: 5000000000, + }, + { + Total: 100000000000, + Free: 90000000000, + }, + }, + }, + expPrintStr: ` +Target: type unknown, state new +- Storage tier 0 (SCM): + Total size: 6.0 GB + Free: 5.0 GB +- Storage tier 1 (NVMe): + Total size: 100 GB + Free: 90 GB +`, + }, + "valid: single target (unknown, drain)": { + pqti: &daos.PoolQueryTargetInfo{ + Type: 0, + State: daos.PoolTargetStateDrain, + Space: []*daos.StorageUsageStats{ + { + Total: 6000000000, + Free: 5000000000, + }, + { + Total: 100000000000, + Free: 90000000000, + }, + }, + }, + expPrintStr: ` +Target: type unknown, state drain +- Storage tier 0 (SCM): + Total size: 6.0 GB + Free: 5.0 GB +- Storage tier 1 (NVMe): + Total size: 100 GB + Free: 90 GB +`, + }, + } { + t.Run(name, func(t *testing.T) { + var bld strings.Builder + gotErr := PrintPoolQueryTargetInfo(tc.pqti, &bld) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return + } + + if diff := cmp.Diff(strings.TrimLeft(tc.expPrintStr, "\n"), bld.String()); diff != "" { + t.Fatalf("unexpected format string (-want, +got):\n%s\n", diff) + } + }) + } +} diff --git a/src/control/cmd/daos_agent/security_rpc_test.go b/src/control/cmd/daos_agent/security_rpc_test.go index 378571e9809..89fb83b8852 100644 --- a/src/control/cmd/daos_agent/security_rpc_test.go +++ b/src/control/cmd/daos_agent/security_rpc_test.go @@ -73,7 +73,7 @@ func setupTestUnixConn(t *testing.T) (*net.UnixConn, func()) { return newConn, cleanup } -func getClientConn(t *testing.T, path string) *drpc.ClientConnection { +func getClientConn(t *testing.T, path string) drpc.DomainSocketClient { client := drpc.NewClientConnection(path) if err := client.Connect(test.Context(t)); err != nil { t.Fatalf("Failed to connect: %v", err) diff --git a/src/control/cmd/daos_server/start.go b/src/control/cmd/daos_server/start.go index 775ee70959d..0d060bfd407 100644 --- a/src/control/cmd/daos_server/start.go +++ b/src/control/cmd/daos_server/start.go @@ -20,18 +20,17 @@ type serverStarter func(logging.Logger, *config.Server) error type startCmd struct { cmdutil.LogCmd cfgCmd - start serverStarter - Port uint16 `short:"p" long:"port" description:"Port for the gRPC management interfect to listen on"` - MountPath string `short:"s" long:"storage" description:"Storage path"` - Modules *string `short:"m" long:"modules" description:"List of server modules to load"` - Targets uint16 `short:"t" long:"targets" description:"Number of targets to use (default use all cores)"` - NrXsHelpers *uint16 `short:"x" long:"xshelpernr" description:"Number of helper XS per VOS target"` - FirstCore *uint16 `short:"f" long:"firstcore" description:"Index of first core for service thread"` - Group string `short:"g" long:"group" description:"Server group name"` - SocketDir string `short:"d" long:"socket_dir" description:"Location for all daos_server & daos_engine sockets"` - Insecure bool `short:"i" long:"insecure" description:"Allow for insecure connections"` - RecreateSuperblocks bool `long:"recreate-superblocks" description:"Recreate missing superblocks rather than failing"` - AutoFormat bool `long:"auto-format" description:"Automatically format storage on server start to bring-up engines without requiring dmg storage format command"` + start serverStarter + Port uint16 `short:"p" long:"port" description:"Port for the gRPC management interfect to listen on"` + MountPath string `short:"s" long:"storage" description:"Storage path"` + Modules *string `short:"m" long:"modules" description:"List of server modules to load"` + Targets uint16 `short:"t" long:"targets" description:"Number of targets to use (default use all cores)"` + NrXsHelpers *uint16 `short:"x" long:"xshelpernr" description:"Number of helper XS per VOS target"` + FirstCore *uint16 `short:"f" long:"firstcore" description:"Index of first core for service thread"` + Group string `short:"g" long:"group" description:"Server group name"` + SocketDir string `short:"d" long:"socket_dir" description:"Location for all daos_server & daos_engine sockets"` + Insecure bool `short:"i" long:"insecure" description:"Allow for insecure connections"` + AutoFormat bool `long:"auto-format" description:"Automatically format storage on server start to bring-up engines without requiring dmg storage format command"` } func (cmd *startCmd) setCLIOverrides() error { @@ -62,9 +61,6 @@ func (cmd *startCmd) setCLIOverrides() error { if cmd.Modules != nil { cmd.config.WithModules(*cmd.Modules) } - if cmd.RecreateSuperblocks { - cmd.Notice("--recreate-superblocks is deprecated and no longer needed to use externally-managed tmpfs") - } for _, srv := range cmd.config.Engines { if cmd.Targets > 0 { diff --git a/src/control/cmd/daos_server/storage_nvme.go b/src/control/cmd/daos_server/storage_nvme.go index 20a2dc7db88..7781fb9159e 100644 --- a/src/control/cmd/daos_server/storage_nvme.go +++ b/src/control/cmd/daos_server/storage_nvme.go @@ -78,10 +78,6 @@ func updateNVMePrepReqFromConfig(log logging.Logger, cfg *config.Server, req *st return nil } - for idx, ec := range cfg.Engines { - ec.ConvertLegacyStorage(log, idx) - } - if cfg.DisableHugepages { return errors.New("hugepage usage has been disabled in the config file") } diff --git a/src/control/cmd/daos_server/storage_nvme_test.go b/src/control/cmd/daos_server/storage_nvme_test.go index d648981e73f..b6016ab4285 100644 --- a/src/control/cmd/daos_server/storage_nvme_test.go +++ b/src/control/cmd/daos_server/storage_nvme_test.go @@ -192,34 +192,6 @@ func TestDaosServer_prepareNVMe(t *testing.T) { cfg: new(config.Server).WithDisableVFIO(true), expErr: errors.New("can not be disabled if running as non-root"), }, - "config parameters applied; legacy storage": { - prepCmd: &prepareNVMeCmd{}, - cfg: new(config.Server).WithEngines( - engine.NewConfig(). - WithLegacyStorage(engine.LegacyStorage{ - BdevClass: storage.ClassNvme, - BdevConfig: storage.BdevConfig{ - DeviceList: storage.MustNewBdevDeviceList(test.MockPCIAddr(7)), - }, - }), - engine.NewConfig(). - WithLegacyStorage(engine.LegacyStorage{ - BdevClass: storage.ClassNvme, - BdevConfig: storage.BdevConfig{ - DeviceList: storage.MustNewBdevDeviceList(test.MockPCIAddr(8)), - }, - }), - ). - WithBdevExclude(test.MockPCIAddr(9)). - WithNrHugepages(1024). - WithDisableVMD(true), - expPrepCall: &storage.BdevPrepareRequest{ - HugepageCount: 1024, - PCIAllowList: fmt.Sprintf("%s%s%s", test.MockPCIAddr(7), storage.BdevPciAddrSep, - test.MockPCIAddr(8)), - PCIBlockList: test.MockPCIAddr(9), - }, - }, "nil config; parameters not applied (simulates effect of --ignore-config)": { prepCmd: &prepareNVMeCmd{}, expPrepCall: &storage.BdevPrepareRequest{ @@ -443,39 +415,6 @@ func TestDaosServer_resetNVMe(t *testing.T) { cfg: new(config.Server).WithDisableVFIO(true), expErr: errors.New("can not be disabled if running as non-root"), }, - "config parameters applied; legacy storage": { - resetCmd: &resetNVMeCmd{}, - cfg: new(config.Server).WithEngines( - engine.NewConfig(). - WithLegacyStorage(engine.LegacyStorage{ - BdevClass: storage.ClassNvme, - BdevConfig: storage.BdevConfig{ - DeviceList: storage.MustNewBdevDeviceList( - test.MockPCIAddr(7)), - }, - }), - engine.NewConfig(). - WithLegacyStorage(engine.LegacyStorage{ - BdevClass: storage.ClassNvme, - BdevConfig: storage.BdevConfig{ - DeviceList: storage.MustNewBdevDeviceList( - test.MockPCIAddr(8)), - }, - }), - ). - WithBdevExclude(test.MockPCIAddr(9)). - WithNrHugepages(1024). - WithDisableVMD(true), - expResetCalls: []storage.BdevPrepareRequest{ - { - PCIAllowList: fmt.Sprintf("%s%s%s", test.MockPCIAddr(7), - storage.BdevPciAddrSep, test.MockPCIAddr(8)), - PCIBlockList: test.MockPCIAddr(9), - TargetUser: getCurrentUsername(t), - Reset_: true, - }, - }, - }, "nil config; parameters not applied (simulates effect of --ignore-config)": { resetCmd: &resetNVMeCmd{}, expResetCalls: []storage.BdevPrepareRequest{ diff --git a/src/control/cmd/dmg/pool.go b/src/control/cmd/dmg/pool.go index 918d5899b88..57924aeeb1d 100644 --- a/src/control/cmd/dmg/pool.go +++ b/src/control/cmd/dmg/pool.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -21,6 +21,7 @@ import ( "github.com/daos-stack/daos/src/control/common" "github.com/daos-stack/daos/src/control/common/cmdutil" "github.com/daos-stack/daos/src/control/lib/control" + "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/ui" "github.com/daos-stack/daos/src/control/logging" @@ -408,16 +409,21 @@ func (cmd *PoolListCmd) Execute(_ []string) (errOut error) { NoQuery: cmd.NoQuery, } - initialResp, err := control.ListPools(cmd.MustLogCtx(), cmd.ctlInvoker, req) + resp, err := control.ListPools(cmd.MustLogCtx(), cmd.ctlInvoker, req) if err != nil { return err // control api returned an error, disregard response } // If rebuild-only pools requested, list the pools which has been rebuild only // and not in idle state, otherwise list all the pools. - resp := new(control.ListPoolsResp) - if err := updateListPoolsResponse(resp, initialResp, cmd.RebuildOnly); err != nil { - return err + if cmd.RebuildOnly { + filtered := resp.Pools[:0] // reuse backing array + for _, p := range resp.Pools { + if p.Rebuild != nil && p.Rebuild.State != daos.PoolRebuildStateIdle { + filtered = append(filtered, p) + } + } + resp.Pools = filtered } if cmd.JSONOutputEnabled() { @@ -438,17 +444,6 @@ func (cmd *PoolListCmd) Execute(_ []string) (errOut error) { return resp.Errors() } -// Update the pool list, which has been rebuild and not in idle state. -func updateListPoolsResponse(finalResp *control.ListPoolsResp, resp *control.ListPoolsResp, rebuildOnly bool) error { - for _, pool := range resp.Pools { - if !rebuildOnly || pool.RebuildState != "idle" { - finalResp.Pools = append(finalResp.Pools, pool) - } - } - - return nil -} - type PoolID struct { ui.LabelOrUUIDFlag } @@ -638,20 +633,29 @@ type PoolQueryCmd struct { poolCmd ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"` ShowDisabledRanks bool `short:"b" long:"show-disabled" description:"Show engine unique identifiers (ranks) which are disabled"` + HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"` } // Execute is run when PoolQueryCmd subcommand is activated func (cmd *PoolQueryCmd) Execute(args []string) error { req := &control.PoolQueryReq{ - ID: cmd.PoolID().String(), + ID: cmd.PoolID().String(), + QueryMask: daos.DefaultPoolQueryMask, } + if cmd.HealthOnly { + req.QueryMask = daos.HealthOnlyPoolQueryMask + } // TODO (DAOS-10250) The two options should not be incompatible (i.e. engine limitation) if cmd.ShowEnabledRanks && cmd.ShowDisabledRanks { return errIncompatFlags("show-enabled-ranks", "show-disabled-ranks") } - req.IncludeEnabledRanks = cmd.ShowEnabledRanks - req.IncludeDisabledRanks = cmd.ShowDisabledRanks + if cmd.ShowEnabledRanks { + req.QueryMask.SetOptions(daos.PoolQueryOptionEnabledEngines) + } + if cmd.ShowDisabledRanks { + req.QueryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) + } resp, err := control.PoolQuery(cmd.MustLogCtx(), cmd.ctlInvoker, req) @@ -667,6 +671,8 @@ func (cmd *PoolQueryCmd) Execute(args []string) error { if err := pretty.PrintPoolQueryResponse(resp, &bld); err != nil { return err } + + cmd.Debugf("Pool query options: %s", resp.PoolInfo.QueryMask) cmd.Info(bld.String()) return nil } diff --git a/src/control/cmd/dmg/pool_test.go b/src/control/cmd/dmg/pool_test.go index eda0492511b..263243f034f 100644 --- a/src/control/cmd/dmg/pool_test.go +++ b/src/control/cmd/dmg/pool_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -27,7 +27,6 @@ import ( "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/logging" - "github.com/daos-stack/daos/src/control/system" ) func Test_Dmg_PoolTierRatioFlag(t *testing.T) { @@ -188,6 +187,12 @@ func TestPoolCommands(t *testing.T) { return prop } + setQueryMask := func(xfrm func(qm *daos.PoolQueryMask)) daos.PoolQueryMask { + qm := daos.DefaultPoolQueryMask + xfrm(&qm) + return qm + } + runCmdTests(t, []cmdTest{ { "Pool create with extra argument", @@ -957,7 +962,8 @@ func TestPoolCommands(t *testing.T) { "pool query 12345678-1234-1234-1234-1234567890ab", strings.Join([]string{ printRequest(t, &control.PoolQueryReq{ - ID: "12345678-1234-1234-1234-1234567890ab", + ID: "12345678-1234-1234-1234-1234567890ab", + QueryMask: daos.DefaultPoolQueryMask, }), }, " "), nil, @@ -967,8 +973,8 @@ func TestPoolCommands(t *testing.T) { "pool query --show-enabled 12345678-1234-1234-1234-1234567890ab", strings.Join([]string{ printRequest(t, &control.PoolQueryReq{ - ID: "12345678-1234-1234-1234-1234567890ab", - IncludeEnabledRanks: true, + ID: "12345678-1234-1234-1234-1234567890ab", + QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetOptions(daos.PoolQueryOptionEnabledEngines) }), }), }, " "), nil, @@ -978,8 +984,8 @@ func TestPoolCommands(t *testing.T) { "pool query -e 12345678-1234-1234-1234-1234567890ab", strings.Join([]string{ printRequest(t, &control.PoolQueryReq{ - ID: "12345678-1234-1234-1234-1234567890ab", - IncludeEnabledRanks: true, + ID: "12345678-1234-1234-1234-1234567890ab", + QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetOptions(daos.PoolQueryOptionEnabledEngines) }), }), }, " "), nil, @@ -989,8 +995,8 @@ func TestPoolCommands(t *testing.T) { "pool query --show-disabled 12345678-1234-1234-1234-1234567890ab", strings.Join([]string{ printRequest(t, &control.PoolQueryReq{ - ID: "12345678-1234-1234-1234-1234567890ab", - IncludeDisabledRanks: true, + ID: "12345678-1234-1234-1234-1234567890ab", + QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetOptions(daos.PoolQueryOptionDisabledEngines) }), }), }, " "), nil, @@ -1000,8 +1006,19 @@ func TestPoolCommands(t *testing.T) { "pool query -b 12345678-1234-1234-1234-1234567890ab", strings.Join([]string{ printRequest(t, &control.PoolQueryReq{ - ID: "12345678-1234-1234-1234-1234567890ab", - IncludeDisabledRanks: true, + ID: "12345678-1234-1234-1234-1234567890ab", + QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { qm.SetOptions(daos.PoolQueryOptionDisabledEngines) }), + }), + }, " "), + nil, + }, + { + "Query pool for health only", + "pool query --health-only 12345678-1234-1234-1234-1234567890ab", + strings.Join([]string{ + printRequest(t, &control.PoolQueryReq{ + ID: "12345678-1234-1234-1234-1234567890ab", + QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) { *qm = daos.HealthOnlyPoolQueryMask }), }), }, " "), nil, @@ -1011,7 +1028,8 @@ func TestPoolCommands(t *testing.T) { "pool query test_label", strings.Join([]string{ printRequest(t, &control.PoolQueryReq{ - ID: "test_label", + ID: "test_label", + QueryMask: daos.DefaultPoolQueryMask, }), }, " "), nil, @@ -1135,7 +1153,7 @@ func TestDmg_PoolListCmd_Errors(t *testing.T) { { Uuid: test.MockUUID(1), SvcReps: []uint32{1, 3, 5, 8}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady.String(), }, }, }, diff --git a/src/control/cmd/dmg/pretty/pool.go b/src/control/cmd/dmg/pretty/pool.go index 35667e8909b..f2bff92a7d8 100644 --- a/src/control/cmd/dmg/pretty/pool.go +++ b/src/control/cmd/dmg/pretty/pool.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2023 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -13,6 +13,7 @@ import ( "github.com/dustin/go-humanize" "github.com/pkg/errors" + pretty "github.com/daos-stack/daos/src/control/cmd/daos/pretty" "github.com/daos-stack/daos/src/control/lib/control" "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ranklist" @@ -23,9 +24,9 @@ const msgNoPools = "No pools in system" func getTierNameText(tierIdx int) string { switch tierIdx { - case int(control.StorageMediaTypeScm): + case int(daos.StorageMediaTypeScm): return fmt.Sprintf("- Storage tier %d (SCM):", tierIdx) - case int(control.StorageMediaTypeNvme): + case int(daos.StorageMediaTypeNvme): return fmt.Sprintf("- Storage tier %d (NVMe):", tierIdx) default: return fmt.Sprintf("- Storage tier %d (unknown):", tierIdx) @@ -35,46 +36,7 @@ func getTierNameText(tierIdx int) string { // PrintPoolQueryResponse generates a human-readable representation of the supplied // PoolQueryResp struct and writes it to the supplied io.Writer. func PrintPoolQueryResponse(pqr *control.PoolQueryResp, out io.Writer, opts ...PrintConfigOption) error { - if pqr == nil { - return errors.Errorf("nil %T", pqr) - } - w := txtfmt.NewErrWriter(out) - - // Maintain output compatibility with the `daos pool query` output. - fmt.Fprintf(w, "Pool %s, ntarget=%d, disabled=%d, leader=%d, version=%d, state=%s\n", - pqr.UUID, pqr.TotalTargets, pqr.DisabledTargets, pqr.Leader, pqr.Version, pqr.State) - - if pqr.PoolLayoutVer != pqr.UpgradeLayoutVer { - fmt.Fprintf(w, "Pool layout out of date (%d < %d) -- see `dmg pool upgrade` for details.\n", - pqr.PoolLayoutVer, pqr.UpgradeLayoutVer) - } - fmt.Fprintln(w, "Pool space info:") - if pqr.EnabledRanks != nil { - fmt.Fprintf(w, "- Enabled ranks: %s\n", pqr.EnabledRanks) - } - if pqr.DisabledRanks != nil { - fmt.Fprintf(w, "- Disabled ranks: %s\n", pqr.DisabledRanks) - } - fmt.Fprintf(w, "- Target(VOS) count:%d\n", pqr.ActiveTargets) - if pqr.TierStats != nil { - for tierIdx, tierStats := range pqr.TierStats { - fmt.Fprintln(w, getTierNameText(tierIdx)) - fmt.Fprintf(w, " Total size: %s\n", humanize.Bytes(tierStats.Total)) - fmt.Fprintf(w, " Free: %s, min:%s, max:%s, mean:%s\n", - humanize.Bytes(tierStats.Free), humanize.Bytes(tierStats.Min), - humanize.Bytes(tierStats.Max), humanize.Bytes(tierStats.Mean)) - } - } - if pqr.Rebuild != nil { - if pqr.Rebuild.Status == 0 { - fmt.Fprintf(w, "Rebuild %s, %d objs, %d recs\n", - pqr.Rebuild.State, pqr.Rebuild.Objects, pqr.Rebuild.Records) - } else { - fmt.Fprintf(w, "Rebuild failed, rc=%d, status=%d\n", pqr.Status, pqr.Rebuild.Status) - } - } - - return w.Err + return pretty.PrintPoolInfo(&pqr.PoolInfo, out) } // PrintPoolQueryTargetResponse generates a human-readable representation of the supplied @@ -83,21 +45,14 @@ func PrintPoolQueryTargetResponse(pqtr *control.PoolQueryTargetResp, out io.Writ if pqtr == nil { return errors.Errorf("nil %T", pqtr) } - w := txtfmt.NewErrWriter(out) - - // Maintain output compatibility with the `daos pool query-targets` output. - for infosIdx := range pqtr.Infos { - fmt.Fprintf(w, "Target: type %s, state %s\n", pqtr.Infos[infosIdx].Type, pqtr.Infos[infosIdx].State) - if pqtr.Infos[infosIdx].Space != nil { - for tierIdx, tierUsage := range pqtr.Infos[infosIdx].Space { - fmt.Fprintln(w, getTierNameText(tierIdx)) - fmt.Fprintf(w, " Total size: %s\n", humanize.Bytes(tierUsage.Total)) - fmt.Fprintf(w, " Free: %s\n", humanize.Bytes(tierUsage.Free)) - } + + for _, info := range pqtr.Infos { + if err := pretty.PrintPoolQueryTargetInfo(info, out); err != nil { + return err } } - return w.Err + return nil } // PrintTierRatio generates a human-readable representation of the supplied @@ -159,20 +114,21 @@ func PrintPoolCreateResponse(pcr *control.PoolCreateResp, out io.Writer, opts .. return err } -func poolListCreateRow(pool *control.Pool, upgrade bool) txtfmt.TableRow { +func poolListCreateRow(pool *daos.PoolInfo, upgrade bool) txtfmt.TableRow { // display size of the largest non-empty tier var size uint64 - for ti := len(pool.Usage) - 1; ti >= 0; ti-- { - if pool.Usage[ti].Size != 0 { - size = pool.Usage[ti].Size + poolUsage := pool.Usage() + for ti := len(poolUsage) - 1; ti >= 0; ti-- { + if poolUsage[ti].Size != 0 { + size = poolUsage[ti].Size break } } // display usage of the most used tier var used int - for ti := 0; ti < len(pool.Usage); ti++ { - t := pool.Usage[ti] + for ti := 0; ti < len(poolUsage); ti++ { + t := poolUsage[ti] u := float64(t.Size-t.Free) / float64(t.Size) if int(u*100) > used { @@ -182,19 +138,19 @@ func poolListCreateRow(pool *control.Pool, upgrade bool) txtfmt.TableRow { // display imbalance of the most imbalanced tier var imbalance uint32 - for ti := 0; ti < len(pool.Usage); ti++ { - if pool.Usage[ti].Imbalance > imbalance { - imbalance = pool.Usage[ti].Imbalance + for ti := 0; ti < len(poolUsage); ti++ { + if poolUsage[ti].Imbalance > imbalance { + imbalance = poolUsage[ti].Imbalance } } row := txtfmt.TableRow{ - "Pool": pool.GetName(), - "Size": fmt.Sprintf("%s", humanize.Bytes(size)), - "State": pool.State, + "Pool": pool.Name(), + "Size": humanize.Bytes(size), + "State": pool.State.String(), "Used": fmt.Sprintf("%d%%", used), "Imbalance": fmt.Sprintf("%d%%", imbalance), - "Disabled": fmt.Sprintf("%d/%d", pool.TargetsDisabled, pool.TargetsTotal), + "Disabled": fmt.Sprintf("%d/%d", pool.DisabledTargets, pool.TotalTargets), } if upgrade { @@ -216,7 +172,7 @@ func printListPoolsResp(out io.Writer, resp *control.ListPoolsResp) error { } upgrade := false for _, pool := range resp.Pools { - if pool.HasErrors() { + if resp.PoolQueryError(pool.UUID) != nil { continue } if pool.PoolLayoutVer != pool.UpgradeLayoutVer { @@ -232,7 +188,7 @@ func printListPoolsResp(out io.Writer, resp *control.ListPoolsResp) error { var table []txtfmt.TableRow for _, pool := range resp.Pools { - if pool.HasErrors() { + if resp.PoolQueryError(pool.UUID) != nil { continue } table = append(table, poolListCreateRow(pool, upgrade)) @@ -243,7 +199,7 @@ func printListPoolsResp(out io.Writer, resp *control.ListPoolsResp) error { return nil } -func addVerboseTierUsage(row txtfmt.TableRow, usage *control.PoolTierUsage) txtfmt.TableRow { +func addVerboseTierUsage(row txtfmt.TableRow, usage *daos.PoolTierUsage) txtfmt.TableRow { row[usage.TierName+" Size"] = humanize.Bytes(usage.Size) row[usage.TierName+" Used"] = humanize.Bytes(usage.Size - usage.Free) row[usage.TierName+" Imbalance"] = fmt.Sprintf("%d%%", usage.Imbalance) @@ -251,7 +207,7 @@ func addVerboseTierUsage(row txtfmt.TableRow, usage *control.PoolTierUsage) txtf return row } -func poolListCreateRowVerbose(pool *control.Pool) txtfmt.TableRow { +func poolListCreateRowVerbose(pool *daos.PoolInfo) txtfmt.TableRow { label := pool.Label if label == "" { label = "-" @@ -270,15 +226,15 @@ func poolListCreateRowVerbose(pool *control.Pool) txtfmt.TableRow { row := txtfmt.TableRow{ "Label": label, - "UUID": pool.UUID, - "State": pool.State, + "UUID": pool.UUID.String(), + "State": pool.State.String(), "SvcReps": svcReps, - "Disabled": fmt.Sprintf("%d/%d", pool.TargetsDisabled, pool.TargetsTotal), + "Disabled": fmt.Sprintf("%d/%d", pool.DisabledTargets, pool.TotalTargets), "UpgradeNeeded?": upgrade, - "Rebuild State": pool.RebuildState, + "Rebuild State": pool.RebuildState(), } - for _, tu := range pool.Usage { + for _, tu := range pool.Usage() { row = addVerboseTierUsage(row, tu) } @@ -292,7 +248,7 @@ func printListPoolsRespVerbose(noQuery bool, out io.Writer, resp *control.ListPo } titles := []string{"Label", "UUID", "State", "SvcReps"} - for _, t := range resp.Pools[0].Usage { + for _, t := range resp.Pools[0].Usage() { titles = append(titles, t.TierName+" Size", t.TierName+" Used", @@ -308,7 +264,7 @@ func printListPoolsRespVerbose(noQuery bool, out io.Writer, resp *control.ListPo var table []txtfmt.TableRow for _, pool := range resp.Pools { - if pool.HasErrors() { + if resp.PoolQueryError(pool.UUID) != nil { continue } table = append(table, poolListCreateRowVerbose(pool)) diff --git a/src/control/cmd/dmg/pretty/pool_test.go b/src/control/cmd/dmg/pretty/pool_test.go index 1a28ec85f74..e61870cef1c 100644 --- a/src/control/cmd/dmg/pretty/pool_test.go +++ b/src/control/cmd/dmg/pretty/pool_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2023 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -14,264 +14,14 @@ import ( "github.com/dustin/go-humanize" "github.com/google/go-cmp/cmp" + "github.com/google/uuid" "github.com/daos-stack/daos/src/control/common/test" "github.com/daos-stack/daos/src/control/lib/control" + "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ranklist" - "github.com/daos-stack/daos/src/control/system" ) -func TestPretty_PrintPoolQueryResp(t *testing.T) { - backtickStr := "`" + "dmg pool upgrade" + "`" - for name, tc := range map[string]struct { - pqr *control.PoolQueryResp - expPrintStr string - }{ - "empty response": { - pqr: &control.PoolQueryResp{}, - expPrintStr: ` -Pool , ntarget=0, disabled=0, leader=0, version=0, state=Unknown -Pool space info: -- Target(VOS) count:0 -`, - }, - "normal response": { - pqr: &control.PoolQueryResp{ - UUID: test.MockUUID(), - State: system.PoolServiceStateDegraded, - PoolInfo: control.PoolInfo{ - TotalTargets: 2, - DisabledTargets: 1, - ActiveTargets: 1, - Leader: 42, - Version: 100, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - Rebuild: &control.PoolRebuildStatus{ - State: control.PoolRebuildStateBusy, - Objects: 42, - Records: 21, - }, - TierStats: []*control.StorageUsageStats{ - { - Total: 2, - Free: 1, - }, - { - Total: 2, - Free: 1, - }, - }, - }, - }, - expPrintStr: fmt.Sprintf(` -Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded -Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. -Pool space info: -- Target(VOS) count:1 -- Storage tier 0 (SCM): - Total size: 2 B - Free: 1 B, min:0 B, max:0 B, mean:0 B -- Storage tier 1 (NVMe): - Total size: 2 B - Free: 1 B, min:0 B, max:0 B, mean:0 B -Rebuild busy, 42 objs, 21 recs -`, test.MockUUID()), - }, - "normal response; enabled ranks": { - pqr: &control.PoolQueryResp{ - UUID: test.MockUUID(), - State: system.PoolServiceStateDegraded, - PoolInfo: control.PoolInfo{ - TotalTargets: 2, - DisabledTargets: 1, - ActiveTargets: 1, - Leader: 42, - Version: 100, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - EnabledRanks: ranklist.MustCreateRankSet("[0,1,2]"), - Rebuild: &control.PoolRebuildStatus{ - State: control.PoolRebuildStateBusy, - Objects: 42, - Records: 21, - }, - TierStats: []*control.StorageUsageStats{ - { - Total: 2, - Free: 1, - }, - { - Total: 2, - Free: 1, - }, - }, - }, - }, - expPrintStr: fmt.Sprintf(` -Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded -Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. -Pool space info: -- Enabled ranks: 0-2 -- Target(VOS) count:1 -- Storage tier 0 (SCM): - Total size: 2 B - Free: 1 B, min:0 B, max:0 B, mean:0 B -- Storage tier 1 (NVMe): - Total size: 2 B - Free: 1 B, min:0 B, max:0 B, mean:0 B -Rebuild busy, 42 objs, 21 recs -`, test.MockUUID()), - }, - "normal response; disabled ranks": { - pqr: &control.PoolQueryResp{ - UUID: test.MockUUID(), - State: system.PoolServiceStateDegraded, - PoolInfo: control.PoolInfo{ - TotalTargets: 2, - DisabledTargets: 1, - ActiveTargets: 1, - Leader: 42, - Version: 100, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - DisabledRanks: ranklist.MustCreateRankSet("[0,1,3]"), - Rebuild: &control.PoolRebuildStatus{ - State: control.PoolRebuildStateBusy, - Objects: 42, - Records: 21, - }, - TierStats: []*control.StorageUsageStats{ - { - Total: 2, - Free: 1, - }, - { - Total: 2, - Free: 1, - }, - }, - }, - }, - expPrintStr: fmt.Sprintf(` -Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded -Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. -Pool space info: -- Disabled ranks: 0-1,3 -- Target(VOS) count:1 -- Storage tier 0 (SCM): - Total size: 2 B - Free: 1 B, min:0 B, max:0 B, mean:0 B -- Storage tier 1 (NVMe): - Total size: 2 B - Free: 1 B, min:0 B, max:0 B, mean:0 B -Rebuild busy, 42 objs, 21 recs -`, test.MockUUID()), - }, - "unknown/invalid rebuild state response": { - pqr: &control.PoolQueryResp{ - UUID: test.MockUUID(), - State: system.PoolServiceStateDegraded, - PoolInfo: control.PoolInfo{ - TotalTargets: 2, - DisabledTargets: 1, - ActiveTargets: 1, - Leader: 42, - Version: 100, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - DisabledRanks: ranklist.MustCreateRankSet("[0,1,3]"), - Rebuild: &control.PoolRebuildStatus{ - State: 42, - Objects: 42, - Records: 21, - }, - TierStats: []*control.StorageUsageStats{ - { - Total: 2, - Free: 1, - }, - { - Total: 2, - Free: 1, - }, - }, - }, - }, - expPrintStr: fmt.Sprintf(` -Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded -Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. -Pool space info: -- Disabled ranks: 0-1,3 -- Target(VOS) count:1 -- Storage tier 0 (SCM): - Total size: 2 B - Free: 1 B, min:0 B, max:0 B, mean:0 B -- Storage tier 1 (NVMe): - Total size: 2 B - Free: 1 B, min:0 B, max:0 B, mean:0 B -Rebuild unknown, 42 objs, 21 recs -`, test.MockUUID()), - }, - "rebuild failed": { - pqr: &control.PoolQueryResp{ - UUID: test.MockUUID(), - State: system.PoolServiceStateDegraded, - PoolInfo: control.PoolInfo{ - TotalTargets: 2, - DisabledTargets: 1, - ActiveTargets: 1, - Leader: 42, - Version: 100, - PoolLayoutVer: 1, - UpgradeLayoutVer: 2, - Rebuild: &control.PoolRebuildStatus{ - Status: 2, - State: control.PoolRebuildStateBusy, - Objects: 42, - Records: 21, - }, - TierStats: []*control.StorageUsageStats{ - { - Total: 2, - Free: 1, - }, - { - Total: 2, - Free: 1, - }, - }, - }, - }, - expPrintStr: fmt.Sprintf(` -Pool %s, ntarget=2, disabled=1, leader=42, version=100, state=Degraded -Pool layout out of date (1 < 2) -- see `+backtickStr+` for details. -Pool space info: -- Target(VOS) count:1 -- Storage tier 0 (SCM): - Total size: 2 B - Free: 1 B, min:0 B, max:0 B, mean:0 B -- Storage tier 1 (NVMe): - Total size: 2 B - Free: 1 B, min:0 B, max:0 B, mean:0 B -Rebuild failed, rc=0, status=2 -`, test.MockUUID()), - }, - } { - t.Run(name, func(t *testing.T) { - var bld strings.Builder - tc.pqr.UpdateState() - if err := PrintPoolQueryResponse(tc.pqr, &bld); err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(strings.TrimLeft(tc.expPrintStr, "\n"), bld.String()); diff != "" { - t.Fatalf("unexpected format string (-want, +got):\n%s\n", diff) - } - }) - } -} - func TestPretty_PrintPoolQueryTargetResp(t *testing.T) { for name, tc := range map[string]struct { pqtr *control.PoolQueryTargetResp @@ -287,194 +37,14 @@ func TestPretty_PrintPoolQueryTargetResp(t *testing.T) { }, expPrintStr: "\n", }, - "valid: single target (unknown, down_out)": { - pqtr: &control.PoolQueryTargetResp{ - Status: 0, - Infos: []*control.PoolQueryTargetInfo{ - { - Type: 0, - State: control.PoolTargetStateDownOut, - Space: []*control.StorageTargetUsage{ - { - Total: 6000000000, - Free: 5000000000, - }, - { - Total: 100000000000, - Free: 90000000000, - }, - }, - }, - }, - }, - expPrintStr: ` -Target: type unknown, state down_out -- Storage tier 0 (SCM): - Total size: 6.0 GB - Free: 5.0 GB -- Storage tier 1 (NVMe): - Total size: 100 GB - Free: 90 GB -`, - }, - "valid: single target (unknown, down)": { - pqtr: &control.PoolQueryTargetResp{ - Status: 0, - Infos: []*control.PoolQueryTargetInfo{ - { - Type: 0, - State: control.PoolTargetStateDown, - Space: []*control.StorageTargetUsage{ - { - Total: 6000000000, - Free: 5000000000, - }, - { - Total: 100000000000, - Free: 90000000000, - }, - }, - }, - }, - }, - expPrintStr: ` -Target: type unknown, state down -- Storage tier 0 (SCM): - Total size: 6.0 GB - Free: 5.0 GB -- Storage tier 1 (NVMe): - Total size: 100 GB - Free: 90 GB -`, - }, - "valid: single target (unknown, up)": { - pqtr: &control.PoolQueryTargetResp{ - Status: 0, - Infos: []*control.PoolQueryTargetInfo{ - { - Type: 0, - State: control.PoolTargetStateUp, - Space: []*control.StorageTargetUsage{ - { - Total: 6000000000, - Free: 5000000000, - }, - { - Total: 100000000000, - Free: 90000000000, - }, - }, - }, - }, - }, - expPrintStr: ` -Target: type unknown, state up -- Storage tier 0 (SCM): - Total size: 6.0 GB - Free: 5.0 GB -- Storage tier 1 (NVMe): - Total size: 100 GB - Free: 90 GB -`, - }, - "valid: single target (unknown, up_in)": { - pqtr: &control.PoolQueryTargetResp{ - Status: 0, - Infos: []*control.PoolQueryTargetInfo{ - { - Type: 0, - State: control.PoolTargetStateUpIn, - Space: []*control.StorageTargetUsage{ - { - Total: 6000000000, - Free: 5000000000, - }, - { - Total: 100000000000, - Free: 90000000000, - }, - }, - }, - }, - }, - expPrintStr: ` -Target: type unknown, state up_in -- Storage tier 0 (SCM): - Total size: 6.0 GB - Free: 5.0 GB -- Storage tier 1 (NVMe): - Total size: 100 GB - Free: 90 GB -`, - }, - "valid: single target (unknown, new)": { - pqtr: &control.PoolQueryTargetResp{ - Status: 0, - Infos: []*control.PoolQueryTargetInfo{ - { - Type: 0, - State: control.PoolTargetStateNew, - Space: []*control.StorageTargetUsage{ - { - Total: 6000000000, - Free: 5000000000, - }, - { - Total: 100000000000, - Free: 90000000000, - }, - }, - }, - }, - }, - expPrintStr: ` -Target: type unknown, state new -- Storage tier 0 (SCM): - Total size: 6.0 GB - Free: 5.0 GB -- Storage tier 1 (NVMe): - Total size: 100 GB - Free: 90 GB -`, - }, - "valid: single target (unknown, drain)": { - pqtr: &control.PoolQueryTargetResp{ - Status: 0, - Infos: []*control.PoolQueryTargetInfo{ - { - Type: 0, - State: control.PoolTargetStateDrain, - Space: []*control.StorageTargetUsage{ - { - Total: 6000000000, - Free: 5000000000, - }, - { - Total: 100000000000, - Free: 90000000000, - }, - }, - }, - }, - }, - expPrintStr: ` -Target: type unknown, state drain -- Storage tier 0 (SCM): - Total size: 6.0 GB - Free: 5.0 GB -- Storage tier 1 (NVMe): - Total size: 100 GB - Free: 90 GB -`, - }, "valid: multiple target (mixed statuses exclude 2 targets in progress)": { pqtr: &control.PoolQueryTargetResp{ Status: 0, - Infos: []*control.PoolQueryTargetInfo{ + Infos: []*daos.PoolQueryTargetInfo{ { Type: 0, - State: control.PoolTargetStateDown, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateDown, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -487,8 +57,8 @@ Target: type unknown, state drain }, { Type: 0, - State: control.PoolTargetStateUpIn, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateUpIn, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -501,8 +71,8 @@ Target: type unknown, state drain }, { Type: 0, - State: control.PoolTargetStateDownOut, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateDownOut, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -515,8 +85,8 @@ Target: type unknown, state drain }, { Type: 0, - State: control.PoolTargetStateUpIn, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateUpIn, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -563,11 +133,11 @@ Target: type unknown, state up_in "invalid target state": { pqtr: &control.PoolQueryTargetResp{ Status: 0, - Infos: []*control.PoolQueryTargetInfo{ + Infos: []*daos.PoolQueryTargetInfo{ { Type: 0, State: 42, - Space: []*control.StorageTargetUsage{ + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -580,8 +150,8 @@ Target: type unknown, state up_in }, { Type: 0, - State: control.PoolTargetStateUpIn, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateUpIn, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -594,8 +164,8 @@ Target: type unknown, state up_in }, { Type: 0, - State: control.PoolTargetStateDownOut, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateDownOut, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -608,8 +178,8 @@ Target: type unknown, state up_in }, { Type: 0, - State: control.PoolTargetStateUpIn, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateUpIn, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -656,11 +226,11 @@ Target: type unknown, state up_in "invalid target type": { pqtr: &control.PoolQueryTargetResp{ Status: 0, - Infos: []*control.PoolQueryTargetInfo{ + Infos: []*daos.PoolQueryTargetInfo{ { Type: 42, - State: control.PoolTargetStateDown, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateDown, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -673,8 +243,8 @@ Target: type unknown, state up_in }, { Type: 0, - State: control.PoolTargetStateUpIn, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateUpIn, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -687,8 +257,8 @@ Target: type unknown, state up_in }, { Type: 0, - State: control.PoolTargetStateDownOut, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateDownOut, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -701,8 +271,8 @@ Target: type unknown, state up_in }, { Type: 0, - State: control.PoolTargetStateUpIn, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateUpIn, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -749,11 +319,11 @@ Target: type unknown, state up_in "three tiers; third tier unknown StorageMediaType": { pqtr: &control.PoolQueryTargetResp{ Status: 0, - Infos: []*control.PoolQueryTargetInfo{ + Infos: []*daos.PoolQueryTargetInfo{ { Type: 0, - State: control.PoolTargetStateDown, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateDown, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -770,8 +340,8 @@ Target: type unknown, state up_in }, { Type: 0, - State: control.PoolTargetStateUpIn, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateUpIn, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -788,8 +358,8 @@ Target: type unknown, state up_in }, { Type: 0, - State: control.PoolTargetStateDownOut, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateDownOut, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -806,8 +376,8 @@ Target: type unknown, state up_in }, { Type: 0, - State: control.PoolTargetStateUpIn, - Space: []*control.StorageTargetUsage{ + State: daos.PoolTargetStateUpIn, + Space: []*daos.StorageUsageStats{ { Total: 6000000000, Free: 5000000000, @@ -919,7 +489,7 @@ Pool created with 5.66%%,94.34%% storage tier ratio Storage tier 0 (SCM) : 2.4 GB (600 MB / rank) Storage tier 1 (NVMe): 40 GB (10 GB / rank) -`, test.MockUUID()), +`, test.MockPoolUUID()), }, "no nvme": { pcr: &control.PoolCreateResp{ @@ -940,7 +510,7 @@ Pool created with 100.00%% storage tier ratio Total Size : 2.4 GB Storage tier 0 (SCM) : 2.4 GB (600 MB / rank) -`, test.MockUUID()), +`, test.MockPoolUUID()), }, } { t.Run(name, func(t *testing.T) { @@ -959,18 +529,20 @@ Pool created with 100.00%% storage tier ratio } func TestPretty_PrintListPoolsResponse(t *testing.T) { - exampleUsage := []*control.PoolTierUsage{ + exampleTierStats := []*daos.StorageUsageStats{ { - TierName: "SCM", - Size: 100 * humanize.GByte, + MediaType: daos.StorageMediaTypeScm, + Total: 100 * humanize.GByte, Free: 20 * humanize.GByte, - Imbalance: 12, + Min: 5 * humanize.GByte, + Max: 6 * humanize.GByte, }, { - TierName: "NVME", - Size: 6 * humanize.TByte, + MediaType: daos.StorageMediaTypeNvme, + Total: 6 * humanize.TByte, Free: 1 * humanize.TByte, - Imbalance: 1, + Min: 20 * humanize.GByte, + Max: 50 * humanize.GByte, }, } @@ -987,11 +559,11 @@ func TestPretty_PrintListPoolsResponse(t *testing.T) { }, "one pool; no usage": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ + Pools: []*daos.PoolInfo{ { - UUID: test.MockUUID(1), + UUID: test.MockPoolUUID(1), ServiceReplicas: []ranklist.Rank{0, 1, 2}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady, }, }, }, @@ -1004,10 +576,10 @@ Pool Size State Used Imbalance Disabled }, "one pool; no uuid": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ + Pools: []*daos.PoolInfo{ { ServiceReplicas: []ranklist.Rank{0, 1, 2}, - Usage: exampleUsage, + TierStats: exampleTierStats, }, }, }, @@ -1015,23 +587,25 @@ Pool Size State Used Imbalance Disabled }, "two pools; diff num tiers": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ + Pools: []*daos.PoolInfo{ { - UUID: test.MockUUID(1), + UUID: test.MockPoolUUID(1), ServiceReplicas: []ranklist.Rank{0, 1, 2}, - Usage: exampleUsage, - TargetsTotal: 16, - TargetsDisabled: 0, + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, PoolLayoutVer: 1, UpgradeLayoutVer: 2, }, { - UUID: test.MockUUID(2), + UUID: test.MockPoolUUID(2), Label: "one", ServiceReplicas: []ranklist.Rank{3, 4, 5}, - Usage: exampleUsage[1:], - TargetsTotal: 64, - TargetsDisabled: 8, + TierStats: exampleTierStats[1:], + TotalTargets: 64, + ActiveTargets: 56, + DisabledTargets: 8, PoolLayoutVer: 2, UpgradeLayoutVer: 2, }, @@ -1041,25 +615,27 @@ Pool Size State Used Imbalance Disabled }, "two pools; only one labeled": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ + Pools: []*daos.PoolInfo{ { - UUID: test.MockUUID(1), + UUID: test.MockPoolUUID(1), ServiceReplicas: []ranklist.Rank{0, 1, 2}, - Usage: exampleUsage, - TargetsTotal: 16, - TargetsDisabled: 0, - State: system.PoolServiceStateReady.String(), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, PoolLayoutVer: 1, UpgradeLayoutVer: 2, }, { Label: "two", - UUID: test.MockUUID(2), + UUID: test.MockPoolUUID(2), ServiceReplicas: []ranklist.Rank{3, 4, 5}, - Usage: exampleUsage, - TargetsTotal: 64, - TargetsDisabled: 8, - State: system.PoolServiceStateReady.String(), + TierStats: exampleTierStats, + TotalTargets: 64, + ActiveTargets: 56, + DisabledTargets: 8, + State: daos.PoolServiceStateReady, PoolLayoutVer: 1, UpgradeLayoutVer: 2, }, @@ -1068,36 +644,38 @@ Pool Size State Used Imbalance Disabled expPrintStr: ` Pool Size State Used Imbalance Disabled UpgradeNeeded? ---- ---- ----- ---- --------- -------- -------------- -00000001 6.0 TB Ready 83% 12% 0/16 1->2 -two 6.0 TB Ready 83% 12% 8/64 1->2 +00000001 6.0 TB Ready 83% 16% 0/16 1->2 +two 6.0 TB Ready 83% 56% 8/64 1->2 `, }, "two pools; one SCM only": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ + Pools: []*daos.PoolInfo{ { Label: "one", - UUID: test.MockUUID(1), + UUID: test.MockPoolUUID(1), ServiceReplicas: []ranklist.Rank{0, 1, 2}, - Usage: exampleUsage, - TargetsTotal: 16, - TargetsDisabled: 0, - State: system.PoolServiceStateReady.String(), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, PoolLayoutVer: 1, UpgradeLayoutVer: 2, }, { Label: "two", - UUID: test.MockUUID(2), + UUID: test.MockPoolUUID(2), ServiceReplicas: []ranklist.Rank{3, 4, 5}, - Usage: []*control.PoolTierUsage{ - exampleUsage[0], - {TierName: "NVME"}, - }, - TargetsTotal: 64, - TargetsDisabled: 8, - State: system.PoolServiceStateReady.String(), + TierStats: []*daos.StorageUsageStats{ + exampleTierStats[0], + {MediaType: daos.StorageMediaTypeNvme}, + }, + TotalTargets: 64, + ActiveTargets: 56, + DisabledTargets: 8, + State: daos.PoolServiceStateReady, PoolLayoutVer: 2, UpgradeLayoutVer: 2, }, @@ -1106,30 +684,35 @@ two 6.0 TB Ready 83% 12% 8/64 1->2 expPrintStr: ` Pool Size State Used Imbalance Disabled UpgradeNeeded? ---- ---- ----- ---- --------- -------- -------------- -one 6.0 TB Ready 83% 12% 0/16 1->2 -two 100 GB Ready 80% 12% 8/64 None +one 6.0 TB Ready 83% 16% 0/16 1->2 +two 100 GB Ready 80% 56% 8/64 None `, }, "two pools; one failed query": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ + Pools: []*daos.PoolInfo{ { Label: "one", - UUID: test.MockUUID(1), + UUID: test.MockPoolUUID(1), ServiceReplicas: []ranklist.Rank{0, 1, 2}, - Usage: exampleUsage, - TargetsTotal: 16, - TargetsDisabled: 0, - State: system.PoolServiceStateReady.String(), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, PoolLayoutVer: 1, UpgradeLayoutVer: 2, }, { Label: "two", - UUID: test.MockUUID(2), + UUID: test.MockPoolUUID(2), ServiceReplicas: []ranklist.Rank{3, 4, 5}, - QueryErrorMsg: "stats unavailable", + }, + }, + QueryErrors: map[uuid.UUID]*control.PoolQueryErr{ + test.MockPoolUUID(2): { + Error: errors.New("stats unavailable"), }, }, }, @@ -1138,46 +721,53 @@ Query on pool "two" unsuccessful, error: "stats unavailable" Pool Size State Used Imbalance Disabled UpgradeNeeded? ---- ---- ----- ---- --------- -------- -------------- -one 6.0 TB Ready 83% 12% 0/16 1->2 +one 6.0 TB Ready 83% 16% 0/16 1->2 `, }, "three pools; one failed query; one query bad status": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ + Pools: []*daos.PoolInfo{ { Label: "one", - UUID: test.MockUUID(1), + UUID: test.MockPoolUUID(1), ServiceReplicas: []ranklist.Rank{0, 1, 2}, - Usage: exampleUsage, - TargetsTotal: 16, - TargetsDisabled: 0, - State: system.PoolServiceStateReady.String(), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, PoolLayoutVer: 1, UpgradeLayoutVer: 1, }, { - UUID: test.MockUUID(2), + UUID: test.MockPoolUUID(2), ServiceReplicas: []ranklist.Rank{3, 4, 5}, - QueryErrorMsg: "stats unavailable", }, { Label: "three", - UUID: test.MockUUID(3), + UUID: test.MockPoolUUID(3), ServiceReplicas: []ranklist.Rank{3, 4, 5}, - QueryStatusMsg: "DER_UNINIT", + }, + }, + QueryErrors: map[uuid.UUID]*control.PoolQueryErr{ + test.MockPoolUUID(2): { + Error: errors.New("stats unavailable"), + }, + test.MockPoolUUID(3): { + Status: daos.NotInit, }, }, }, - expPrintStr: ` + expPrintStr: fmt.Sprintf(` Query on pool "00000002" unsuccessful, error: "stats unavailable" -Query on pool "three" unsuccessful, status: "DER_UNINIT" +Query on pool "three" unsuccessful, status: %q Pool Size State Used Imbalance Disabled ---- ---- ----- ---- --------- -------- -one 6.0 TB Ready 83% 12% 0/16 +one 6.0 TB Ready 83%% 16%% 0/16 -`, +`, daos.NotInit), }, "verbose, empty response": { resp: &control.ListPoolsResp{}, @@ -1186,16 +776,19 @@ one 6.0 TB Ready 83% 12% 0/16 }, "verbose; zero svc replicas": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ - { - UUID: test.MockUUID(1), - Usage: exampleUsage, - TargetsTotal: 16, - TargetsDisabled: 0, - State: system.PoolServiceStateReady.String(), + Pools: []*daos.PoolInfo{ + { + UUID: test.MockPoolUUID(1), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, PoolLayoutVer: 1, UpgradeLayoutVer: 2, - RebuildState: "idle", + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateIdle, + }, }, }, }, @@ -1203,19 +796,20 @@ one 6.0 TB Ready 83% 12% 0/16 expPrintStr: ` Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State ----- ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- -- 00000001-0001-0001-0001-000000000001 Ready N/A 100 GB 80 GB 12% 6.0 TB 5.0 TB 1% 0/16 1->2 idle +- 00000001-0001-0001-0001-000000000001 Ready N/A 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 idle `, }, "verbose; zero svc replicas with no query": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ - { - UUID: test.MockUUID(1), - Usage: exampleUsage, - TargetsTotal: 16, - TargetsDisabled: 0, - State: system.PoolServiceStateReady.String(), + Pools: []*daos.PoolInfo{ + { + UUID: test.MockPoolUUID(1), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, PoolLayoutVer: 1, UpgradeLayoutVer: 2, }, @@ -1226,36 +820,42 @@ Label UUID State SvcReps SCM Size SCM Used SCM I expPrintStr: ` Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? ----- ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- -- 00000001-0001-0001-0001-000000000001 Ready N/A 100 GB 80 GB 12% 6.0 TB 5.0 TB 1% 0/16 1->2 +- 00000001-0001-0001-0001-000000000001 Ready N/A 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 `, }, "verbose; two pools; one destroying": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ + Pools: []*daos.PoolInfo{ { Label: "one", - UUID: test.MockUUID(1), + UUID: test.MockPoolUUID(1), ServiceReplicas: []ranklist.Rank{0, 1, 2}, - Usage: exampleUsage, - TargetsTotal: 16, - TargetsDisabled: 0, - State: system.PoolServiceStateReady.String(), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 16, + DisabledTargets: 0, + State: daos.PoolServiceStateReady, PoolLayoutVer: 1, UpgradeLayoutVer: 2, - RebuildState: "idle", + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateIdle, + }, }, { Label: "two", - UUID: test.MockUUID(2), + UUID: test.MockPoolUUID(2), ServiceReplicas: []ranklist.Rank{3, 4, 5}, - Usage: exampleUsage, - TargetsTotal: 64, - TargetsDisabled: 8, - State: system.PoolServiceStateDestroying.String(), + TierStats: exampleTierStats, + TotalTargets: 64, + ActiveTargets: 56, + DisabledTargets: 8, + State: daos.PoolServiceStateDestroying, PoolLayoutVer: 2, UpgradeLayoutVer: 2, - RebuildState: "done", + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateDone, + }, }, }, }, @@ -1263,25 +863,28 @@ Label UUID State SvcReps SCM Size SCM Used SCM I expPrintStr: ` Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State ----- ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- -one 00000001-0001-0001-0001-000000000001 Ready [0-2] 100 GB 80 GB 12% 6.0 TB 5.0 TB 1% 0/16 1->2 idle -two 00000002-0002-0002-0002-000000000002 Destroying [3-5] 100 GB 80 GB 12% 6.0 TB 5.0 TB 1% 8/64 None done +one 00000001-0001-0001-0001-000000000001 Ready [0-2] 100 GB 80 GB 16% 6.0 TB 5.0 TB 8% 0/16 1->2 idle +two 00000002-0002-0002-0002-000000000002 Destroying [3-5] 100 GB 80 GB 56% 6.0 TB 5.0 TB 27% 8/64 None done `, }, "verbose; one pools; rebuild state busy": { resp: &control.ListPoolsResp{ - Pools: []*control.Pool{ + Pools: []*daos.PoolInfo{ { Label: "one", - UUID: test.MockUUID(1), + UUID: test.MockPoolUUID(1), ServiceReplicas: []ranklist.Rank{0, 1, 2}, - Usage: exampleUsage, - TargetsTotal: 16, - TargetsDisabled: 8, - State: system.PoolServiceStateDegraded.String(), + TierStats: exampleTierStats, + TotalTargets: 16, + ActiveTargets: 8, + DisabledTargets: 8, + State: daos.PoolServiceStateDegraded, PoolLayoutVer: 1, UpgradeLayoutVer: 2, - RebuildState: "busy", + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, + }, }, }, }, @@ -1289,7 +892,7 @@ two 00000002-0002-0002-0002-000000000002 Destroying [3-5] 100 GB 80 GB expPrintStr: ` Label UUID State SvcReps SCM Size SCM Used SCM Imbalance NVME Size NVME Used NVME Imbalance Disabled UpgradeNeeded? Rebuild State ----- ---- ----- ------- -------- -------- ------------- --------- --------- -------------- -------- -------------- ------------- -one 00000001-0001-0001-0001-000000000001 Degraded [0-2] 100 GB 80 GB 12% 6.0 TB 5.0 TB 1% 8/16 1->2 busy +one 00000001-0001-0001-0001-000000000001 Degraded [0-2] 100 GB 80 GB 8% 6.0 TB 5.0 TB 4% 8/16 1->2 busy `, }, diff --git a/src/control/common/proto/mgmt/pool.pb.go b/src/control/common/proto/mgmt/pool.pb.go index 01e7c87067b..5508ec79489 100644 --- a/src/control/common/proto/mgmt/pool.pb.go +++ b/src/control/common/proto/mgmt/pool.pb.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -1615,11 +1615,10 @@ type PoolQueryReq struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Sys string `protobuf:"bytes,1,opt,name=sys,proto3" json:"sys,omitempty"` // DAOS system identifier - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` - SvcRanks []uint32 `protobuf:"varint,3,rep,packed,name=svc_ranks,json=svcRanks,proto3" json:"svc_ranks,omitempty"` // List of pool service ranks - IncludeEnabledRanks bool `protobuf:"varint,4,opt,name=include_enabled_ranks,json=includeEnabledRanks,proto3" json:"include_enabled_ranks,omitempty"` // True if the list of enabled ranks shall be returned - IncludeDisabledRanks bool `protobuf:"varint,5,opt,name=include_disabled_ranks,json=includeDisabledRanks,proto3" json:"include_disabled_ranks,omitempty"` // True if the list of disabled ranks shall be returned + Sys string `protobuf:"bytes,1,opt,name=sys,proto3" json:"sys,omitempty"` // DAOS system identifier + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + SvcRanks []uint32 `protobuf:"varint,3,rep,packed,name=svc_ranks,json=svcRanks,proto3" json:"svc_ranks,omitempty"` // List of pool service ranks + QueryMask uint64 `protobuf:"varint,4,opt,name=query_mask,json=queryMask,proto3" json:"query_mask,omitempty"` // Bitmask of pool query options } func (x *PoolQueryReq) Reset() { @@ -1675,18 +1674,11 @@ func (x *PoolQueryReq) GetSvcRanks() []uint32 { return nil } -func (x *PoolQueryReq) GetIncludeEnabledRanks() bool { +func (x *PoolQueryReq) GetQueryMask() uint64 { if x != nil { - return x.IncludeEnabledRanks + return x.QueryMask } - return false -} - -func (x *PoolQueryReq) GetIncludeDisabledRanks() bool { - if x != nil { - return x.IncludeDisabledRanks - } - return false + return 0 } // StorageUsageStats represents usage statistics for a storage subsystem. @@ -1864,13 +1856,16 @@ type PoolQueryResp struct { Rebuild *PoolRebuildStatus `protobuf:"bytes,7,opt,name=rebuild,proto3" json:"rebuild,omitempty"` // pool rebuild status TierStats []*StorageUsageStats `protobuf:"bytes,8,rep,name=tier_stats,json=tierStats,proto3" json:"tier_stats,omitempty"` // storage tiers usage stats Version uint32 `protobuf:"varint,10,opt,name=version,proto3" json:"version,omitempty"` // latest pool map version - Leader uint32 `protobuf:"varint,11,opt,name=leader,proto3" json:"leader,omitempty"` // current raft leader + Leader uint32 `protobuf:"varint,11,opt,name=leader,proto3" json:"leader,omitempty"` // current raft leader (2.4) EnabledRanks string `protobuf:"bytes,12,opt,name=enabled_ranks,json=enabledRanks,proto3" json:"enabled_ranks,omitempty"` // optional set of ranks enabled DisabledRanks string `protobuf:"bytes,13,opt,name=disabled_ranks,json=disabledRanks,proto3" json:"disabled_ranks,omitempty"` // optional set of ranks disabled TotalEngines uint32 `protobuf:"varint,14,opt,name=total_engines,json=totalEngines,proto3" json:"total_engines,omitempty"` // total engines in pool PoolLayoutVer uint32 `protobuf:"varint,15,opt,name=pool_layout_ver,json=poolLayoutVer,proto3" json:"pool_layout_ver,omitempty"` // current pool global version UpgradeLayoutVer uint32 `protobuf:"varint,16,opt,name=upgrade_layout_ver,json=upgradeLayoutVer,proto3" json:"upgrade_layout_ver,omitempty"` // latest pool global version to upgrade State PoolServiceState `protobuf:"varint,17,opt,name=state,proto3,enum=mgmt.PoolServiceState" json:"state,omitempty"` // pool state + SvcLdr uint32 `protobuf:"varint,18,opt,name=svc_ldr,json=svcLdr,proto3" json:"svc_ldr,omitempty"` // current raft leader (2.6+) + SvcReps []uint32 `protobuf:"varint,19,rep,packed,name=svc_reps,json=svcReps,proto3" json:"svc_reps,omitempty"` // service replica ranks + QueryMask uint64 `protobuf:"varint,20,opt,name=query_mask,json=queryMask,proto3" json:"query_mask,omitempty"` // Bitmask of pool query options used } func (x *PoolQueryResp) Reset() { @@ -2017,6 +2012,27 @@ func (x *PoolQueryResp) GetState() PoolServiceState { return PoolServiceState_Creating } +func (x *PoolQueryResp) GetSvcLdr() uint32 { + if x != nil { + return x.SvcLdr + } + return 0 +} + +func (x *PoolQueryResp) GetSvcReps() []uint32 { + if x != nil { + return x.SvcReps + } + return nil +} + +func (x *PoolQueryResp) GetQueryMask() uint64 { + if x != nil { + return x.QueryMask + } + return 0 +} + type PoolProperty struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3018,175 +3034,176 @@ var file_mgmt_pool_proto_rawDesc = []byte{ 0x74, 0x52, 0x65, 0x73, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x1a, 0x1a, 0x0a, 0x04, 0x43, 0x6f, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, - 0x69, 0x64, 0x22, 0xb7, 0x01, 0x0a, 0x0c, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, - 0x6b, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, - 0x6b, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x64, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x13, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, - 0x64, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x12, 0x34, 0x0a, 0x16, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, - 0x65, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x44, - 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0xac, 0x01, 0x0a, - 0x11, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x55, 0x73, 0x61, 0x67, 0x65, 0x53, 0x74, 0x61, - 0x74, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x66, 0x72, 0x65, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x6d, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, - 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x6d, 0x61, 0x78, - 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x65, 0x61, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, - 0x6d, 0x65, 0x61, 0x6e, 0x12, 0x35, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x22, 0xbb, 0x01, 0x0a, 0x11, - 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x33, 0x0a, 0x05, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, - 0x0a, 0x07, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x07, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f, - 0x72, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, - 0x64, 0x73, 0x22, 0x25, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x49, - 0x44, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, - 0x08, 0x0a, 0x04, 0x42, 0x55, 0x53, 0x59, 0x10, 0x02, 0x22, 0xed, 0x04, 0x0a, 0x0d, 0x50, 0x6f, - 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x23, 0x0a, - 0x0d, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x61, 0x63, 0x74, 0x69, - 0x76, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x64, 0x69, 0x73, - 0x61, 0x62, 0x6c, 0x65, 0x64, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x73, 0x12, 0x31, 0x0a, 0x07, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, - 0x6c, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, - 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x12, 0x36, 0x0a, 0x0a, 0x74, 0x69, 0x65, 0x72, 0x5f, - 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x67, - 0x6d, 0x74, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x55, 0x73, 0x61, 0x67, 0x65, 0x53, - 0x74, 0x61, 0x74, 0x73, 0x52, 0x09, 0x74, 0x69, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, - 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x5f, 0x72, 0x61, 0x6e, - 0x6b, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, - 0x64, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, - 0x65, 0x64, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x12, 0x23, 0x0a, - 0x0d, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x0e, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x6e, 0x67, 0x69, 0x6e, - 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6c, 0x61, 0x79, 0x6f, 0x75, - 0x74, 0x5f, 0x76, 0x65, 0x72, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x70, 0x6f, 0x6f, - 0x6c, 0x4c, 0x61, 0x79, 0x6f, 0x75, 0x74, 0x56, 0x65, 0x72, 0x12, 0x2c, 0x0a, 0x12, 0x75, 0x70, - 0x67, 0x72, 0x61, 0x64, 0x65, 0x5f, 0x6c, 0x61, 0x79, 0x6f, 0x75, 0x74, 0x5f, 0x76, 0x65, 0x72, - 0x18, 0x10, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x4c, - 0x61, 0x79, 0x6f, 0x75, 0x74, 0x56, 0x65, 0x72, 0x12, 0x2c, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, - 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, - 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0a, 0x52, 0x0b, 0x74, 0x6f, - 0x74, 0x61, 0x6c, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x22, 0x63, 0x0a, 0x0c, 0x50, 0x6f, 0x6f, - 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x75, 0x6d, - 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, - 0x72, 0x12, 0x18, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x18, 0x0a, 0x06, 0x6e, - 0x75, 0x6d, 0x76, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x48, 0x00, 0x52, 0x06, 0x6e, - 0x75, 0x6d, 0x76, 0x61, 0x6c, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x83, - 0x01, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, - 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, - 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, - 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, - 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x74, 0x50, - 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, - 0x83, 0x01, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, + 0x69, 0x64, 0x22, 0x6c, 0x0a, 0x0c, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, - 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, - 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x5d, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x47, 0x65, 0x74, - 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, - 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x22, 0x4f, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, - 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, - 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, - 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, - 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x22, 0x81, 0x01, 0x0a, 0x12, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x6e, - 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x72, 0x61, 0x6e, 0x6b, 0x12, 0x18, 0x0a, - 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, - 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, - 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x75, 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, - 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, - 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, - 0x66, 0x72, 0x65, 0x65, 0x12, 0x35, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, - 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x22, 0xda, 0x02, 0x0a, 0x13, - 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x38, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x24, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, - 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x6d, - 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x67, 0x6d, 0x74, - 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, 0x73, - 0x61, 0x67, 0x65, 0x52, 0x05, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x3b, 0x0a, 0x0a, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, - 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x48, 0x44, 0x44, 0x10, 0x01, 0x12, 0x07, - 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x50, 0x4d, 0x10, 0x03, 0x12, - 0x06, 0x0a, 0x02, 0x56, 0x4d, 0x10, 0x04, 0x22, 0x5f, 0x0a, 0x0b, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, - 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x4f, 0x57, - 0x4e, 0x5f, 0x4f, 0x55, 0x54, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x57, 0x4e, 0x10, - 0x02, 0x12, 0x06, 0x0a, 0x02, 0x55, 0x50, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x50, 0x5f, - 0x49, 0x4e, 0x10, 0x04, 0x12, 0x07, 0x0a, 0x03, 0x4e, 0x45, 0x57, 0x10, 0x05, 0x12, 0x09, 0x0a, - 0x05, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x10, 0x06, 0x22, 0x5e, 0x0a, 0x13, 0x50, 0x6f, 0x6f, 0x6c, - 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x12, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, + 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x73, 0x6b, + 0x22, 0xac, 0x01, 0x0a, 0x11, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x55, 0x73, 0x61, 0x67, + 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x12, 0x0a, 0x04, + 0x66, 0x72, 0x65, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x66, 0x72, 0x65, 0x65, + 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x6d, + 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x03, 0x6d, 0x61, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x65, 0x61, 0x6e, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x04, 0x6d, 0x65, 0x61, 0x6e, 0x12, 0x35, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, + 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6d, + 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x22, + 0xbb, 0x01, 0x0a, 0x11, 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x33, 0x0a, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6d, + 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x07, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, + 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x72, + 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x25, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x08, 0x0a, 0x04, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x4e, + 0x45, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x55, 0x53, 0x59, 0x10, 0x02, 0x22, 0xc0, 0x05, + 0x0a, 0x0d, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, + 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, + 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x29, 0x0a, + 0x10, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, + 0x64, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x31, 0x0a, 0x07, 0x72, 0x65, 0x62, 0x75, + 0x69, 0x6c, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, + 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x12, 0x36, 0x0a, 0x0a, 0x74, + 0x69, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x17, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x55, 0x73, + 0x61, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x09, 0x74, 0x69, 0x65, 0x72, 0x53, 0x74, + 0x61, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, + 0x06, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6c, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, + 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x69, + 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x0d, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0d, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x6b, + 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x65, 0x6e, 0x67, 0x69, 0x6e, + 0x65, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x45, + 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6c, + 0x61, 0x79, 0x6f, 0x75, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x0d, 0x70, 0x6f, 0x6f, 0x6c, 0x4c, 0x61, 0x79, 0x6f, 0x75, 0x74, 0x56, 0x65, 0x72, 0x12, 0x2c, + 0x0a, 0x12, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x5f, 0x6c, 0x61, 0x79, 0x6f, 0x75, 0x74, + 0x5f, 0x76, 0x65, 0x72, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x75, 0x70, 0x67, 0x72, + 0x61, 0x64, 0x65, 0x4c, 0x61, 0x79, 0x6f, 0x75, 0x74, 0x56, 0x65, 0x72, 0x12, 0x2c, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x6d, 0x67, + 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x76, + 0x63, 0x5f, 0x6c, 0x64, 0x72, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x73, 0x76, 0x63, + 0x4c, 0x64, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x65, 0x70, 0x73, 0x18, + 0x13, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x73, 0x76, 0x63, 0x52, 0x65, 0x70, 0x73, 0x12, 0x1d, + 0x0a, 0x0a, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x14, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x09, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x61, 0x73, 0x6b, 0x4a, 0x04, 0x08, + 0x09, 0x10, 0x0a, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73, + 0x22, 0x63, 0x0a, 0x0c, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, + 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x76, + 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x72, 0x76, + 0x61, 0x6c, 0x12, 0x18, 0x0a, 0x06, 0x6e, 0x75, 0x6d, 0x76, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x04, 0x48, 0x00, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x76, 0x61, 0x6c, 0x42, 0x07, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x83, 0x01, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, + 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, + 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1b, + 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x29, 0x0a, 0x0f, 0x50, + 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, + 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x83, 0x01, 0x0a, 0x0e, 0x50, 0x6f, 0x6f, 0x6c, 0x47, + 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x79, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x70, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, + 0x72, 0x74, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, + 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x5d, 0x0a, 0x0f, + 0x50, 0x6f, 0x6f, 0x6c, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x32, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, + 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x67, + 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, + 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x4f, 0x0a, 0x0e, 0x50, + 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x12, 0x10, 0x0a, + 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x1b, 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x29, 0x0a, 0x0f, + 0x50, 0x6f, 0x6f, 0x6c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, - 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x2a, 0x25, 0x0a, 0x10, 0x53, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, - 0x53, 0x43, 0x4d, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x56, 0x4d, 0x45, 0x10, 0x01, 0x2a, - 0x56, 0x0a, 0x10, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, - 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x65, 0x61, 0x64, 0x79, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, - 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, - 0x44, 0x65, 0x67, 0x72, 0x61, 0x64, 0x65, 0x64, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, - 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x04, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, - 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, - 0x67, 0x6d, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x81, 0x01, 0x0a, 0x12, 0x50, 0x6f, 0x6f, 0x6c, + 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x12, 0x10, + 0x0a, 0x03, 0x73, 0x79, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x79, 0x73, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, + 0x72, 0x61, 0x6e, 0x6b, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x1b, + 0x0a, 0x09, 0x73, 0x76, 0x63, 0x5f, 0x72, 0x61, 0x6e, 0x6b, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x0d, 0x52, 0x08, 0x73, 0x76, 0x63, 0x52, 0x61, 0x6e, 0x6b, 0x73, 0x22, 0x75, 0x0a, 0x12, 0x53, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, 0x73, 0x61, 0x67, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x66, 0x72, 0x65, 0x65, 0x12, 0x35, 0x0a, 0x0a, 0x6d, + 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x16, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x65, + 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, + 0x70, 0x65, 0x22, 0xda, 0x02, 0x0a, 0x13, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x38, 0x0a, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, + 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, + 0x6e, 0x66, 0x6f, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x18, 0x2e, 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x05, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x22, 0x3b, 0x0a, 0x0a, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, + 0x48, 0x44, 0x44, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x53, 0x44, 0x10, 0x02, 0x12, 0x06, + 0x0a, 0x02, 0x50, 0x4d, 0x10, 0x03, 0x12, 0x06, 0x0a, 0x02, 0x56, 0x4d, 0x10, 0x04, 0x22, 0x5f, + 0x0a, 0x0b, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x11, 0x0a, + 0x0d, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, + 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x4f, 0x57, 0x4e, 0x5f, 0x4f, 0x55, 0x54, 0x10, 0x01, 0x12, 0x08, + 0x0a, 0x04, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x55, 0x50, 0x10, 0x03, + 0x12, 0x09, 0x0a, 0x05, 0x55, 0x50, 0x5f, 0x49, 0x4e, 0x10, 0x04, 0x12, 0x07, 0x0a, 0x03, 0x4e, + 0x45, 0x57, 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x10, 0x06, 0x22, + 0x5e, 0x0a, 0x13, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, + 0x0a, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, + 0x6d, 0x67, 0x6d, 0x74, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x69, 0x6e, 0x66, 0x6f, 0x73, 0x2a, + 0x25, 0x0a, 0x10, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x43, 0x4d, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, + 0x4e, 0x56, 0x4d, 0x45, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x10, 0x50, 0x6f, 0x6f, 0x6c, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x65, 0x61, 0x64, + 0x79, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x69, 0x6e, + 0x67, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x65, 0x67, 0x72, 0x61, 0x64, 0x65, 0x64, 0x10, + 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x04, 0x42, 0x3a, + 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, + 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, + 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x67, 0x6d, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/src/control/common/proto/mgmt/svc.pb.go b/src/control/common/proto/mgmt/svc.pb.go index 2004e9b6f60..59f66c18efe 100644 --- a/src/control/common/proto/mgmt/svc.pb.go +++ b/src/control/common/proto/mgmt/svc.pb.go @@ -6,8 +6,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.5.0 +// protoc-gen-go v1.31.0 +// protoc v3.21.12 // source: mgmt/svc.proto package mgmt diff --git a/src/control/common/proto/mgmt/system.pb.go b/src/control/common/proto/mgmt/system.pb.go index 9a23ad39fde..b4d66dd7364 100644 --- a/src/control/common/proto/mgmt/system.pb.go +++ b/src/control/common/proto/mgmt/system.pb.go @@ -6,8 +6,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.5.0 +// protoc-gen-go v1.31.0 +// protoc v3.21.12 // source: mgmt/system.proto package mgmt diff --git a/src/control/common/proto/srv/srv.pb.go b/src/control/common/proto/srv/srv.pb.go index ae26aad0aad..232a011a42e 100644 --- a/src/control/common/proto/srv/srv.pb.go +++ b/src/control/common/proto/srv/srv.pb.go @@ -8,8 +8,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.5.0 +// protoc-gen-go v1.31.0 +// protoc v3.21.12 // source: srv/srv.proto package srv diff --git a/src/control/drpc/drpc_client.go b/src/control/drpc/drpc_client.go index 86291b31a63..f3d324c330a 100644 --- a/src/control/drpc/drpc_client.go +++ b/src/control/drpc/drpc_client.go @@ -186,7 +186,7 @@ func (c *ClientConnection) GetSocketPath() string { } // NewClientConnection creates a new dRPC client -func NewClientConnection(socket string) *ClientConnection { +func NewClientConnection(socket string) DomainSocketClient { return &ClientConnection{ socketPath: socket, dialer: &clientDialer{}, diff --git a/src/control/drpc/drpc_client_test.go b/src/control/drpc/drpc_client_test.go index 06301c51b44..10bf5a527a6 100644 --- a/src/control/drpc/drpc_client_test.go +++ b/src/control/drpc/drpc_client_test.go @@ -72,12 +72,13 @@ func TestNewClientConnection(t *testing.T) { t.Fatal("Expected a real client") return } - test.AssertEqual(t, client.socketPath, testSockPath, + clientConn := client.(*ClientConnection) + test.AssertEqual(t, clientConn.socketPath, testSockPath, "Should match the path we passed in") test.AssertFalse(t, client.IsConnected(), "Shouldn't be connected yet") // Dialer should be the private implementation type - _ = client.dialer.(*clientDialer) + _ = clientConn.dialer.(*clientDialer) } func TestClient_Connect_Success(t *testing.T) { diff --git a/src/control/fault/code/codes.go b/src/control/fault/code/codes.go index a6389a862bf..3eba1b3e8cd 100644 --- a/src/control/fault/code/codes.go +++ b/src/control/fault/code/codes.go @@ -61,6 +61,7 @@ const ( StorageDeviceAlreadyMounted StorageTargetAlreadyMounted StoragePathAccessDenied + StorageDeviceWithFsNoMountpoint ) // SCM fault codes diff --git a/src/control/lib/control/pool.go b/src/control/lib/control/pool.go index 411b74e60b7..6bf9ad0032c 100644 --- a/src/control/lib/control/pool.go +++ b/src/control/lib/control/pool.go @@ -12,7 +12,6 @@ import ( "fmt" "math" "sort" - "strconv" "strings" "time" @@ -443,54 +442,14 @@ type ( // PoolQueryReq contains the parameters for a pool query request. PoolQueryReq struct { poolRequest - ID string - IncludeEnabledRanks bool - IncludeDisabledRanks bool - } - - // StorageUsageStats represents DAOS storage usage statistics. - StorageUsageStats struct { - Total uint64 `json:"total"` - Free uint64 `json:"free"` - Min uint64 `json:"min"` - Max uint64 `json:"max"` - Mean uint64 `json:"mean"` - MediaType StorageMediaType `json:"media_type"` - } - - // PoolRebuildState indicates the current state of the pool rebuild process. - PoolRebuildState int32 - - // PoolRebuildStatus contains detailed information about the pool rebuild process. - PoolRebuildStatus struct { - Status int32 `json:"status"` - State PoolRebuildState `json:"state"` - Objects uint64 `json:"objects"` - Records uint64 `json:"records"` - } - - // PoolInfo contains information about the pool. - PoolInfo struct { - TotalTargets uint32 `json:"total_targets"` - ActiveTargets uint32 `json:"active_targets"` - TotalEngines uint32 `json:"total_engines"` - DisabledTargets uint32 `json:"disabled_targets"` - Version uint32 `json:"version"` - Leader uint32 `json:"leader"` - Rebuild *PoolRebuildStatus `json:"rebuild"` - TierStats []*StorageUsageStats `json:"tier_stats"` - EnabledRanks *ranklist.RankSet `json:"-"` - DisabledRanks *ranklist.RankSet `json:"-"` - PoolLayoutVer uint32 `json:"pool_layout_ver"` - UpgradeLayoutVer uint32 `json:"upgrade_layout_ver"` + ID string + QueryMask daos.PoolQueryMask } // PoolQueryResp contains the pool query response. PoolQueryResp struct { - Status int32 `json:"status"` - State system.PoolServiceState `json:"state"` - UUID string `json:"uuid"` - PoolInfo + Status int32 `json:"status"` + daos.PoolInfo } // PoolQueryTargetReq contains parameters for a pool query target request @@ -501,51 +460,29 @@ type ( Targets []uint32 } - StorageMediaType int32 - - // StorageTargetUsage represents DAOS target storage usage - StorageTargetUsage struct { - Total uint64 `json:"total"` - Free uint64 `json:"free"` - MediaType StorageMediaType `json:"media_type"` - } - - PoolQueryTargetType int32 - PoolQueryTargetState int32 - - // PoolQueryTargetInfo contains information about a single target - PoolQueryTargetInfo struct { - Type PoolQueryTargetType `json:"target_type"` - State PoolQueryTargetState `json:"target_state"` - Space []*StorageTargetUsage - } - // PoolQueryTargetResp contains a pool query target response PoolQueryTargetResp struct { Status int32 `json:"status"` - Infos []*PoolQueryTargetInfo + Infos []*daos.PoolQueryTargetInfo } ) -const ( - // StorageMediaTypeScm indicates that the media is storage class (persistent) memory - StorageMediaTypeScm StorageMediaType = iota - // StorageMediaTypeNvme indicates that the media is NVMe SSD - StorageMediaTypeNvme -) - func (pqr *PoolQueryResp) MarshalJSON() ([]byte, error) { if pqr == nil { return []byte("null"), nil } - type Alias PoolQueryResp + piJSON, err := json.Marshal(&pqr.PoolInfo) + if err != nil { + return nil, err + } + aux := &struct { EnabledRanks *[]ranklist.Rank `json:"enabled_ranks"` DisabledRanks *[]ranklist.Rank `json:"disabled_ranks"` - *Alias + Status int32 `json:"status"` }{ - Alias: (*Alias)(pqr), + Status: pqr.Status, } if pqr.EnabledRanks != nil { @@ -558,7 +495,15 @@ func (pqr *PoolQueryResp) MarshalJSON() ([]byte, error) { aux.DisabledRanks = &ranks } - return json.Marshal(&aux) + auxJSON, err := json.Marshal(&aux) + if err != nil { + return nil, err + } + + // Kinda gross, but needed to merge the embedded struct's MarshalJSON + // output with this one's. + piJSON[0] = ',' + return append(auxJSON[:len(auxJSON)-1], piJSON...), nil } func unmarshallRankSet(ranks string) (*ranklist.RankSet, error) { @@ -601,60 +546,20 @@ func (pqr *PoolQueryResp) UnmarshalJSON(data []byte) error { return nil } -const ( - // PoolRebuildStateIdle indicates that the rebuild process is idle. - PoolRebuildStateIdle PoolRebuildState = iota - // PoolRebuildStateDone indicates that the rebuild process has completed. - PoolRebuildStateDone - // PoolRebuildStateBusy indicates that the rebuild process is in progress. - PoolRebuildStateBusy -) - -func (prs PoolRebuildState) String() string { - prss, ok := mgmtpb.PoolRebuildStatus_State_name[int32(prs)] - if !ok { - return "unknown" - } - return strings.ToLower(prss) -} - -func (prs PoolRebuildState) MarshalJSON() ([]byte, error) { - stateStr, ok := mgmtpb.PoolRebuildStatus_State_name[int32(prs)] - if !ok { - return nil, errors.Errorf("invalid rebuild state %d", prs) - } - return []byte(`"` + strings.ToLower(stateStr) + `"`), nil -} - -func (prs *PoolRebuildState) UnmarshalJSON(data []byte) error { - stateStr := strings.ToUpper(string(data)) - state, ok := mgmtpb.PoolRebuildStatus_State_value[stateStr] - if !ok { - // Try converting the string to an int32, to handle the - // conversion from protobuf message using convert.Types(). - si, err := strconv.ParseInt(stateStr, 0, 32) - if err != nil { - return errors.Errorf("invalid rebuild state %q", stateStr) - } - - if _, ok = mgmtpb.PoolRebuildStatus_State_name[int32(si)]; !ok { - return errors.Errorf("invalid rebuild state %q", stateStr) - } - state = int32(si) - } - *prs = PoolRebuildState(state) - - return nil -} - // PoolQuery performs a pool query operation for the specified pool ID on a // DAOS Management Server instance. func PoolQuery(ctx context.Context, rpcClient UnaryInvoker, req *PoolQueryReq) (*PoolQueryResp, error) { + return poolQueryInt(ctx, rpcClient, req, nil) +} + +// poolQueryInt is the internal implementation of PoolQuery that +// allows the caller to supply an optional pointer to the response +// that will be filled out with the query details. +func poolQueryInt(ctx context.Context, rpcClient UnaryInvoker, req *PoolQueryReq, resp *PoolQueryResp) (*PoolQueryResp, error) { pbReq := &mgmtpb.PoolQueryReq{ - Sys: req.getSystem(rpcClient), - Id: req.ID, - IncludeEnabledRanks: req.IncludeEnabledRanks, - IncludeDisabledRanks: req.IncludeDisabledRanks, + Sys: req.getSystem(rpcClient), + Id: req.ID, + QueryMask: uint64(req.QueryMask), } req.setRPC(func(ctx context.Context, conn *grpc.ClientConn) (proto.Message, error) { return mgmtpb.NewMgmtSvcClient(conn).PoolQuery(ctx, pbReq) @@ -666,35 +571,36 @@ func PoolQuery(ctx context.Context, rpcClient UnaryInvoker, req *PoolQueryReq) ( return nil, err } - pqr := new(PoolQueryResp) - err = convertMSResponse(ur, pqr) - if err != nil { + if resp == nil { + resp = new(PoolQueryResp) + } + if err := convertMSResponse(ur, resp); err != nil { return nil, err } - err = pqr.UpdateState() + err = resp.UpdateState() if err != nil { return nil, err } - return pqr, err + return resp, err } // UpdateState update the pool state. func (pqr *PoolQueryResp) UpdateState() error { // Update the state as Ready if DAOS return code is 0. if pqr.Status == 0 { - pqr.State = system.PoolServiceStateReady + pqr.State = daos.PoolServiceStateReady } // Pool state is unknown, if TotalTargets is 0. if pqr.TotalTargets == 0 { - pqr.State = system.PoolServiceStateUnknown + pqr.State = daos.PoolServiceStateUnknown } // Update the Pool state as Degraded, if initial state is Ready and any target is disabled - if pqr.State == system.PoolServiceStateReady && pqr.DisabledTargets > 0 { - pqr.State = system.PoolServiceStateDegraded + if pqr.State == daos.PoolServiceStateReady && pqr.DisabledTargets > 0 { + pqr.State = daos.PoolServiceStateDegraded } return nil @@ -742,85 +648,21 @@ func PoolQueryTargets(ctx context.Context, rpcClient UnaryInvoker, req *PoolQuer return pqtr, nil } -func (smt StorageMediaType) MarshalJSON() ([]byte, error) { - typeStr, ok := mgmtpb.StorageMediaType_name[int32(smt)] - if !ok { - return nil, errors.Errorf("invalid storage media type %d", smt) - } - return []byte(`"` + strings.ToLower(typeStr) + `"`), nil -} - -func (smt StorageMediaType) String() string { - smts, ok := mgmtpb.StorageMediaType_name[int32(smt)] - if !ok { - return "unknown" - } - return strings.ToLower(smts) -} - -func (pqtt PoolQueryTargetType) MarshalJSON() ([]byte, error) { - typeStr, ok := mgmtpb.PoolQueryTargetInfo_TargetType_name[int32(pqtt)] - if !ok { - return nil, errors.Errorf("invalid target type %d", pqtt) - } - return []byte(`"` + strings.ToLower(typeStr) + `"`), nil -} - -func (ptt PoolQueryTargetType) String() string { - ptts, ok := mgmtpb.PoolQueryTargetInfo_TargetType_name[int32(ptt)] - if !ok { - return "invalid" - } - return strings.ToLower(ptts) -} - -const ( - PoolTargetStateUnknown PoolQueryTargetState = iota - // PoolTargetStateDownOut indicates the target is not available - PoolTargetStateDownOut - // PoolTargetStateDown indicates the target is not available, may need rebuild - PoolTargetStateDown - // PoolTargetStateUp indicates the target is up - PoolTargetStateUp - // PoolTargetStateUpIn indicates the target is up and running - PoolTargetStateUpIn - // PoolTargetStateNew indicates the target is in an intermediate state (pool map change) - PoolTargetStateNew - // PoolTargetStateDrain indicates the target is being drained - PoolTargetStateDrain -) - -func (pqts PoolQueryTargetState) MarshalJSON() ([]byte, error) { - stateStr, ok := mgmtpb.PoolQueryTargetInfo_TargetState_name[int32(pqts)] - if !ok { - return nil, errors.Errorf("invalid target state %d", pqts) - } - return []byte(`"` + strings.ToLower(stateStr) + `"`), nil -} - -func (pts PoolQueryTargetState) String() string { - ptss, ok := mgmtpb.PoolQueryTargetInfo_TargetState_name[int32(pts)] - if !ok { - return "invalid" - } - return strings.ToLower(ptss) -} - // For using the pretty printer that dmg uses for this target info. -func convertPoolTargetInfo(pbInfo *mgmtpb.PoolQueryTargetInfo) (*PoolQueryTargetInfo, error) { - pqti := new(PoolQueryTargetInfo) - pqti.Type = PoolQueryTargetType(pbInfo.Type) - pqti.State = PoolQueryTargetState(pbInfo.State) - pqti.Space = []*StorageTargetUsage{ +func convertPoolTargetInfo(pbInfo *mgmtpb.PoolQueryTargetInfo) (*daos.PoolQueryTargetInfo, error) { + pqti := new(daos.PoolQueryTargetInfo) + pqti.Type = daos.PoolQueryTargetType(pbInfo.Type) + pqti.State = daos.PoolQueryTargetState(pbInfo.State) + pqti.Space = []*daos.StorageUsageStats{ { - Total: uint64(pbInfo.Space[StorageMediaTypeScm].Total), - Free: uint64(pbInfo.Space[StorageMediaTypeScm].Free), - MediaType: StorageMediaTypeScm, + Total: uint64(pbInfo.Space[daos.StorageMediaTypeScm].Total), + Free: uint64(pbInfo.Space[daos.StorageMediaTypeScm].Free), + MediaType: daos.StorageMediaTypeScm, }, { - Total: uint64(pbInfo.Space[StorageMediaTypeNvme].Total), - Free: uint64(pbInfo.Space[StorageMediaTypeNvme].Free), - MediaType: StorageMediaTypeNvme, + Total: uint64(pbInfo.Space[daos.StorageMediaTypeNvme].Total), + Free: uint64(pbInfo.Space[daos.StorageMediaTypeNvme].Free), + MediaType: daos.StorageMediaTypeNvme, }, } @@ -1092,95 +934,6 @@ func PoolReintegrate(ctx context.Context, rpcClient UnaryInvoker, req *PoolReint return errors.Wrap(ur.getMSError(), "pool reintegrate failed") } -type ( - // PoolTierUsage describes usage of a single pool storage tier. - PoolTierUsage struct { - // TierName identifies a pool's storage tier. - TierName string `json:"tier_name"` - // Size is the total number of bytes in the pool tier. - Size uint64 `json:"size"` - // Free is the number of free bytes in the pool tier. - Free uint64 `json:"free"` - // Imbalance is the percentage imbalance of pool tier usage - // across all the targets. - Imbalance uint32 `json:"imbalance"` - } - - // Pool contains a representation of a DAOS Storage Pool including usage - // statistics. - Pool struct { - // UUID uniquely identifies a pool within the system. - UUID string `json:"uuid"` - // Label is an optional human-friendly identifier for a pool. - Label string `json:"label,omitempty"` - // ServiceLeader is the current pool service leader. - ServiceLeader uint32 `json:"svc_ldr"` - // ServiceReplicas is the list of ranks on which this pool's - // service replicas are running. - ServiceReplicas []ranklist.Rank `json:"svc_reps"` - // State is the current state of the pool. - State string `json:"state"` - - // TargetsTotal is the total number of targets in pool. - TargetsTotal uint32 `json:"targets_total"` - // TargetsDisabled is the number of inactive targets in pool. - TargetsDisabled uint32 `json:"targets_disabled"` - - // UpgradeLayoutVer is latest pool layout version to be upgraded. - UpgradeLayoutVer uint32 `json:"upgrade_layout_ver"` - // PoolLayoutVer is current pool layout version. - PoolLayoutVer uint32 `json:"pool_layout_ver"` - - // QueryErrorMsg reports an RPC error returned from a query. - QueryErrorMsg string `json:"query_error_msg"` - // QueryStatusMsg reports any DAOS error returned from a query - // operation converted into human readable message. - QueryStatusMsg string `json:"query_status_msg"` - - // Usage contains pool usage statistics for each storage tier. - Usage []*PoolTierUsage `json:"usage"` - - // PoolRebuildStatus contains detailed information about the pool rebuild process. - RebuildState string `json:"rebuild_state"` - } -) - -func (p *Pool) setUsage(pqr *PoolQueryResp) { - for idx, tu := range pqr.TierStats { - spread := tu.Max - tu.Min - imbalance := float64(spread) / (float64(tu.Total) / float64(pqr.ActiveTargets)) - - tn := "NVMe" - if idx == 0 { - tn = "SCM" - } - - p.Usage = append(p.Usage, - &PoolTierUsage{ - TierName: tn, - Size: tu.Total, - Free: tu.Free, - Imbalance: uint32(imbalance * 100), - }, - ) - } -} - -// HasErrors indicates whether a pool query operation failed on this pool. -func (p *Pool) HasErrors() bool { - return p.QueryErrorMsg != "" || p.QueryStatusMsg != "" -} - -// GetPool retrieves effective name for pool from either label or UUID. -func (p *Pool) GetName() string { - name := p.Label - if name == "" { - // use short version of uuid if no label - name = strings.Split(p.UUID, "-")[0] - } - return name -} - // ListPoolsReq contains the inputs for the list pools command. type ListPoolsReq struct { unaryRequest @@ -1188,11 +941,32 @@ type ListPoolsReq struct { NoQuery bool } +// PoolQueryErr contains the error if a pool query failed. +type PoolQueryErr struct { + Error error + Status daos.Status +} + // ListPoolsResp contains the status of the request and, if successful, the list // of pools in the system. type ListPoolsResp struct { - Status int32 `json:"status"` - Pools []*Pool `json:"pools"` + Status int32 `json:"status"` + Pools []*daos.PoolInfo `json:"pools"` + QueryErrors map[uuid.UUID]*PoolQueryErr `json:"-"` // NB: Exported because of tests in other packages. +} + +// PoolQueryError returns the error if PoolQuery failed for the given pool. +func (lpr *ListPoolsResp) PoolQueryError(poolUUID uuid.UUID) error { + qe, found := lpr.QueryErrors[poolUUID] + if !found { + return nil + } + + if qe.Status != daos.Success { + return qe.Status + } + + return qe.Error } // Validate returns error if response contents are unexpected, string of @@ -1202,27 +976,31 @@ func (lpr *ListPoolsResp) Validate() (string, error) { out := new(strings.Builder) for i, p := range lpr.Pools { - if p.UUID == "" { + if p.UUID == uuid.Nil { return "", errors.Errorf("pool with index %d has no uuid", i) } - if p.QueryErrorMsg != "" { - fmt.Fprintf(out, "Query on pool %q unsuccessful, error: %q\n", - p.GetName(), p.QueryErrorMsg) - continue // no usage stats expected - } - if p.QueryStatusMsg != "" { - fmt.Fprintf(out, "Query on pool %q unsuccessful, status: %q\n", - p.GetName(), p.QueryStatusMsg) - continue // no usage stats expected + if qe, found := lpr.QueryErrors[p.UUID]; found { + if qe.Error != nil { + fmt.Fprintf(out, "Query on pool %q unsuccessful, error: %q\n", + p.Name(), qe.Error) + continue // no usage stats expected + } + if qe.Status != daos.Success { + fmt.Fprintf(out, "Query on pool %q unsuccessful, status: %q\n", + p.Name(), qe.Status) + continue // no usage stats expected + } } - if len(p.Usage) == 0 { + + poolUsage := p.Usage() + if len(poolUsage) == 0 { continue // no usage stats in response } - if numTiers != 0 && len(p.Usage) != numTiers { + if numTiers != 0 && len(poolUsage) != numTiers { return "", errors.Errorf("pool %s has %d storage tiers, want %d", - p.UUID, len(p.Usage), numTiers) + p.UUID, len(poolUsage), numTiers) } - numTiers = len(p.Usage) + numTiers = len(poolUsage) } return out.String(), nil @@ -1240,6 +1018,12 @@ func (lpr *ListPoolsResp) Errors() error { return nil } +func newListPoolsResp() *ListPoolsResp { + return &ListPoolsResp{ + QueryErrors: make(map[uuid.UUID]*PoolQueryErr), + } +} + // ListPools fetches the list of all pools and their service replicas from the // system. func ListPools(ctx context.Context, rpcClient UnaryInvoker, req *ListPoolsReq) (*ListPoolsResp, error) { @@ -1254,7 +1038,7 @@ func ListPools(ctx context.Context, rpcClient UnaryInvoker, req *ListPoolsReq) ( return nil, err } - resp := new(ListPoolsResp) + resp := newListPoolsResp() if err := convertMSResponse(ur, resp); err != nil { return nil, err } @@ -1264,39 +1048,28 @@ func ListPools(ctx context.Context, rpcClient UnaryInvoker, req *ListPoolsReq) ( } // issue query request and populate usage statistics for each pool - for _, p := range resp.Pools { - if p.State != system.PoolServiceStateReady.String() { + for i, p := range resp.Pools { + if p.State != daos.PoolServiceStateReady { rpcClient.Debugf("Skipping query of pool in state: %s", p.State) continue } rpcClient.Debugf("Fetching details for discovered pool: %v", p) - resp, err := PoolQuery(ctx, rpcClient, &PoolQueryReq{ID: p.UUID}) + pqr := &PoolQueryResp{PoolInfo: *p} + _, err := poolQueryInt(ctx, rpcClient, &PoolQueryReq{ID: p.UUID.String()}, pqr) if err != nil { - p.QueryErrorMsg = err.Error() - if p.QueryErrorMsg == "" { - p.QueryErrorMsg = "unknown error" - } + resp.QueryErrors[p.UUID] = &PoolQueryErr{Error: err} continue } - if resp.Status != 0 { - p.QueryStatusMsg = daos.Status(resp.Status).Error() - if p.QueryStatusMsg == "" { - p.QueryStatusMsg = "unknown error" - } + if pqr.Status != 0 { + resp.QueryErrors[p.UUID] = &PoolQueryErr{Status: daos.Status(pqr.Status)} continue } - if p.UUID != resp.UUID { + if p.UUID != pqr.UUID { return nil, errors.New("pool query response uuid does not match request") - } - p.State = resp.State.String() - p.TargetsTotal = resp.TotalTargets - p.TargetsDisabled = resp.DisabledTargets - p.PoolLayoutVer = resp.PoolLayoutVer - p.UpgradeLayoutVer = resp.UpgradeLayoutVer - p.setUsage(resp) - p.RebuildState = resp.Rebuild.State.String() + } + resp.Pools[i] = &pqr.PoolInfo } sort.Slice(resp.Pools, func(i int, j int) bool { diff --git a/src/control/lib/control/pool_test.go b/src/control/lib/control/pool_test.go index b71d6d35cfd..024cd80b752 100644 --- a/src/control/lib/control/pool_test.go +++ b/src/control/lib/control/pool_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2023 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -17,6 +17,7 @@ import ( "github.com/dustin/go-humanize" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" "github.com/pkg/errors" "github.com/daos-stack/daos/src/control/common/proto/convert" @@ -721,6 +722,8 @@ func TestControl_PoolCreate(t *testing.T) { } func TestControl_UpdateState(t *testing.T) { + poolUUID := test.MockPoolUUID() + for name, tc := range map[string]struct { pqr *PoolQueryResp expState string @@ -728,47 +731,47 @@ func TestControl_UpdateState(t *testing.T) { "Pool state as Ready": { pqr: &PoolQueryResp{ Status: 0, - UUID: "foo", - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + UUID: poolUUID, TotalTargets: 1, DisabledTargets: 0, }, }, - expState: system.PoolServiceStateReady.String(), + expState: daos.PoolServiceStateReady.String(), }, "Pool state as Degraded": { pqr: &PoolQueryResp{ Status: 0, - UUID: "foo", - State: system.PoolServiceStateReady, - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + UUID: poolUUID, TotalTargets: 1, DisabledTargets: 4, + State: daos.PoolServiceStateReady, }, }, - expState: system.PoolServiceStateDegraded.String(), + expState: daos.PoolServiceStateDegraded.String(), }, "Pool state as Unknown": { pqr: &PoolQueryResp{ Status: 0, - UUID: "foo", - State: system.PoolServiceStateReady, - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + UUID: poolUUID, TotalTargets: 0, + State: daos.PoolServiceStateReady, }, }, - expState: system.PoolServiceStateUnknown.String(), + expState: daos.PoolServiceStateUnknown.String(), }, "Pool state as Default": { pqr: &PoolQueryResp{ Status: 0, - UUID: "foo", - State: system.PoolServiceStateUnknown, - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + UUID: poolUUID, TotalTargets: 1, + State: daos.PoolServiceStateUnknown, }, }, - expState: system.PoolServiceStateReady.String(), + expState: daos.PoolServiceStateReady.String(), }, } { t.Run(name, func(t *testing.T) { @@ -781,7 +784,9 @@ func TestControl_UpdateState(t *testing.T) { } } -func TestControl_PoolQueryResp_MarshallJSON(t *testing.T) { +func TestControl_PoolQueryResp_MarshalJSON(t *testing.T) { + poolUUID := test.MockPoolUUID() + for name, tc := range map[string]struct { pqr *PoolQueryResp exp string @@ -792,40 +797,44 @@ func TestControl_PoolQueryResp_MarshallJSON(t *testing.T) { "null rankset": { pqr: &PoolQueryResp{ Status: 0, - UUID: "foo", - State: system.PoolServiceStateReady, - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateReady, + UUID: poolUUID, TotalTargets: 1, ActiveTargets: 2, TotalEngines: 3, DisabledTargets: 4, Version: 5, - Leader: 6, + ServiceLeader: 6, + ServiceReplicas: []ranklist.Rank{0, 1, 2}, PoolLayoutVer: 7, UpgradeLayoutVer: 8, }, }, - exp: `{"enabled_ranks":null,"disabled_ranks":null,"status":0,"state":"Ready","uuid":"foo","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"leader":6,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, + exp: `{"enabled_ranks":null,"disabled_ranks":null,"status":0,"query_mask":"rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, }, "valid rankset": { pqr: &PoolQueryResp{ Status: 0, - UUID: "foo", - State: system.PoolServiceStateReady, - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateReady, + UUID: poolUUID, TotalTargets: 1, ActiveTargets: 2, TotalEngines: 3, DisabledTargets: 4, Version: 5, - Leader: 6, + ServiceLeader: 6, + ServiceReplicas: []ranklist.Rank{0, 1, 2}, EnabledRanks: ranklist.MustCreateRankSet("[0-3,5]"), DisabledRanks: &ranklist.RankSet{}, PoolLayoutVer: 7, UpgradeLayoutVer: 8, }, }, - exp: `{"enabled_ranks":[0,1,2,3,5],"disabled_ranks":[],"status":0,"state":"Ready","uuid":"foo","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"leader":6,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, + exp: `{"enabled_ranks":[0,1,2,3,5],"disabled_ranks":[],"status":0,"query_mask":"rebuild,space","state":"Ready","uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":[0,1,2],"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, }, } { t.Run(name, func(t *testing.T) { @@ -841,41 +850,43 @@ func TestControl_PoolQueryResp_MarshallJSON(t *testing.T) { } } -func TestControl_PoolQueryResp_UnmarshallJSON(t *testing.T) { +func TestControl_PoolQueryResp_UnmarshalJSON(t *testing.T) { + poolUUID := test.MockPoolUUID() + for name, tc := range map[string]struct { data string expResp PoolQueryResp expErr error }{ "null rankset": { - data: `{"enabled_ranks":null,"disabled_ranks":null,"status":0,"uuid":"foo","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"leader":6,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, + data: `{"enabled_ranks":null,"disabled_ranks":null,"status":0,"uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":null,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, expResp: PoolQueryResp{ Status: 0, - UUID: "foo", - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + UUID: poolUUID, TotalTargets: 1, ActiveTargets: 2, TotalEngines: 3, DisabledTargets: 4, Version: 5, - Leader: 6, + ServiceLeader: 6, PoolLayoutVer: 7, UpgradeLayoutVer: 8, }, }, }, "valid rankset": { - data: `{"enabled_ranks":"[0,1-3,5]","disabled_ranks":"[]","status":0,"uuid":"foo","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"leader":6,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, + data: `{"enabled_ranks":"[0,1-3,5]","disabled_ranks":"[]","status":0,"uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"svc_ldr":6,"svc_reps":null,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, expResp: PoolQueryResp{ Status: 0, - UUID: "foo", - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + UUID: poolUUID, TotalTargets: 1, ActiveTargets: 2, TotalEngines: 3, DisabledTargets: 4, Version: 5, - Leader: 6, + ServiceLeader: 6, EnabledRanks: ranklist.MustCreateRankSet("[0-3,5]"), DisabledRanks: &ranklist.RankSet{}, PoolLayoutVer: 7, @@ -888,7 +899,7 @@ func TestControl_PoolQueryResp_UnmarshallJSON(t *testing.T) { expErr: errors.New("invalid character"), }, "invalid rankset": { - data: `{"enabled_ranks":"a cow goes quack","disabled_ranks":null,"status":0,"uuid":"foo","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"leader":6,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, + data: `{"enabled_ranks":"a cow goes quack","disabled_ranks":null,"status":0,"uuid":"` + poolUUID.String() + `","total_targets":1,"active_targets":2,"total_engines":3,"disabled_targets":4,"version":5,"leader":6,"svc_reps":null,"rebuild":null,"tier_stats":null,"pool_layout_ver":7,"upgrade_layout_ver":8}`, expErr: errors.New("unexpected alphabetic character(s)"), }, } { @@ -908,6 +919,8 @@ func TestControl_PoolQueryResp_UnmarshallJSON(t *testing.T) { } func TestControl_PoolQuery(t *testing.T) { + poolUUID := test.MockPoolUUID() + for name, tc := range map[string]struct { mic *MockInvokerConfig req *PoolQueryReq @@ -930,7 +943,7 @@ func TestControl_PoolQuery(t *testing.T) { mic: &MockInvokerConfig{ UnaryResponse: MockMSResponse("host1", nil, &mgmtpb.PoolQueryResp{ - Uuid: test.MockUUID(), + Uuid: poolUUID.String(), TotalTargets: 42, ActiveTargets: 16, DisabledTargets: 17, @@ -949,7 +962,7 @@ func TestControl_PoolQuery(t *testing.T) { Min: 1, Max: 2, Mean: 3, - MediaType: mgmtpb.StorageMediaType(StorageMediaTypeScm), + MediaType: mgmtpb.StorageMediaType(daos.StorageMediaTypeScm), }, { Total: 123456, @@ -957,34 +970,34 @@ func TestControl_PoolQuery(t *testing.T) { Min: 1, Max: 2, Mean: 3, - MediaType: mgmtpb.StorageMediaType(StorageMediaTypeNvme), + MediaType: mgmtpb.StorageMediaType(daos.StorageMediaTypeNvme), }, }, }, ), }, expResp: &PoolQueryResp{ - UUID: test.MockUUID(), - State: system.PoolServiceStateDegraded, - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + UUID: poolUUID, TotalTargets: 42, ActiveTargets: 16, DisabledTargets: 17, PoolLayoutVer: 1, UpgradeLayoutVer: 2, - Rebuild: &PoolRebuildStatus{ - State: PoolRebuildStateBusy, + State: daos.PoolServiceStateDegraded, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, Objects: 1, Records: 2, }, - TierStats: []*StorageUsageStats{ + TierStats: []*daos.StorageUsageStats{ { Total: 123456, Free: 0, Min: 1, Max: 2, Mean: 3, - MediaType: StorageMediaTypeScm, + MediaType: daos.StorageMediaTypeScm, }, { Total: 123456, @@ -992,7 +1005,7 @@ func TestControl_PoolQuery(t *testing.T) { Min: 1, Max: 2, Mean: 3, - MediaType: StorageMediaTypeNvme, + MediaType: daos.StorageMediaTypeNvme, }, }, }, @@ -1002,7 +1015,7 @@ func TestControl_PoolQuery(t *testing.T) { mic: &MockInvokerConfig{ UnaryResponse: MockMSResponse("host1", nil, &mgmtpb.PoolQueryResp{ - Uuid: test.MockUUID(), + Uuid: poolUUID.String(), TotalTargets: 42, ActiveTargets: 16, DisabledTargets: 17, @@ -1021,7 +1034,7 @@ func TestControl_PoolQuery(t *testing.T) { Min: 1, Max: 2, Mean: 3, - MediaType: mgmtpb.StorageMediaType(StorageMediaTypeScm), + MediaType: mgmtpb.StorageMediaType(daos.StorageMediaTypeScm), }, { Total: 123456, @@ -1029,7 +1042,7 @@ func TestControl_PoolQuery(t *testing.T) { Min: 1, Max: 2, Mean: 3, - MediaType: mgmtpb.StorageMediaType(StorageMediaTypeNvme), + MediaType: mgmtpb.StorageMediaType(daos.StorageMediaTypeNvme), }, }, EnabledRanks: "[0,1,2,3,5]", @@ -1037,27 +1050,27 @@ func TestControl_PoolQuery(t *testing.T) { ), }, expResp: &PoolQueryResp{ - UUID: test.MockUUID(), - State: system.PoolServiceStateDegraded, - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + UUID: poolUUID, TotalTargets: 42, ActiveTargets: 16, DisabledTargets: 17, PoolLayoutVer: 1, UpgradeLayoutVer: 2, - Rebuild: &PoolRebuildStatus{ - State: PoolRebuildStateBusy, + State: daos.PoolServiceStateDegraded, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, Objects: 1, Records: 2, }, - TierStats: []*StorageUsageStats{ + TierStats: []*daos.StorageUsageStats{ { Total: 123456, Free: 0, Min: 1, Max: 2, Mean: 3, - MediaType: StorageMediaTypeScm, + MediaType: daos.StorageMediaTypeScm, }, { Total: 123456, @@ -1065,7 +1078,7 @@ func TestControl_PoolQuery(t *testing.T) { Min: 1, Max: 2, Mean: 3, - MediaType: StorageMediaTypeNvme, + MediaType: daos.StorageMediaTypeNvme, }, }, EnabledRanks: ranklist.MustCreateRankSet("[0-3,5]"), @@ -1076,7 +1089,7 @@ func TestControl_PoolQuery(t *testing.T) { mic: &MockInvokerConfig{ UnaryResponse: MockMSResponse("host1", nil, &mgmtpb.PoolQueryResp{ - Uuid: test.MockUUID(), + Uuid: poolUUID.String(), TotalTargets: 42, ActiveTargets: 16, DisabledTargets: 17, @@ -1095,7 +1108,7 @@ func TestControl_PoolQuery(t *testing.T) { Min: 1, Max: 2, Mean: 3, - MediaType: mgmtpb.StorageMediaType(StorageMediaTypeScm), + MediaType: mgmtpb.StorageMediaType(daos.StorageMediaTypeScm), }, { Total: 123456, @@ -1103,7 +1116,7 @@ func TestControl_PoolQuery(t *testing.T) { Min: 1, Max: 2, Mean: 3, - MediaType: mgmtpb.StorageMediaType(StorageMediaTypeNvme), + MediaType: mgmtpb.StorageMediaType(daos.StorageMediaTypeNvme), }, }, DisabledRanks: "[]", @@ -1111,27 +1124,27 @@ func TestControl_PoolQuery(t *testing.T) { ), }, expResp: &PoolQueryResp{ - UUID: test.MockUUID(), - State: system.PoolServiceStateDegraded, - PoolInfo: PoolInfo{ + PoolInfo: daos.PoolInfo{ + UUID: poolUUID, TotalTargets: 42, ActiveTargets: 16, DisabledTargets: 17, PoolLayoutVer: 1, UpgradeLayoutVer: 2, - Rebuild: &PoolRebuildStatus{ - State: PoolRebuildStateBusy, + State: daos.PoolServiceStateDegraded, + Rebuild: &daos.PoolRebuildStatus{ + State: daos.PoolRebuildStateBusy, Objects: 1, Records: 2, }, - TierStats: []*StorageUsageStats{ + TierStats: []*daos.StorageUsageStats{ { Total: 123456, Free: 0, Min: 1, Max: 2, Mean: 3, - MediaType: StorageMediaTypeScm, + MediaType: daos.StorageMediaTypeScm, }, { Total: 123456, @@ -1139,7 +1152,7 @@ func TestControl_PoolQuery(t *testing.T) { Min: 1, Max: 2, Mean: 3, - MediaType: StorageMediaTypeNvme, + MediaType: daos.StorageMediaTypeNvme, }, }, DisabledRanks: &ranklist.RankSet{}, @@ -1661,107 +1674,24 @@ func TestControl_PoolGetPropResp_MarshalJSON(t *testing.T) { } } -func TestControl_Pool_setUsage(t *testing.T) { - for name, tc := range map[string]struct { - status int32 - scmStats *StorageUsageStats - nvmeStats *StorageUsageStats - totalTargets uint32 - activeTargets uint32 - expPool *Pool - expErr error - }{ - "successful query": { - scmStats: &StorageUsageStats{ - Total: humanize.GByte * 30, - Free: humanize.GByte * 15, - Min: humanize.GByte * 1.6, - Max: humanize.GByte * 2, - }, - nvmeStats: &StorageUsageStats{ - Total: humanize.GByte * 500, - Free: humanize.GByte * 250, - Min: humanize.GByte * 29.5, - Max: humanize.GByte * 36, - }, - totalTargets: 8, - activeTargets: 8, - expPool: &Pool{ - Usage: []*PoolTierUsage{ - { - TierName: "SCM", - Size: humanize.GByte * 30, - Free: humanize.GByte * 15, - Imbalance: 10, - }, - { - TierName: "NVMe", - Size: humanize.GByte * 500, - Free: humanize.GByte * 250, - Imbalance: 10, - }, - }, - }, - }, - "disabled targets": { - scmStats: &StorageUsageStats{ - Total: humanize.GByte * 30, - Free: humanize.GByte * 15, - Min: humanize.GByte * 1.6, - Max: humanize.GByte * 2, - }, - nvmeStats: &StorageUsageStats{ - Total: humanize.GByte * 500, - Free: humanize.GByte * 250, - Min: humanize.GByte * 29.5, - Max: humanize.GByte * 36, - }, - totalTargets: 8, - activeTargets: 4, - expPool: &Pool{ - Usage: []*PoolTierUsage{ - { - TierName: "SCM", - Size: humanize.GByte * 30, - Free: humanize.GByte * 15, - Imbalance: 5, - }, - { - TierName: "NVMe", - Size: humanize.GByte * 500, - Free: humanize.GByte * 250, - Imbalance: 5, - }, - }, - }, - }, - } { - t.Run(name, func(t *testing.T) { - resp := &PoolQueryResp{Status: tc.status} - resp.TierStats = append(resp.TierStats, tc.scmStats, tc.nvmeStats) - resp.TotalTargets = tc.totalTargets - resp.ActiveTargets = tc.activeTargets - resp.DisabledTargets = tc.activeTargets - - pool := new(Pool) - pool.setUsage(resp) - - if diff := cmp.Diff(tc.expPool, pool); diff != "" { - t.Fatalf("Unexpected response (-want, +got):\n%s\n", diff) - } - }) - } -} - func TestControl_ListPools(t *testing.T) { queryResp := func(i int32) *mgmtpb.PoolQueryResp { + total := uint32(42) + disabled := uint32(0) + rebuildState := mgmtpb.PoolRebuildStatus_IDLE + if i%2 == 0 { + disabled = total - 16 + rebuildState = mgmtpb.PoolRebuildStatus_BUSY + } + active := uint32(total - disabled) + return &mgmtpb.PoolQueryResp{ Uuid: test.MockUUID(i), - TotalTargets: 42, - ActiveTargets: 16, - DisabledTargets: 17, + TotalTargets: total, + ActiveTargets: active, + DisabledTargets: disabled, Rebuild: &mgmtpb.PoolRebuildStatus{ - State: mgmtpb.PoolRebuildStatus_BUSY, + State: rebuildState, Objects: 1, Records: 2, }, @@ -1782,18 +1712,31 @@ func TestControl_ListPools(t *testing.T) { }, } } - expUsage := []*PoolTierUsage{ + expRebuildStatus := func(i uint32) *daos.PoolRebuildStatus { + rebuildState := daos.PoolRebuildStateIdle + if i%2 == 0 { + rebuildState = daos.PoolRebuildStateBusy + } + return &daos.PoolRebuildStatus{ + State: rebuildState, + Objects: 1, + Records: 2, + } + } + expTierStats := []*daos.StorageUsageStats{ { - TierName: "SCM", - Size: 123456, - Free: 0, - Imbalance: 12, + Total: 123456, + Free: 0, + Min: 1000, + Max: 2000, + Mean: 1500, }, { - TierName: "NVMe", - Size: 1234567, - Free: 600000, - Imbalance: 1, + Total: 1234567, + Free: 600000, + Min: 1000, + Max: 2000, + Mean: 15000, }, } @@ -1823,7 +1766,9 @@ func TestControl_ListPools(t *testing.T) { }, ), }, - expResp: &ListPoolsResp{}, + expResp: &ListPoolsResp{ + QueryErrors: make(map[uuid.UUID]*PoolQueryErr), + }, }, "one pool": { mic: &MockInvokerConfig{ @@ -1833,7 +1778,7 @@ func TestControl_ListPools(t *testing.T) { { Uuid: test.MockUUID(1), SvcReps: []uint32{1, 3, 5, 8}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady.String(), }, }, }), @@ -1841,17 +1786,18 @@ func TestControl_ListPools(t *testing.T) { }, }, expResp: &ListPoolsResp{ - Pools: []*Pool{ + Pools: []*daos.PoolInfo{ { - UUID: test.MockUUID(1), + State: daos.PoolServiceStateReady, + UUID: test.MockPoolUUID(1), + TotalTargets: 42, + ActiveTargets: 42, ServiceReplicas: []ranklist.Rank{1, 3, 5, 8}, - TargetsTotal: 42, - TargetsDisabled: 17, - Usage: expUsage, - State: system.PoolServiceStateDegraded.String(), - RebuildState: "busy", + Rebuild: expRebuildStatus(1), + TierStats: expTierStats, }, }, + QueryErrors: make(map[uuid.UUID]*PoolQueryErr), }, }, "one pool; uuid mismatch in query response": { @@ -1862,7 +1808,7 @@ func TestControl_ListPools(t *testing.T) { { Uuid: test.MockUUID(1), SvcReps: []uint32{1, 3, 5, 8}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady.String(), }, }, }), @@ -1879,12 +1825,12 @@ func TestControl_ListPools(t *testing.T) { { Uuid: test.MockUUID(1), SvcReps: []uint32{1, 3, 5, 8}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady.String(), }, { Uuid: test.MockUUID(2), SvcReps: []uint32{1, 2, 3}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady.String(), }, }, }), @@ -1893,26 +1839,28 @@ func TestControl_ListPools(t *testing.T) { }, }, expResp: &ListPoolsResp{ - Pools: []*Pool{ + Pools: []*daos.PoolInfo{ { - UUID: test.MockUUID(1), + State: daos.PoolServiceStateReady, + UUID: test.MockPoolUUID(1), + TotalTargets: 42, + ActiveTargets: 42, ServiceReplicas: []ranklist.Rank{1, 3, 5, 8}, - TargetsTotal: 42, - TargetsDisabled: 17, - Usage: expUsage, - State: system.PoolServiceStateDegraded.String(), - RebuildState: "busy", + Rebuild: expRebuildStatus(1), + TierStats: expTierStats, }, { - UUID: test.MockUUID(2), + State: daos.PoolServiceStateDegraded, + UUID: test.MockPoolUUID(2), + TotalTargets: 42, + ActiveTargets: 16, + DisabledTargets: 26, ServiceReplicas: []ranklist.Rank{1, 2, 3}, - TargetsTotal: 42, - TargetsDisabled: 17, - Usage: expUsage, - State: system.PoolServiceStateDegraded.String(), - RebuildState: "busy", + Rebuild: expRebuildStatus(2), + TierStats: expTierStats, }, }, + QueryErrors: make(map[uuid.UUID]*PoolQueryErr), }, }, "two pools; one query has error": { @@ -1923,13 +1871,13 @@ func TestControl_ListPools(t *testing.T) { { Uuid: test.MockUUID(1), SvcReps: []uint32{1, 3, 5, 8}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady.String(), RebuildState: "busy", }, { Uuid: test.MockUUID(2), SvcReps: []uint32{1, 2, 3}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady.String(), RebuildState: "busy", }, }, @@ -1939,22 +1887,26 @@ func TestControl_ListPools(t *testing.T) { }, }, expResp: &ListPoolsResp{ - Pools: []*Pool{ + Pools: []*daos.PoolInfo{ { - UUID: test.MockUUID(1), + State: daos.PoolServiceStateReady, + UUID: test.MockPoolUUID(1), ServiceReplicas: []ranklist.Rank{1, 3, 5, 8}, - QueryErrorMsg: "remote failed", - State: system.PoolServiceStateReady.String(), - RebuildState: "busy", }, { - UUID: test.MockUUID(2), + State: daos.PoolServiceStateDegraded, + UUID: test.MockPoolUUID(2), + TotalTargets: 42, + ActiveTargets: 16, + DisabledTargets: 26, ServiceReplicas: []ranklist.Rank{1, 2, 3}, - TargetsTotal: 42, - TargetsDisabled: 17, - Usage: expUsage, - State: system.PoolServiceStateDegraded.String(), - RebuildState: "busy", + Rebuild: expRebuildStatus(2), + TierStats: expTierStats, + }, + }, + QueryErrors: map[uuid.UUID]*PoolQueryErr{ + test.MockPoolUUID(1): { + Error: errors.New("remote failed"), }, }, }, @@ -1967,13 +1919,13 @@ func TestControl_ListPools(t *testing.T) { { Uuid: test.MockUUID(1), SvcReps: []uint32{1, 3, 5, 8}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady.String(), RebuildState: "busy", }, { Uuid: test.MockUUID(2), SvcReps: []uint32{1, 2, 3}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady.String(), RebuildState: "busy", }, }, @@ -1985,22 +1937,26 @@ func TestControl_ListPools(t *testing.T) { }, }, expResp: &ListPoolsResp{ - Pools: []*Pool{ + Pools: []*daos.PoolInfo{ { - UUID: test.MockUUID(1), + State: daos.PoolServiceStateReady, + UUID: test.MockPoolUUID(1), ServiceReplicas: []ranklist.Rank{1, 3, 5, 8}, - QueryStatusMsg: "DER_UNINIT(-1015): Device or resource not initialized", - State: system.PoolServiceStateReady.String(), - RebuildState: "busy", }, { - UUID: test.MockUUID(2), + State: daos.PoolServiceStateDegraded, + UUID: test.MockPoolUUID(2), + TotalTargets: 42, + ActiveTargets: 16, + DisabledTargets: 26, ServiceReplicas: []ranklist.Rank{1, 2, 3}, - TargetsTotal: 42, - TargetsDisabled: 17, - Usage: expUsage, - State: system.PoolServiceStateDegraded.String(), - RebuildState: "busy", + Rebuild: expRebuildStatus(2), + TierStats: expTierStats, + }, + }, + QueryErrors: map[uuid.UUID]*PoolQueryErr{ + test.MockPoolUUID(1): { + Status: daos.NotInit, }, }, }, @@ -2013,12 +1969,12 @@ func TestControl_ListPools(t *testing.T) { { Uuid: test.MockUUID(1), SvcReps: []uint32{1, 3, 5, 8}, - State: system.PoolServiceStateReady.String(), + State: daos.PoolServiceStateReady.String(), }, { Uuid: test.MockUUID(2), SvcReps: []uint32{1, 2, 3}, - State: system.PoolServiceStateDestroying.String(), + State: daos.PoolServiceStateDestroying.String(), RebuildState: "busy", }, }, @@ -2028,23 +1984,23 @@ func TestControl_ListPools(t *testing.T) { }, }, expResp: &ListPoolsResp{ - Pools: []*Pool{ + Pools: []*daos.PoolInfo{ { - UUID: test.MockUUID(1), + State: daos.PoolServiceStateReady, + UUID: test.MockPoolUUID(1), + TotalTargets: 42, + ActiveTargets: 42, ServiceReplicas: []ranklist.Rank{1, 3, 5, 8}, - TargetsTotal: 42, - TargetsDisabled: 17, - Usage: expUsage, - State: system.PoolServiceStateDegraded.String(), - RebuildState: "busy", + Rebuild: expRebuildStatus(1), + TierStats: expTierStats, }, { - UUID: test.MockUUID(2), + State: daos.PoolServiceStateDestroying, + UUID: test.MockPoolUUID(2), ServiceReplicas: []ranklist.Rank{1, 2, 3}, - State: system.PoolServiceStateDestroying.String(), - RebuildState: "busy", }, }, + QueryErrors: make(map[uuid.UUID]*PoolQueryErr), }, }, } { @@ -2070,7 +2026,10 @@ func TestControl_ListPools(t *testing.T) { return } - if diff := cmp.Diff(tc.expResp, gotResp); diff != "" { + cmpOpts := []cmp.Option{ + cmp.Comparer(test.CmpErrBool), + } + if diff := cmp.Diff(tc.expResp, gotResp, cmpOpts...); diff != "" { t.Fatalf("unexpected response (-want, +got):\n%s\n", diff) } }) diff --git a/src/control/lib/daos/pool.go b/src/control/lib/daos/pool.go new file mode 100644 index 00000000000..89215a64b5a --- /dev/null +++ b/src/control/lib/daos/pool.go @@ -0,0 +1,425 @@ +// +// (C) Copyright 2020-2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package daos + +import ( + "encoding/json" + "sort" + "strconv" + "strings" + + "github.com/google/uuid" + "github.com/pkg/errors" + + mgmtpb "github.com/daos-stack/daos/src/control/common/proto/mgmt" + "github.com/daos-stack/daos/src/control/lib/ranklist" +) + +/* +#include <stdint.h> + +#include <daos_pool.h> +*/ +import "C" + +type ( + // PoolTierUsage describes usage of a single pool storage tier in + // a simpler format. + PoolTierUsage struct { + // TierName identifies a pool's storage tier. + TierName string `json:"tier_name"` + // Size is the total number of bytes in the pool tier. + Size uint64 `json:"size"` + // Free is the number of free bytes in the pool tier. + Free uint64 `json:"free"` + // Imbalance is the percentage imbalance of pool tier usage + // across all the targets. + Imbalance uint32 `json:"imbalance"` + } + + // StorageUsageStats represents raw DAOS storage usage statistics. + StorageUsageStats struct { + Total uint64 `json:"total"` + Free uint64 `json:"free"` + Min uint64 `json:"min"` + Max uint64 `json:"max"` + Mean uint64 `json:"mean"` + MediaType StorageMediaType `json:"media_type"` + } + + // PoolRebuildStatus contains detailed information about the pool rebuild process. + PoolRebuildStatus struct { + Status int32 `json:"status"` + State PoolRebuildState `json:"state"` + Objects uint64 `json:"objects"` + Records uint64 `json:"records"` + } + + // PoolInfo contains information about the pool. + PoolInfo struct { + QueryMask PoolQueryMask `json:"query_mask"` + State PoolServiceState `json:"state"` + UUID uuid.UUID `json:"uuid"` + Label string `json:"label,omitempty"` + TotalTargets uint32 `json:"total_targets"` + ActiveTargets uint32 `json:"active_targets"` + TotalEngines uint32 `json:"total_engines"` + DisabledTargets uint32 `json:"disabled_targets"` + Version uint32 `json:"version"` + ServiceLeader uint32 `json:"svc_ldr"` + ServiceReplicas []ranklist.Rank `json:"svc_reps,omitempty"` + Rebuild *PoolRebuildStatus `json:"rebuild"` + TierStats []*StorageUsageStats `json:"tier_stats"` + EnabledRanks *ranklist.RankSet `json:"-"` + DisabledRanks *ranklist.RankSet `json:"-"` + PoolLayoutVer uint32 `json:"pool_layout_ver"` + UpgradeLayoutVer uint32 `json:"upgrade_layout_ver"` + } + + PoolQueryTargetType int32 + PoolQueryTargetState int32 + + // PoolQueryTargetInfo contains information about a single target + PoolQueryTargetInfo struct { + Type PoolQueryTargetType `json:"target_type"` + State PoolQueryTargetState `json:"target_state"` + Space []*StorageUsageStats `json:"space"` + } + + // StorageTargetUsage represents DAOS target storage usage + StorageTargetUsage struct { + Total uint64 `json:"total"` + Free uint64 `json:"free"` + MediaType StorageMediaType `json:"media_type"` + } + + // PoolQueryMask implements a bitmask for pool query options. + PoolQueryMask C.uint64_t +) + +const ( + // DefaultPoolQueryMask defines the default pool query mask. + DefaultPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_ENGINES_DISABLED)) + // HealthOnlyPoolQueryMask defines the mask for health-only queries. + HealthOnlyPoolQueryMask = PoolQueryMask(^uint64(0) &^ (C.DPI_ENGINES_ENABLED | C.DPI_SPACE)) + + // PoolQueryOptionSpace retrieves storage space usage as part of the pool query. + PoolQueryOptionSpace = "space" + // PoolQueryOptionRebuild retrieves pool rebuild status as part of the pool query. + PoolQueryOptionRebuild = "rebuild" + // PoolQueryOptionEnabledEngines retrieves enabled engines as part of the pool query. + PoolQueryOptionEnabledEngines = "enabled_engines" + // PoolQueryOptionDisabledEngines retrieves disabled engines as part of the pool query. + PoolQueryOptionDisabledEngines = "disabled_engines" +) + +var poolQueryOptMap = map[C.int]string{ + C.DPI_SPACE: PoolQueryOptionSpace, + C.DPI_REBUILD_STATUS: PoolQueryOptionRebuild, + C.DPI_ENGINES_ENABLED: PoolQueryOptionEnabledEngines, + C.DPI_ENGINES_DISABLED: PoolQueryOptionDisabledEngines, +} + +func resolvePoolQueryOpt(name string) (C.int, error) { + for opt, optName := range poolQueryOptMap { + if name == optName { + return opt, nil + } + } + return 0, errors.Errorf("invalid pool query option: %q", name) +} + +// SetOptions sets the pool query mask to include the specified options. +func (pqm *PoolQueryMask) SetOptions(optNames ...string) error { + for _, optName := range optNames { + if opt, err := resolvePoolQueryOpt(optName); err != nil { + return err + } else { + *pqm |= PoolQueryMask(opt) + } + } + return nil +} + +// ClearOptions clears the pool query mask of the specified options. +func (pqm *PoolQueryMask) ClearOptions(optNames ...string) error { + for _, optName := range optNames { + if opt, err := resolvePoolQueryOpt(optName); err != nil { + return err + } else { + *pqm &^= PoolQueryMask(opt) + } + } + return nil +} + +// SetAll sets the pool query mask to include all pool query options. +func (pqm *PoolQueryMask) SetAll() { + *pqm = PoolQueryMask(^uint64(0)) // DPI_ALL is -1 +} + +// ClearAll clears the pool query mask of all pool query options. +func (pqm *PoolQueryMask) ClearAll() { + *pqm = 0 +} + +// HasOption returns true if the pool query mask includes the specified option. +func (pqm PoolQueryMask) HasOption(optName string) bool { + return strings.Contains(pqm.String(), optName) +} + +func (pqm PoolQueryMask) String() string { + var flags []string + for i := 0; i < (C.sizeof_int*8)-1; i++ { + opt := C.int(1 << i) + if flag, ok := poolQueryOptMap[opt]; ok { + if pqm&PoolQueryMask(opt) != 0 { + flags = append(flags, flag) + } + } + } + sort.Strings(flags) + return strings.Join(flags, ",") +} + +func (pqm PoolQueryMask) MarshalJSON() ([]byte, error) { + return json.Marshal(pqm.String()) +} + +func (pqm *PoolQueryMask) UnmarshalJSON(data []byte) error { + if len(data) == 0 { + *pqm = 0 + return nil + } + + val, err := strconv.ParseUint(string(data), 10, 64) + if err == nil { + *pqm = PoolQueryMask(val) + return nil + } + + var newVal PoolQueryMask + for _, opt := range strings.Split(string(data), ",") { + for k, v := range poolQueryOptMap { + if v == opt { + newVal |= PoolQueryMask(k) + goto next + } + } + return errors.Errorf("invalid pool query option: %s", opt) + next: + } + *pqm = newVal + + return nil +} + +func (srs *StorageUsageStats) calcImbalance(targCount uint32) uint32 { + spread := srs.Max - srs.Min + return uint32((float64(spread) / (float64(srs.Total) / float64(targCount))) * 100) +} + +func (pi *PoolInfo) MarshalJSON() ([]byte, error) { + type Alias PoolInfo + + return json.Marshal(&struct { + *Alias + Usage []*PoolTierUsage `json:"usage,omitempty"` + }{ + Alias: (*Alias)(pi), + Usage: pi.Usage(), + }) +} + +// Usage returns a slice of PoolTierUsage objects describing the pool's storage +// usage in a simpler format. +func (pi *PoolInfo) Usage() []*PoolTierUsage { + var tiers []*PoolTierUsage + for _, tier := range pi.TierStats { + tiers = append(tiers, &PoolTierUsage{ + TierName: strings.ToUpper(tier.MediaType.String()), + Size: tier.Total, + Free: tier.Free, + Imbalance: tier.calcImbalance(pi.ActiveTargets), + }) + } + return tiers +} + +// RebuildState returns a string representation of the pool rebuild state. +func (pi *PoolInfo) RebuildState() string { + if pi.Rebuild == nil { + return "Unknown" + } + return pi.Rebuild.State.String() +} + +// Name retrieves effective name for pool from either label or UUID. +func (pi *PoolInfo) Name() string { + name := pi.Label + if name == "" { + // use short version of uuid if no label + name = strings.Split(pi.UUID.String(), "-")[0] + } + return name +} + +// PoolServiceState is used to represent the state of the pool service +type PoolServiceState uint + +const ( + // PoolServiceStateCreating indicates that the pool service is being created + PoolServiceStateCreating = PoolServiceState(mgmtpb.PoolServiceState_Creating) + // PoolServiceStateReady indicates that the pool service is ready to be used + PoolServiceStateReady = PoolServiceState(mgmtpb.PoolServiceState_Ready) + // PoolServiceStateDestroying indicates that the pool service is being destroyed + PoolServiceStateDestroying = PoolServiceState(mgmtpb.PoolServiceState_Destroying) + // PoolServiceStateDegraded indicates that the pool service is in a degraded state + PoolServiceStateDegraded = PoolServiceState(mgmtpb.PoolServiceState_Degraded) + // PoolServiceStateUnknown indicates that the pool service state is unknown + PoolServiceStateUnknown = PoolServiceState(mgmtpb.PoolServiceState_Unknown) +) + +func (pss PoolServiceState) String() string { + psss, ok := mgmtpb.PoolServiceState_name[int32(pss)] + if !ok { + return "invalid" + } + return psss +} + +func (pss PoolServiceState) MarshalJSON() ([]byte, error) { + return []byte(`"` + pss.String() + `"`), nil +} + +func unmarshalStrVal(inStr string, name2Vals map[string]int32, val2Names map[int32]string) (int32, error) { + intVal, found := name2Vals[inStr] + if found { + return intVal, nil + } + // Try converting the string to an int32, to handle the + // conversion from protobuf message using convert.Types(). + val, err := strconv.ParseInt(inStr, 0, 32) + if err != nil { + return 0, errors.Errorf("non-numeric string value %q", inStr) + } + + if _, ok := val2Names[int32(val)]; !ok { + return 0, errors.Errorf("unable to resolve string to value %q", inStr) + } + return int32(val), nil +} + +func (pss *PoolServiceState) UnmarshalJSON(data []byte) error { + stateStr := strings.Trim(string(data), "\"") + + state, err := unmarshalStrVal(stateStr, mgmtpb.PoolServiceState_value, mgmtpb.PoolServiceState_name) + if err != nil { + return errors.Wrap(err, "failed to unmarshal PoolServiceState") + } + *pss = PoolServiceState(state) + + return nil +} + +// StorageMediaType indicates the type of storage. +type StorageMediaType int32 + +const ( + // StorageMediaTypeScm indicates that the media is storage class (persistent) memory + StorageMediaTypeScm = StorageMediaType(mgmtpb.StorageMediaType_SCM) + // StorageMediaTypeNvme indicates that the media is NVMe SSD + StorageMediaTypeNvme = StorageMediaType(mgmtpb.StorageMediaType_NVME) +) + +func (smt StorageMediaType) String() string { + smts, ok := mgmtpb.StorageMediaType_name[int32(smt)] + if !ok { + return "unknown" + } + return strings.ToLower(smts) +} + +func (smt StorageMediaType) MarshalJSON() ([]byte, error) { + return []byte(`"` + smt.String() + `"`), nil +} + +// PoolRebuildState indicates the current state of the pool rebuild process. +type PoolRebuildState int32 + +const ( + // PoolRebuildStateIdle indicates that the rebuild process is idle. + PoolRebuildStateIdle = PoolRebuildState(mgmtpb.PoolRebuildStatus_IDLE) + // PoolRebuildStateDone indicates that the rebuild process has completed. + PoolRebuildStateDone = PoolRebuildState(mgmtpb.PoolRebuildStatus_DONE) + // PoolRebuildStateBusy indicates that the rebuild process is in progress. + PoolRebuildStateBusy = PoolRebuildState(mgmtpb.PoolRebuildStatus_BUSY) +) + +func (prs PoolRebuildState) String() string { + prss, ok := mgmtpb.PoolRebuildStatus_State_name[int32(prs)] + if !ok { + return "unknown" + } + return strings.ToLower(prss) +} + +func (prs PoolRebuildState) MarshalJSON() ([]byte, error) { + return []byte(`"` + prs.String() + `"`), nil +} + +func (prs *PoolRebuildState) UnmarshalJSON(data []byte) error { + stateStr := strings.ToUpper(string(data)) + + state, err := unmarshalStrVal(stateStr, mgmtpb.PoolRebuildStatus_State_value, mgmtpb.PoolRebuildStatus_State_name) + if err != nil { + return errors.Wrap(err, "failed to unmarshal PoolRebuildState") + } + *prs = PoolRebuildState(state) + + return nil +} + +func (ptt PoolQueryTargetType) String() string { + ptts, ok := mgmtpb.PoolQueryTargetInfo_TargetType_name[int32(ptt)] + if !ok { + return "invalid" + } + return strings.ToLower(ptts) +} + +func (pqtt PoolQueryTargetType) MarshalJSON() ([]byte, error) { + return []byte(`"` + pqtt.String() + `"`), nil +} + +const ( + PoolTargetStateUnknown = PoolQueryTargetState(mgmtpb.PoolQueryTargetInfo_STATE_UNKNOWN) + // PoolTargetStateDownOut indicates the target is not available + PoolTargetStateDownOut = PoolQueryTargetState(mgmtpb.PoolQueryTargetInfo_DOWN_OUT) + // PoolTargetStateDown indicates the target is not available, may need rebuild + PoolTargetStateDown = PoolQueryTargetState(mgmtpb.PoolQueryTargetInfo_DOWN) + // PoolTargetStateUp indicates the target is up + PoolTargetStateUp = PoolQueryTargetState(mgmtpb.PoolQueryTargetInfo_UP) + // PoolTargetStateUpIn indicates the target is up and running + PoolTargetStateUpIn = PoolQueryTargetState(mgmtpb.PoolQueryTargetInfo_UP_IN) + // PoolTargetStateNew indicates the target is in an intermediate state (pool map change) + PoolTargetStateNew = PoolQueryTargetState(mgmtpb.PoolQueryTargetInfo_NEW) + // PoolTargetStateDrain indicates the target is being drained + PoolTargetStateDrain = PoolQueryTargetState(mgmtpb.PoolQueryTargetInfo_DRAIN) +) + +func (pqts PoolQueryTargetState) String() string { + pqtss, ok := mgmtpb.PoolQueryTargetInfo_TargetState_name[int32(pqts)] + if !ok { + return "invalid" + } + return strings.ToLower(pqtss) +} + +func (pqts PoolQueryTargetState) MarshalJSON() ([]byte, error) { + return []byte(`"` + pqts.String() + `"`), nil +} diff --git a/src/control/lib/daos/pool_test.go b/src/control/lib/daos/pool_test.go new file mode 100644 index 00000000000..facbc7682c8 --- /dev/null +++ b/src/control/lib/daos/pool_test.go @@ -0,0 +1,285 @@ +// +// (C) Copyright 2020-2024 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package daos + +import ( + "strings" + "testing" + + "github.com/dustin/go-humanize" + "github.com/google/go-cmp/cmp" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/common/test" +) + +func TestDaos_PoolInfo_Usage(t *testing.T) { + for name, tc := range map[string]struct { + status int32 + scmStats *StorageUsageStats + nvmeStats *StorageUsageStats + totalTargets uint32 + activeTargets uint32 + expUsage []*PoolTierUsage + expErr error + }{ + "successful query": { + scmStats: &StorageUsageStats{ + MediaType: MediaTypeScm, + Total: humanize.GByte * 30, + Free: humanize.GByte * 15, + Min: humanize.GByte * 1.6, + Max: humanize.GByte * 2, + }, + nvmeStats: &StorageUsageStats{ + MediaType: StorageMediaTypeNvme, + Total: humanize.GByte * 500, + Free: humanize.GByte * 250, + Min: humanize.GByte * 29.5, + Max: humanize.GByte * 36, + }, + totalTargets: 8, + activeTargets: 8, + expUsage: []*PoolTierUsage{ + { + TierName: "SCM", + Size: humanize.GByte * 30, + Free: humanize.GByte * 15, + Imbalance: 10, + }, + { + TierName: "NVME", + Size: humanize.GByte * 500, + Free: humanize.GByte * 250, + Imbalance: 10, + }, + }, + }, + "disabled targets": { + scmStats: &StorageUsageStats{ + MediaType: MediaTypeScm, + Total: humanize.GByte * 30, + Free: humanize.GByte * 15, + Min: humanize.GByte * 1.6, + Max: humanize.GByte * 2, + }, + nvmeStats: &StorageUsageStats{ + MediaType: MediaTypeNvme, + Total: humanize.GByte * 500, + Free: humanize.GByte * 250, + Min: humanize.GByte * 29.5, + Max: humanize.GByte * 36, + }, + totalTargets: 8, + activeTargets: 4, + expUsage: []*PoolTierUsage{ + { + TierName: "SCM", + Size: humanize.GByte * 30, + Free: humanize.GByte * 15, + Imbalance: 5, + }, + { + TierName: "NVME", + Size: humanize.GByte * 500, + Free: humanize.GByte * 250, + Imbalance: 5, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + pool := &PoolInfo{ + TotalTargets: tc.totalTargets, + ActiveTargets: tc.activeTargets, + DisabledTargets: tc.activeTargets, + TierStats: append([]*StorageUsageStats{}, tc.scmStats, tc.nvmeStats), + } + + if diff := cmp.Diff(tc.expUsage, pool.Usage()); diff != "" { + t.Fatalf("Unexpected response (-want, +got):\n%s\n", diff) + } + }) + } +} + +func genTestMask(modifyFn func(pqm *PoolQueryMask)) PoolQueryMask { + testMask := PoolQueryMask(0) + modifyFn(&testMask) + return testMask +} + +func genOptsStr(queryOpts ...string) string { + return strings.Join(queryOpts, ",") +} + +func TestDaos_PoolQueryMask(t *testing.T) { + for name, tc := range map[string]struct { + testMask PoolQueryMask + expString string + }{ + "no mask": { + expString: "", + }, + "default query mask": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + *pqm = DefaultPoolQueryMask + }), + expString: genOptsStr(PoolQueryOptionRebuild, PoolQueryOptionSpace), + }, + "health-only query mask": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + *pqm = HealthOnlyPoolQueryMask + }), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild), + }, + "set query all=true": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.SetAll() + }), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace), + }, + "set query all=false": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + *pqm = PoolQueryMask(^uint64(0)) + pqm.ClearAll() + }), + expString: "", + }, + "set query space=true": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.SetOptions(PoolQueryOptionSpace) + }), + expString: PoolQueryOptionSpace, + }, + "set query space=false": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.SetAll() + pqm.ClearOptions(PoolQueryOptionSpace) + }), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, PoolQueryOptionRebuild), + }, + "set query space=false (already false)": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.ClearOptions(PoolQueryOptionSpace) + }), + expString: "", + }, + "set query rebuild=true": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.SetOptions(PoolQueryOptionRebuild) + }), + expString: PoolQueryOptionRebuild, + }, + "set query rebuild=false": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.SetAll() + pqm.ClearOptions(PoolQueryOptionRebuild) + }), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionEnabledEngines, PoolQueryOptionSpace), + }, + "set query enabled_engines=true": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.SetOptions(PoolQueryOptionEnabledEngines) + }), + expString: PoolQueryOptionEnabledEngines, + }, + "set query enabled_engines=false": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.SetAll() + pqm.ClearOptions(PoolQueryOptionEnabledEngines) + }), + expString: genOptsStr(PoolQueryOptionDisabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace), + }, + "set query disabled_engines=true": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.SetOptions(PoolQueryOptionDisabledEngines) + }), + expString: PoolQueryOptionDisabledEngines, + }, + "set query disabled_engines=false": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.SetAll() + pqm.ClearOptions(PoolQueryOptionDisabledEngines) + }), + expString: genOptsStr(PoolQueryOptionEnabledEngines, PoolQueryOptionRebuild, PoolQueryOptionSpace), + }, + } { + t.Run(name, func(t *testing.T) { + if diff := cmp.Diff(tc.expString, tc.testMask.String()); diff != "" { + t.Fatalf("Unexpected response (-want, +got):\n%s\n", diff) + } + }) + } +} + +func TestDaos_PoolQueryMaskMarshalJSON(t *testing.T) { + // NB: The MarshalJSON implementation uses the stringer, so + // there's no point in testing all of the options here. + for name, tc := range map[string]struct { + testMask PoolQueryMask + expJSON []byte + }{ + "no mask": { + expJSON: []byte(`""`), + }, + "set query all=true": { + testMask: genTestMask(func(pqm *PoolQueryMask) { + pqm.SetAll() + }), + expJSON: []byte(`"disabled_engines,enabled_engines,rebuild,space"`), + }, + } { + t.Run(name, func(t *testing.T) { + gotJSON, err := tc.testMask.MarshalJSON() + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if diff := cmp.Diff(tc.expJSON, gotJSON); diff != "" { + t.Fatalf("Unexpected response (-want, +got):\n%s\n", diff) + } + }) + } +} + +func TestDaos_PoolQueryMaskUnmarshalJSON(t *testing.T) { + for name, tc := range map[string]struct { + testData []byte + expString string + expErr error + }{ + "unknown value": { + testData: []byte("rebuild,foo"), + expErr: errors.New("foo"), + }, + "empty value": { + expString: "", + }, + "uint64 value": { + testData: []byte("18446744073709551603"), + expString: "rebuild,space", + }, + "string values": { + testData: []byte("rebuild,disabled_engines"), + expString: "disabled_engines,rebuild", + }, + } { + t.Run(name, func(t *testing.T) { + var testMask PoolQueryMask + + gotErr := testMask.UnmarshalJSON(tc.testData) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return + } + + if diff := cmp.Diff(tc.expString, testMask.String()); diff != "" { + t.Fatalf("Unexpected mask after UnmarshalJSON (-want, +got):\n%s\n", diff) + } + }) + } +} diff --git a/src/control/lib/hardware/cart/bindings.go b/src/control/lib/hardware/cart/bindings.go index d52c15842d1..ca2b98d5295 100644 --- a/src/control/lib/hardware/cart/bindings.go +++ b/src/control/lib/hardware/cart/bindings.go @@ -7,7 +7,7 @@ package cart /* -#cgo LDFLAGS: -lcart +#cgo LDFLAGS: -lcart -lgurt #include <cart/types.h> #include <cart/api.h> diff --git a/src/control/provider/system/mocks.go b/src/control/provider/system/mocks.go index ab927b760c4..dc52d7b3b96 100644 --- a/src/control/provider/system/mocks.go +++ b/src/control/provider/system/mocks.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2022 Intel Corporation. +// (C) Copyright 2022-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -44,10 +44,19 @@ type ( SourceToTarget map[string]string GetfsIndex int GetfsUsageResps []GetfsUsageRetval - GetFsTypeRes *FsType - GetFsTypeErr []error + GetfsTypeRes *FsType + GetfsTypeErr []error StatErrors map[string]error RealStat bool + ReadFileResults map[string][]byte + ReadFileErrors map[string]error + RealReadFile bool + GeteuidRes int + GetegidRes int + MkdirErr error + RealMkdir bool + RemoveAllErr error + RealRemoveAll bool } // MockSysProvider gives a mock SystemProvider implementation. @@ -57,7 +66,7 @@ type ( cfg MockSysConfig isMounted MountMap IsMountedInputs []string - GetFsTypeCount int + GetfsTypeCount int } ) @@ -162,17 +171,17 @@ func (msp *MockSysProvider) GetfsUsage(_ string) (uint64, uint64, error) { return resp.Total, resp.Avail, resp.Err } -func (msp *MockSysProvider) GetFsType(path string) (*FsType, error) { - idx := msp.GetFsTypeCount - msp.GetFsTypeCount++ +func (msp *MockSysProvider) GetfsType(path string) (*FsType, error) { + idx := msp.GetfsTypeCount + msp.GetfsTypeCount++ var err error var result *FsType - if idx < len(msp.cfg.GetFsTypeErr) { - err = msp.cfg.GetFsTypeErr[idx] + if idx < len(msp.cfg.GetfsTypeErr) { + err = msp.cfg.GetfsTypeErr[idx] } if err == nil { - result = msp.cfg.GetFsTypeRes + result = msp.cfg.GetfsTypeRes } return result, err @@ -191,6 +200,50 @@ func (msp *MockSysProvider) Stat(path string) (os.FileInfo, error) { return nil, msp.cfg.StatErrors[path] } +func (msp *MockSysProvider) ReadFile(path string) ([]byte, error) { + msp.RLock() + defer msp.RUnlock() + + if msp.cfg.RealReadFile { + return os.ReadFile(path) + } + + // default return value for missing key is nil so + // add entries to indicate path failure e.g. perms or not-exist + if msp.cfg.ReadFileErrors[path] != nil { + return nil, msp.cfg.ReadFileErrors[path] + } + + return msp.cfg.ReadFileResults[path], nil +} + +// Geteuid returns the numeric effective user id of the caller. +func (msp *MockSysProvider) Geteuid() int { + return msp.cfg.GeteuidRes +} + +// Getegid returns the numeric effective group id of the caller. +func (msp *MockSysProvider) Getegid() int { + return msp.cfg.GetegidRes +} + +// Mkdir creates a new directory with the specified name and permission +// bits (before umask). +func (msp *MockSysProvider) Mkdir(path string, flags os.FileMode) error { + if msp.cfg.RealMkdir { + return os.Mkdir(path, flags) + } + return msp.cfg.MkdirErr +} + +// RemoveAll removes path and any children it contains. +func (msp *MockSysProvider) RemoveAll(path string) error { + if msp.cfg.RealRemoveAll { + return os.RemoveAll(path) + } + return msp.cfg.RemoveAllErr +} + func NewMockSysProvider(log logging.Logger, cfg *MockSysConfig) *MockSysProvider { if cfg == nil { cfg = &MockSysConfig{} diff --git a/src/control/provider/system/system_linux.go b/src/control/provider/system/system_linux.go index d5986ace89a..e3fb439c0d8 100644 --- a/src/control/provider/system/system_linux.go +++ b/src/control/provider/system/system_linux.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2022 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -205,8 +205,8 @@ type FsType struct { NoSUID bool } -// GetFsType retrieves the filesystem type for a path. -func (s LinuxProvider) GetFsType(path string) (*FsType, error) { +// GetfsType retrieves the filesystem type for a path. +func (s LinuxProvider) GetfsType(path string) (*FsType, error) { stBuf := new(unix.Statfs_t) if err := unix.Statfs(path, stBuf); err != nil { @@ -313,6 +313,11 @@ func (s LinuxProvider) Stat(path string) (os.FileInfo, error) { return os.Stat(path) } +// ReadFile reads the named file and returns the contents. +func (s LinuxProvider) ReadFile(path string) ([]byte, error) { + return os.ReadFile(path) +} + // Chmod changes the mode of the specified path. func (s LinuxProvider) Chmod(path string, mode os.FileMode) error { return os.Chmod(path, mode) @@ -337,3 +342,24 @@ func parseFsType(input string) string { return FsTypeUnknown } } + +// Geteuid returns the numeric effective user id of the caller. +func (s LinuxProvider) Geteuid() int { + return os.Geteuid() +} + +// Getegid returns the numeric effective group id of the caller. +func (s LinuxProvider) Getegid() int { + return os.Getegid() +} + +// Mkdir creates a new directory with the specified name and permission +// bits (before umask). +func (s LinuxProvider) Mkdir(name string, perm os.FileMode) error { + return os.Mkdir(name, perm) +} + +// RemoveAll removes path and any children it contains. +func (s LinuxProvider) RemoveAll(path string) error { + return os.RemoveAll(path) +} diff --git a/src/control/provider/system/system_linux_test.go b/src/control/provider/system/system_linux_test.go index 4a1b0c0d3eb..dc9e6b21a04 100644 --- a/src/control/provider/system/system_linux_test.go +++ b/src/control/provider/system/system_linux_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2022 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -159,7 +159,7 @@ func TestParseFsType(t *testing.T) { } } -func TestSystemLinux_GetFsType(t *testing.T) { +func TestSystemLinux_GetfsType(t *testing.T) { for name, tc := range map[string]struct { path string expResult *FsType @@ -181,7 +181,7 @@ func TestSystemLinux_GetFsType(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - result, err := DefaultProvider().GetFsType(tc.path) + result, err := DefaultProvider().GetfsType(tc.path) test.CmpErr(t, tc.expErr, err) if diff := cmp.Diff(tc.expResult, result); diff != "" { diff --git a/src/control/server/README.md b/src/control/server/README.md index 248877fa846..fa64fcf2241 100644 --- a/src/control/server/README.md +++ b/src/control/server/README.md @@ -186,7 +186,7 @@ storage media which will remove blobstores and remove any filesystem signatures from the SSD controller namespaces. Formatting will be performed on devices identified by PCI addresses specified -in config file parameter `bdev_list` when `bdev_class` is equal to `nvme`. +in config file parameter `bdev_list` when `class` is equal to `nvme`. In order to designate NVMe devices to be used by DAOS data plane instances, the control plane will generate a `daos_nvme.conf` file to be consumed by SPDK diff --git a/src/control/server/config/faults.go b/src/control/server/config/faults.go index 31c034d3e5b..a48c1fe1863 100644 --- a/src/control/server/config/faults.go +++ b/src/control/server/config/faults.go @@ -96,11 +96,6 @@ var ( "hugepages cannot be disabled if bdevs have been specified in config", "either set false (or remove) disable_hugepages parameter or remove nvme storage assignment in config and restart the control server", ) - FaultConfigVMDSettingDuplicate = serverConfigFault( - code.ServerConfigVMDSettingDuplicate, - "enable_vmd and disable_vmd parameters both specified in config", - "remove legacy enable_vmd parameter from config", - ) FaultConfigControlMetadataNoPath = serverConfigFault( code.ServerConfigControlMetadataNoPath, "using a control_metadata device requires a path to use as the mount point", diff --git a/src/control/server/config/server.go b/src/control/server/config/server.go index f4d0202d19a..f168da6bafb 100644 --- a/src/control/server/config/server.go +++ b/src/control/server/config/server.go @@ -76,9 +76,6 @@ type Server struct { Path string `yaml:"-"` // path to config file - // Legacy config file parameters stored in a separate struct. - Legacy ServerLegacy `yaml:",inline"` - // Behavior flags AutoFormat bool `yaml:"-"` } @@ -360,11 +357,6 @@ func (cfg *Server) Load() error { return errors.Errorf("fabric provider string %q includes more than one provider", cfg.Fabric.Provider) } - // Update server config based on legacy parameters. - if err := updateFromLegacyParams(cfg); err != nil { - return errors.Wrap(err, "updating config from legacy parameters") - } - // propagate top-level settings to engine configs for i := range cfg.Engines { cfg.updateServerConfig(&cfg.Engines[i]) @@ -645,12 +637,6 @@ func (cfg *Server) Validate(log logging.Logger) (err error) { } }() - // The config file format no longer supports "servers" - if len(cfg.Legacy.Servers) > 0 { - return errors.New("\"servers\" server config file parameter is deprecated, use " + - "\"engines\" instead") - } - // Set DisableVMD reference if unset in config file. if cfg.DisableVMD == nil { cfg.WithDisableVMD(false) @@ -713,7 +699,6 @@ func (cfg *Server) Validate(log logging.Logger) (err error) { for idx, ec := range cfg.Engines { ec.Storage.ControlMetadata = cfg.Metadata ec.Storage.EngineIdx = uint(idx) - ec.ConvertLegacyStorage(log, idx) ec.Fabric.Update(cfg.Fabric) if err := ec.Validate(); err != nil { diff --git a/src/control/server/config/server_legacy.go b/src/control/server/config/server_legacy.go deleted file mode 100644 index fba515a0c54..00000000000 --- a/src/control/server/config/server_legacy.go +++ /dev/null @@ -1,58 +0,0 @@ -// -// (C) Copyright 2020-2022 Intel Corporation. -// -// SPDX-License-Identifier: BSD-2-Clause-Patent -// - -package config - -import ( - "github.com/daos-stack/daos/src/control/server/engine" -) - -// ServerLegacy describes old configuration options that should be supported for backward -// compatibility and may be deprecated in future releases. -// See utils/config/daos_server.yml for parameter descriptions. -type ServerLegacy struct { - // Detect outdated "enable_vmd" config parameter and direct users to update config file. - EnableVMD *bool `yaml:"enable_vmd,omitempty"` - // Detect outdated "servers" config, to direct users to change their config file. - Servers []*engine.Config `yaml:"servers,omitempty"` - // Detect outdated "recreate_superblocks" config, to direct users to change their config file. - RecreateSuperblocks bool `yaml:"recreate_superblocks,omitempty"` -} - -// WithEnableVMD can be used to set the state of VMD functionality, -// if not enabled then VMD devices will not be used if they exist. -func (sl *ServerLegacy) WithEnableVMD(enabled bool) *ServerLegacy { - sl.EnableVMD = &enabled - return sl -} - -// WithRecreateSuperblocks indicates that a missing superblock should not be treated as -// an error. The server will create new superblocks as necessary. -func (sl *ServerLegacy) WithRecreateSuperblocks() *ServerLegacy { - sl.RecreateSuperblocks = true - return sl -} - -func updateVMDSetting(legacyCfg ServerLegacy, srvCfg *Server) error { - switch { - case legacyCfg.EnableVMD == nil: - return nil // Legacy VMD setting not used. - case srvCfg.DisableVMD != nil: - return FaultConfigVMDSettingDuplicate // Both legacy and current settings used. - default: - disable := !(*legacyCfg.EnableVMD) - srvCfg.DisableVMD = &disable - return nil - } -} - -func updateFromLegacyParams(srvCfg *Server) error { - if err := updateVMDSetting(srvCfg.Legacy, srvCfg); err != nil { - return err - } - - return nil -} diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index 179421e0df2..4c5a3c6a3ab 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -24,7 +24,7 @@ import ( "github.com/pkg/errors" "github.com/daos-stack/daos/src/control/common" - . "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/common/test" "github.com/daos-stack/daos/src/control/logging" "github.com/daos-stack/daos/src/control/security" "github.com/daos-stack/daos/src/control/server/engine" @@ -37,7 +37,6 @@ const ( verbsExample = "../../../../utils/config/examples/daos_server_verbs.yml" mdOnSSDExample = "../../../../utils/config/examples/daos_server_mdonssd.yml" defaultConfig = "../../../../utils/config/daos_server.yml" - legacyConfig = "../../../../utils/config/examples/daos_server_unittests.yml" ) var ( @@ -151,9 +150,9 @@ func TestServerConfig_MarshalUnmarshal(t *testing.T) { } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) - defer ShowBufferOnFailure(t, buf) + defer test.ShowBufferOnFailure(t, buf) - testDir, cleanup := CreateTestDir(t) + testDir, cleanup := test.CreateTestDir(t) defer cleanup() testFile := filepath.Join(testDir, "test.yml") @@ -169,7 +168,7 @@ func TestServerConfig_MarshalUnmarshal(t *testing.T) { err = configA.Validate(log) } - CmpErr(t, tt.expErr, err) + test.CmpErr(t, tt.expErr, err) if tt.expErr != nil { return } @@ -218,7 +217,7 @@ func TestServerConfig_MarshalUnmarshal(t *testing.T) { } func TestServerConfig_Constructed(t *testing.T) { - testDir, cleanup := CreateTestDir(t) + testDir, cleanup := test.CreateTestDir(t) defer cleanup() // First, load a config based on the server config with all options uncommented. @@ -406,7 +405,7 @@ func TestServerConfig_updateServerConfig(t *testing.T) { func TestServerConfig_MDonSSD_Constructed(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) - defer ShowBufferOnFailure(t, buf) + defer test.ShowBufferOnFailure(t, buf) mdOnSSDCfg, err := mockConfigFromFile(t, mdOnSSDExample) if err != nil { @@ -469,7 +468,7 @@ func TestServerConfig_MDonSSD_Constructed(t *testing.T) { } func TestServerConfig_Validation(t *testing.T) { - testDir, cleanup := CreateTestDir(t) + testDir, cleanup := test.CreateTestDir(t) defer cleanup() // First, load a config based on the server config with all options uncommented. @@ -950,7 +949,7 @@ func TestServerConfig_Validation(t *testing.T) { } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) - defer ShowBufferOnFailure(t, buf) + defer test.ShowBufferOnFailure(t, buf) if tt.extraConfig == nil { tt.extraConfig = noopExtra @@ -961,7 +960,7 @@ func TestServerConfig_Validation(t *testing.T) { log.Debugf("baseCfg metadata: %+v", cfg.Metadata) - CmpErr(t, tt.expErr, cfg.Validate(log)) + test.CmpErr(t, tt.expErr, cfg.Validate(log)) if tt.expErr != nil || tt.expConfig == nil { return } @@ -974,7 +973,7 @@ func TestServerConfig_Validation(t *testing.T) { } func TestServerConfig_SetNrHugepages(t *testing.T) { - testDir, cleanup := CreateTestDir(t) + testDir, cleanup := test.CreateTestDir(t) defer cleanup() // First, load a config based on the server config with all options uncommented. @@ -1100,7 +1099,7 @@ func TestServerConfig_SetNrHugepages(t *testing.T) { } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) - defer ShowBufferOnFailure(t, buf) + defer test.ShowBufferOnFailure(t, buf) // Apply test case changes to basic config cfg := tc.extraConfig(baseCfg(t, testFile)) @@ -1109,19 +1108,19 @@ func TestServerConfig_SetNrHugepages(t *testing.T) { HugepageSizeKiB: 2048, } - CmpErr(t, tc.expErr, cfg.SetNrHugepages(log, mi)) + test.CmpErr(t, tc.expErr, cfg.SetNrHugepages(log, mi)) if tc.expErr != nil { return } - AssertEqual(t, tc.expNrHugepages, cfg.NrHugepages, + test.AssertEqual(t, tc.expNrHugepages, cfg.NrHugepages, "unexpected number of hugepages set in config") }) } } func TestServerConfig_SetRamdiskSize(t *testing.T) { - testDir, cleanup := CreateTestDir(t) + testDir, cleanup := test.CreateTestDir(t) defer cleanup() // First, load a config based on the server config with all options uncommented. @@ -1283,7 +1282,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) { } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) - defer ShowBufferOnFailure(t, buf) + defer test.ShowBufferOnFailure(t, buf) // Apply test case changes to basic config cfg := tc.extraConfig(baseCfg(t, testFile)) @@ -1297,7 +1296,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) { MemTotalKiB: int(val), } - CmpErr(t, tc.expErr, cfg.SetRamdiskSize(log, mi)) + test.CmpErr(t, tc.expErr, cfg.SetRamdiskSize(log, mi)) if tc.expErr != nil { return } @@ -1313,7 +1312,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) { if scmTiers[0].Class != storage.ClassRam { t.Fatal("expected scm tier to have class RAM") } - AssertEqual(t, tc.expRamdiskSize, int(scmTiers[0].Scm.RamdiskSize), + test.AssertEqual(t, tc.expRamdiskSize, int(scmTiers[0].Scm.RamdiskSize), "unexpected ramdisk size set in config") } }) @@ -1341,6 +1340,8 @@ func replaceLine(r io.Reader, w io.Writer, oldTxt, newTxt string) (int, error) { } func replaceFile(t *testing.T, name, oldTxt, newTxt string) { + t.Helper() + // open original file f, err := os.Open(name) if err != nil { @@ -1361,7 +1362,7 @@ func replaceFile(t *testing.T, name, oldTxt, newTxt string) { t.Fatal(err) } if linesChanged == 0 { - t.Fatalf("no recurrences of %q in file %q", oldTxt, name) + t.Fatalf("no occurrences of %q in file %q", oldTxt, name) } // make sure the tmp file was successfully written to @@ -1401,7 +1402,7 @@ func TestServerConfig_Parsing(t *testing.T) { } // load a config based on the server config with all options uncommented. - loadFromDefaultFile := func(t *testing.T, testDir string, matchText, replaceText []string) (*Server, error) { + loadFromFile := func(t *testing.T, testDir string, matchText, replaceText []string) (*Server, error) { t.Helper() defaultConfigFile := filepath.Join(testDir, sConfigUncomment) @@ -1410,25 +1411,6 @@ func TestServerConfig_Parsing(t *testing.T) { return cfgFromFile(t, defaultConfigFile, matchText, replaceText) } - // load a config file with a legacy storage config - loadFromLegacyFile := func(t *testing.T, testDir string, matchText, replaceText []string) (*Server, error) { - t.Helper() - - lcp := strings.Split(legacyConfig, "/") - testLegacyConfigFile := filepath.Join(testDir, lcp[len(lcp)-1]) - CopyFile(t, legacyConfig, testLegacyConfigFile) - - return cfgFromFile(t, testLegacyConfigFile, matchText, replaceText) - } - - loadFromFile := func(t *testing.T, testDir string, matchText, replaceText []string, legacy bool) (*Server, error) { - if legacy { - return loadFromLegacyFile(t, testDir, matchText, replaceText) - } - - return loadFromDefaultFile(t, testDir, matchText, replaceText) - } - for name, tt := range map[string]struct { inTxt string outTxt string @@ -1445,11 +1427,6 @@ func TestServerConfig_Parsing(t *testing.T) { outTxt: "engine:", expParseErr: errors.New("field engine not found"), }, - "use legacy servers conf directive rather than engines": { - inTxt: "engines:", - outTxt: "servers:", - expValidateErr: errors.New("use \"engines\" instead"), - }, "duplicates in bdev_list from config": { extraConfig: func(c *Server) *Server { return c.WithEngines( @@ -1462,7 +1439,7 @@ func TestServerConfig_Parsing(t *testing.T) { WithScmMountPoint("/mnt/daos/2"), storage.NewTierConfig(). WithStorageClass("nvme"). - WithBdevDeviceList(MockPCIAddr(1), MockPCIAddr(1)), + WithBdevDeviceList(test.MockPCIAddr(1), test.MockPCIAddr(1)), ). WithTargetCount(8)) }, @@ -1486,86 +1463,19 @@ func TestServerConfig_Parsing(t *testing.T) { return nil }, }, - "specify legacy servers conf directive in addition to engines": { - inTxt: "engines:", - outTxt: "servers:", - extraConfig: func(c *Server) *Server { - var nilEngineConfig *engine.Config - return c.WithEngines(nilEngineConfig) - }, - expValidateErr: errors.New("use \"engines\" instead"), - }, - "legacy storage; empty bdev_list": { - legacyStorage: true, - expCheck: func(c *Server) error { - nr := len(c.Engines[0].Storage.Tiers) - if nr != 1 { - return errors.Errorf("want 1 storage tier, got %d", nr) - } - return nil - }, - }, - "legacy storage; no bdev_list": { - legacyStorage: true, - inTxt: " bdev_list: []", - outTxt: "", - expCheck: func(c *Server) error { - nr := len(c.Engines[0].Storage.Tiers) - if nr != 1 { - return errors.Errorf("want 1 storage tier, got %d", nr) - } - return nil - }, - }, - "legacy storage; no bdev_class": { - legacyStorage: true, - inTxt: " bdev_class: nvme", - outTxt: "", - expCheck: func(c *Server) error { - nr := len(c.Engines[0].Storage.Tiers) - if nr != 1 { - return errors.Errorf("want 1 storage tier, got %d", nr) - } - return nil - }, - }, - "legacy storage; non-empty bdev_list": { - legacyStorage: true, - inTxt: " bdev_list: []", - outTxt: " bdev_list: [0000:80:00.0]", - expCheck: func(c *Server) error { - nr := len(c.Engines[0].Storage.Tiers) - if nr != 2 { - return errors.Errorf("want 2 storage tiers, got %d", nr) - } - return nil - }, + "no bdev_list": { + inTxt: " bdev_list: [\"0000:81:00.0\", \"0000:82:00.0\"] # generate regular nvme.conf", + outTxt: "", + expValidateErr: errors.New("valid PCI addresses"), }, - "legacy storage; non-empty bdev_busid_range": { - legacyStorage: true, - inTxtList: []string{ - " bdev_list: []", " bdev_busid_range: \"\"", - }, - outTxtList: []string{ - " bdev_list: [0000:80:00.0]", " bdev_busid_range: \"0x00-0x80\"", - }, - expCheck: func(c *Server) error { - nr := len(c.Engines[0].Storage.Tiers) - if nr != 2 { - return errors.Errorf("want 2 storage tiers, got %d", nr) - } - want := storage.MustNewBdevBusRange("0x00-0x80") - got := c.Engines[0].Storage.Tiers.BdevConfigs()[0].Bdev.BusidRange - if want.String() != got.String() { - return errors.Errorf("want %s bus-id range, got %s", want, got) - } - return nil - }, + "no bdev_class": { + inTxt: " class: nvme", + outTxt: "", + expValidateErr: errors.New("no storage class"), }, - "legacy storage; empty bdev_list; hugepages disabled": { - legacyStorage: true, - inTxt: "telemetry_port: 9191", - outTxt: "disable_hugepages: true", + "non-empty bdev_list; hugepages disabled": { + inTxt: "disable_hugepages: false", + outTxt: "disable_hugepages: true", expCheck: func(c *Server) error { if !c.DisableHugepages { return errors.Errorf("expected hugepages to be disabled") @@ -1573,31 +1483,6 @@ func TestServerConfig_Parsing(t *testing.T) { return nil }, }, - "legacy vmd enable": { - inTxt: "disable_vmd: true", - outTxt: "enable_vmd: true", - expCheck: func(c *Server) error { - if *c.DisableVMD != false { - return errors.Errorf("expected vmd to be not disabled") - } - return nil - }, - }, - "legacy vmd disable": { - inTxt: "disable_vmd: true", - outTxt: "enable_vmd: false", - expCheck: func(c *Server) error { - if *c.DisableVMD != true { - return errors.Errorf("expected vmd to be disabled") - } - return nil - }, - }, - "legacy vmd disable; current vmd enable": { - inTxt: "disable_vfio: true", - outTxt: "enable_vmd: true", - expParseErr: FaultConfigVMDSettingDuplicate, - }, "check default system_ram_reserved": { inTxt: "system_ram_reserved: 5", outTxt: "", @@ -1612,9 +1497,9 @@ func TestServerConfig_Parsing(t *testing.T) { } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) - defer ShowBufferOnFailure(t, buf) + defer test.ShowBufferOnFailure(t, buf) - testDir, cleanup := CreateTestDir(t) + testDir, cleanup := test.CreateTestDir(t) defer cleanup() if tt.extraConfig == nil { @@ -1636,14 +1521,14 @@ func TestServerConfig_Parsing(t *testing.T) { tt.outTxtList = []string{tt.outTxt} } - config, errParse := loadFromFile(t, testDir, tt.inTxtList, tt.outTxtList, tt.legacyStorage) - CmpErr(t, tt.expParseErr, errParse) + config, errParse := loadFromFile(t, testDir, tt.inTxtList, tt.outTxtList) + test.CmpErr(t, tt.expParseErr, errParse) if tt.expParseErr != nil { return } config = tt.extraConfig(config) - CmpErr(t, tt.expValidateErr, config.Validate(log)) + test.CmpErr(t, tt.expValidateErr, config.Validate(log)) if tt.expCheck != nil { if err := tt.expCheck(config); err != nil { @@ -1663,7 +1548,7 @@ func TestServerConfig_RelativeWorkingPath(t *testing.T) { "path does not exist": {expErrMsg: "no such file or directory"}, } { t.Run(name, func(t *testing.T) { - testDir, cleanup := CreateTestDir(t) + testDir, cleanup := test.CreateTestDir(t) defer cleanup() testFile := filepath.Join(testDir, "test.yml") @@ -1811,13 +1696,14 @@ func TestServerConfig_validateMultiEngineConfig(t *testing.T) { AppendStorage( storage.NewTierConfig(). WithStorageClass(storage.ClassNvme.String()). - WithBdevDeviceList(MockPCIAddr(1)), + WithBdevDeviceList(test.MockPCIAddr(1)), ), configB: configB(). AppendStorage( storage.NewTierConfig(). WithStorageClass(storage.ClassNvme.String()). - WithBdevDeviceList(MockPCIAddr(2), MockPCIAddr(1)), + WithBdevDeviceList(test.MockPCIAddr(2), + test.MockPCIAddr(1)), ), expErr: FaultConfigOverlappingBdevDeviceList(1, 0), }, @@ -1826,13 +1712,15 @@ func TestServerConfig_validateMultiEngineConfig(t *testing.T) { AppendStorage( storage.NewTierConfig(). WithStorageClass(storage.ClassNvme.String()). - WithBdevDeviceList(MockPCIAddr(1), MockPCIAddr(2)), + WithBdevDeviceList(test.MockPCIAddr(1), + test.MockPCIAddr(2)), ), configB: configB(). AppendStorage( storage.NewTierConfig(). WithStorageClass(storage.ClassNvme.String()). - WithBdevDeviceList(MockPCIAddr(2), MockPCIAddr(1)), + WithBdevDeviceList(test.MockPCIAddr(2), + test.MockPCIAddr(1)), ), expErr: errors.New("engine 1 overlaps with entries in engine 0"), }, @@ -1852,38 +1740,39 @@ func TestServerConfig_validateMultiEngineConfig(t *testing.T) { AppendStorage( storage.NewTierConfig(). WithStorageClass(storage.ClassNvme.String()). - WithBdevDeviceList(MockPCIAddr(1)), + WithBdevDeviceList(test.MockPCIAddr(1)), ), configB: configB(). AppendStorage( storage.NewTierConfig(). WithStorageClass(storage.ClassNvme.String()). - WithBdevDeviceList(MockPCIAddr(2), MockPCIAddr(3)), + WithBdevDeviceList(test.MockPCIAddr(2), + test.MockPCIAddr(3)), ), expLog: "engine 1 has 2 but engine 0 has 1", }, } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) - defer ShowBufferOnFailure(t, buf) + defer test.ShowBufferOnFailure(t, buf) conf := DefaultServer(). WithFabricProvider("test"). WithEngines(tc.configA, tc.configB) gotErr := conf.Validate(log) - CmpErr(t, tc.expErr, gotErr) + test.CmpErr(t, tc.expErr, gotErr) if tc.expLog != "" { hasEntry := strings.Contains(buf.String(), tc.expLog) - AssertTrue(t, hasEntry, "expected entries not found in log") + test.AssertTrue(t, hasEntry, "expected entries not found in log") } }) } } func TestServerConfig_SaveActiveConfig(t *testing.T) { - testDir, cleanup := CreateTestDir(t) + testDir, cleanup := test.CreateTestDir(t) defer cleanup() t.Logf("test dir: %s", testDir) @@ -1903,13 +1792,13 @@ func TestServerConfig_SaveActiveConfig(t *testing.T) { } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) - defer ShowBufferOnFailure(t, buf) + defer test.ShowBufferOnFailure(t, buf) cfg := DefaultServer().WithSocketDir(tc.cfgPath) cfg.SaveActiveConfig(log) - AssertTrue(t, strings.Contains(buf.String(), tc.expLogOut), + test.AssertTrue(t, strings.Contains(buf.String(), tc.expLogOut), fmt.Sprintf("expected %q in %q", tc.expLogOut, buf.String())) }) } @@ -1967,15 +1856,15 @@ func TestConfig_detectEngineAffinity(t *testing.T) { } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) - defer ShowBufferOnFailure(t, buf) + defer test.ShowBufferOnFailure(t, buf) detected, err := detectEngineAffinity(log, tc.cfg, tc.affSrcSet...) - CmpErr(t, tc.expErr, err) + test.CmpErr(t, tc.expErr, err) if tc.expErr != nil { return } - AssertEqual(t, tc.expDetected, detected, + test.AssertEqual(t, tc.expDetected, detected, "unexpected detected numa node") }) } @@ -2015,16 +1904,16 @@ func TestConfig_SetNUMAAffinity(t *testing.T) { } { t.Run(name, func(t *testing.T) { err := tc.cfg.SetNUMAAffinity(tc.setNUMA) - CmpErr(t, tc.expErr, err) + test.CmpErr(t, tc.expErr, err) if tc.expErr != nil { return } - AssertEqual(t, tc.expNUMA, *tc.cfg.PinnedNumaNode, + test.AssertEqual(t, tc.expNUMA, *tc.cfg.PinnedNumaNode, "unexpected pinned numa node") - AssertEqual(t, tc.expNUMA, tc.cfg.Fabric.NumaNodeIndex, + test.AssertEqual(t, tc.expNUMA, tc.cfg.Fabric.NumaNodeIndex, "unexpected numa node in fabric config") - AssertEqual(t, tc.expNUMA, tc.cfg.Storage.NumaNodeIndex, + test.AssertEqual(t, tc.expNUMA, tc.cfg.Storage.NumaNodeIndex, "unexpected numa node in storage config") }) } @@ -2195,7 +2084,7 @@ func TestConfig_SetEngineAffinities(t *testing.T) { fabNumaSet := make([]int, 0, len(tc.expFabNumas)) log, buf := logging.NewTestLogger(t.Name()) - defer ShowBufferOnFailure(t, buf) + defer test.ShowBufferOnFailure(t, buf) if tc.affSrcSet == nil { tc.affSrcSet = []EngineAffinityFn{ @@ -2205,7 +2094,7 @@ func TestConfig_SetEngineAffinities(t *testing.T) { } gotErr := tc.cfg.SetEngineAffinities(log, tc.affSrcSet...) - CmpErr(t, tc.expErr, gotErr) + test.CmpErr(t, tc.expErr, gotErr) if tc.expErr != nil { return } diff --git a/src/control/server/ctl_ranks_rpc_test.go b/src/control/server/ctl_ranks_rpc_test.go index 62889f14ed3..0d2c1849e4e 100644 --- a/src/control/server/ctl_ranks_rpc_test.go +++ b/src/control/server/ctl_ranks_rpc_test.go @@ -217,7 +217,9 @@ func TestServer_CtlSvc_PrepShutdownRanks(t *testing.T) { cfg.setResponseDelay(tc.responseDelay) } } - srv.setDrpcClient(newMockDrpcClient(cfg)) + srv.getDrpcClientFn = func(s string) drpc.DomainSocketClient { + return newMockDrpcClient(cfg) + } } var cancel context.CancelFunc @@ -911,7 +913,9 @@ func TestServer_CtlSvc_SetEngineLogMasks(t *testing.T) { cfg.setResponseDelay(tc.responseDelay) } } - srv.setDrpcClient(newMockDrpcClient(cfg)) + srv.getDrpcClientFn = func(s string) drpc.DomainSocketClient { + return newMockDrpcClient(cfg) + } } gotResp, gotErr := svc.SetEngineLogMasks(test.Context(t), tc.req) diff --git a/src/control/server/ctl_smd_rpc_test.go b/src/control/server/ctl_smd_rpc_test.go index 0a40f048525..a37c6548c68 100644 --- a/src/control/server/ctl_smd_rpc_test.go +++ b/src/control/server/ctl_smd_rpc_test.go @@ -702,7 +702,10 @@ func TestServer_CtlSvc_SmdQuery(t *testing.T) { cfg.setSendMsgResponseList(t, mock) } } - srv.setDrpcClient(newMockDrpcClient(cfg)) + mdc := newMockDrpcClient(cfg) + srv.getDrpcClientFn = func(s string) drpc.DomainSocketClient { + return mdc + } srv.ready.SetTrue() } if tc.harnessStopped { @@ -1577,7 +1580,10 @@ func TestServer_CtlSvc_SmdManage(t *testing.T) { cfg.setSendMsgResponseList(t, mock) } } - ei.setDrpcClient(newMockDrpcClient(cfg)) + mdc := newMockDrpcClient(cfg) + ei.getDrpcClientFn = func(s string) drpc.DomainSocketClient { + return mdc + } ei.ready.SetTrue() } if tc.harnessStopped { diff --git a/src/control/server/ctl_storage_rpc_test.go b/src/control/server/ctl_storage_rpc_test.go index 4df8b2b0e65..42d817802c9 100644 --- a/src/control/server/ctl_storage_rpc_test.go +++ b/src/control/server/ctl_storage_rpc_test.go @@ -1659,6 +1659,7 @@ func TestServer_CtlSvc_StorageFormat(t *testing.T) { IsMountedBool: tc.scmMounted, GetfsStr: getFsRetStr, SourceToTarget: devToMount, + RealReadFile: true, } if tc.sClass == storage.ClassRam { total := uint64(1234) diff --git a/src/control/server/ctl_svc_test.go b/src/control/server/ctl_svc_test.go index cb87c590788..98e07fdb515 100644 --- a/src/control/server/ctl_svc_test.go +++ b/src/control/server/ctl_svc_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -79,6 +79,7 @@ func newMockControlServiceFromBackends(t *testing.T, log logging.Logger, cfg *co }) if started[idx] { ei.ready.SetTrue() + ei.setDrpcSocket("/dontcare") } if err := cs.harness.AddInstance(ei); err != nil { t.Fatal(err) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 65516b40062..3e6ab0b64d7 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -248,7 +248,6 @@ type Config struct { SocketDir string `yaml:"-" cmdLongFlag:"--socket_dir" cmdShortFlag:"-d"` LogMask string `yaml:"log_mask,omitempty" cmdEnv:"D_LOG_MASK"` LogFile string `yaml:"log_file,omitempty" cmdEnv:"D_LOG_FILE"` - LegacyStorage LegacyStorage `yaml:",inline,omitempty"` Storage storage.Config `yaml:",inline,omitempty"` Fabric FabricConfig `yaml:",inline"` EnvVars []string `yaml:"env_vars,omitempty"` diff --git a/src/control/server/engine/config_legacy.go b/src/control/server/engine/config_legacy.go deleted file mode 100644 index 3dd455a163d..00000000000 --- a/src/control/server/engine/config_legacy.go +++ /dev/null @@ -1,75 +0,0 @@ -// -// (C) Copyright 2022 Intel Corporation. -// -// SPDX-License-Identifier: BSD-2-Clause-Patent -// - -package engine - -import ( - "github.com/daos-stack/daos/src/control/logging" - "github.com/daos-stack/daos/src/control/server/storage" -) - -// LegacyStorage struct contains the old format of specifying SCM and Bdev storage. -type LegacyStorage struct { - storage.ScmConfig `yaml:",inline,omitempty"` - ScmClass storage.Class `yaml:"scm_class,omitempty"` - storage.BdevConfig `yaml:",inline,omitempty"` - BdevClass storage.Class `yaml:"bdev_class,omitempty"` -} - -// WasDefined returns true if the LegacyStorage reference refers to a populated struct. -func (ls *LegacyStorage) WasDefined() bool { - return ls.ScmClass != storage.ClassNone || ls.BdevClass != storage.ClassNone -} - -// WithLegacyStorage populates the engine config field appropriately. -func (ec *Config) WithLegacyStorage(lc LegacyStorage) *Config { - ec.LegacyStorage = lc - return ec -} - -// ConvertLegacyStorage takes engine config legacy storage field and populates relevant config -// storage tiers. -func (ec *Config) ConvertLegacyStorage(log logging.Logger, idx int) { - ls := ec.LegacyStorage - if ls.WasDefined() { - log.Noticef("engine %d: Legacy storage configuration detected. Please "+ - "migrate to new-style storage configuration.", idx) - var tierCfgs storage.TierConfigs - if ls.ScmClass != storage.ClassNone { - tierCfgs = append(tierCfgs, - storage.NewTierConfig(). - WithStorageClass(ls.ScmClass.String()). - WithScmDeviceList(ls.ScmConfig.DeviceList...). - WithScmMountPoint(ls.MountPoint). - WithScmRamdiskSize(ls.RamdiskSize), - ) - } - - // Do not add bdev tier if BdevClass is none or nvme has no devices. - bc := ls.BdevClass - switch { - case bc == storage.ClassNvme && ls.BdevConfig.DeviceList.Len() == 0: - log.Debugf("legacy storage config conversion skipped for class "+ - "%s with empty bdev_list", storage.ClassNvme) - case bc == storage.ClassNone: - log.Debugf("legacy storage config bdev bonversion skipped for class %s", - storage.ClassNone) - default: - tierCfgs = append(tierCfgs, - storage.NewTierConfig(). - WithStorageClass(ls.BdevClass.String()). - WithBdevDeviceCount(ls.DeviceCount). - WithBdevDeviceList( - ls.BdevConfig.DeviceList.Devices()...). - WithBdevFileSize(ls.FileSize). - WithBdevBusidRange( - ls.BdevConfig.BusidRange.String()), - ) - } - ec.WithStorage(tierCfgs...) - ec.LegacyStorage = LegacyStorage{} - } -} diff --git a/src/control/server/faults.go b/src/control/server/faults.go index daf569fd737..cf7d4e2ade2 100644 --- a/src/control/server/faults.go +++ b/src/control/server/faults.go @@ -143,7 +143,7 @@ func FaultScmUnmanaged(mntPoint string) *fault.Fault { return serverFault( code.ServerScmUnmanaged, fmt.Sprintf("the SCM mountpoint at %s is unavailable and can't be created/mounted", mntPoint), - fmt.Sprintf("manually create %s or remove --recreate-superblocks from the server arguments", mntPoint), + fmt.Sprintf("manually create %s and retry", mntPoint), ) } diff --git a/src/control/server/harness_test.go b/src/control/server/harness_test.go index 116b5edda0b..0b0c6ad877d 100644 --- a/src/control/server/harness_test.go +++ b/src/control/server/harness_test.go @@ -294,13 +294,17 @@ func TestServer_Harness_Start(t *testing.T) { } instances := harness.Instances() - + mockDrpcClients := make([]*mockDrpcClient, 0, len(instances)) // set mock dRPC client to record call details for _, e := range instances { ei := e.(*EngineInstance) - ei.setDrpcClient(newMockDrpcClient(&mockDrpcClientConfig{ + cli := newMockDrpcClient(&mockDrpcClientConfig{ SendMsgResponse: &drpc.Response{}, - })) + }) + mockDrpcClients = append(mockDrpcClients, cli) + ei.getDrpcClientFn = func(s string) drpc.DomainSocketClient { + return cli + } } ctx, cancel := context.WithCancel(test.Context(t)) @@ -417,13 +421,10 @@ func TestServer_Harness_Start(t *testing.T) { defer joinMu.Unlock() // verify expected RPCs were made, ranks allocated and // members added to membership - for _, e := range instances { + for i, e := range instances { ei := e.(*EngineInstance) - dc, err := ei.getDrpcClient() - if err != nil { - t.Fatal(err) - } - gotDrpcCalls := dc.(*mockDrpcClient).CalledMethods() + dc := mockDrpcClients[i] + gotDrpcCalls := dc.CalledMethods() AssertEqual(t, tc.expDrpcCalls[ei.Index()], gotDrpcCalls, fmt.Sprintf("%s: unexpected dRPCs for instance %d", name, ei.Index())) diff --git a/src/control/server/instance.go b/src/control/server/instance.go index 215b8c424b5..9c1e19cfa63 100644 --- a/src/control/server/instance.go +++ b/src/control/server/instance.go @@ -58,12 +58,13 @@ type EngineInstance struct { onStorageReady []onStorageReadyFn onReady []onReadyFn onInstanceExit []onInstanceExitFn + getDrpcClientFn func(string) drpc.DomainSocketClient sync.RWMutex // these must be protected by a mutex in order to // avoid racy access. + _drpcSocket string _cancelCtx context.CancelFunc - _drpcClient drpc.DomainSocketClient _superblock *Superblock _lastErr error // populated when harness receives signal } @@ -168,11 +169,7 @@ func (ei *EngineInstance) SetCheckerMode(enabled bool) { func (ei *EngineInstance) removeSocket() error { fMsg := fmt.Sprintf("removing instance %d socket file", ei.Index()) - dc, err := ei.getDrpcClient() - if err != nil { - return errors.Wrap(err, fMsg) - } - engineSock := dc.GetSocketPath() + engineSock := ei.getDrpcSocket() if err := checkDrpcClientSocketPath(engineSock); err != nil { return errors.Wrap(err, fMsg) diff --git a/src/control/server/instance_drpc.go b/src/control/server/instance_drpc.go index d516ba7e636..5339a18d23b 100644 --- a/src/control/server/instance_drpc.go +++ b/src/control/server/instance_drpc.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2023 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -25,23 +25,29 @@ import ( ) var ( - errDRPCNotReady = errors.New("no dRPC client set (data plane not started?)") + errDRPCNotReady = errors.New("dRPC socket not ready (data plane not started?)") errEngineNotReady = errors.New("engine not ready yet") ) -func (ei *EngineInstance) setDrpcClient(c drpc.DomainSocketClient) { +func (ei *EngineInstance) setDrpcSocket(sock string) { ei.Lock() defer ei.Unlock() - ei._drpcClient = c + ei._drpcSocket = sock } -func (ei *EngineInstance) getDrpcClient() (drpc.DomainSocketClient, error) { +func (ei *EngineInstance) getDrpcSocket() string { ei.RLock() defer ei.RUnlock() - if ei._drpcClient == nil { - return nil, errDRPCNotReady + return ei._drpcSocket +} + +func (ei *EngineInstance) getDrpcClient() drpc.DomainSocketClient { + ei.Lock() + defer ei.Unlock() + if ei.getDrpcClientFn == nil { + ei.getDrpcClientFn = drpc.NewClientConnection } - return ei._drpcClient, nil + return ei.getDrpcClientFn(ei._drpcSocket) } // NotifyDrpcReady receives a ready message from the running Engine @@ -49,8 +55,7 @@ func (ei *EngineInstance) getDrpcClient() (drpc.DomainSocketClient, error) { func (ei *EngineInstance) NotifyDrpcReady(msg *srvpb.NotifyReadyReq) { ei.log.Debugf("%s instance %d drpc ready: %v", build.DataPlaneName, ei.Index(), msg) - // activate the dRPC client connection to this engine - ei.setDrpcClient(drpc.NewClientConnection(msg.DrpcListenerSock)) + ei.setDrpcSocket(msg.DrpcListenerSock) go func() { ei.drpcReady <- msg @@ -64,11 +69,12 @@ func (ei *EngineInstance) awaitDrpcReady() chan *srvpb.NotifyReadyReq { return ei.drpcReady } +func (ei *EngineInstance) isDrpcSocketReady() bool { + return ei.getDrpcSocket() != "" +} + func (ei *EngineInstance) callDrpc(ctx context.Context, method drpc.Method, body proto.Message) (*drpc.Response, error) { - dc, err := ei.getDrpcClient() - if err != nil { - return nil, err - } + dc := ei.getDrpcClient() rankMsg := "" if sb := ei.getSuperblock(); sb != nil && sb.Rank != nil { @@ -91,6 +97,9 @@ func (ei *EngineInstance) CallDrpc(ctx context.Context, method drpc.Method, body if !ei.IsReady() { return nil, errEngineNotReady } + if !ei.isDrpcSocketReady() { + return nil, errDRPCNotReady + } return ei.callDrpc(ctx, method, body) } diff --git a/src/control/server/instance_drpc_test.go b/src/control/server/instance_drpc_test.go index c42d9ce87f1..6383ddcf236 100644 --- a/src/control/server/instance_drpc_test.go +++ b/src/control/server/instance_drpc_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2023 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -7,7 +7,9 @@ package server import ( + "context" "fmt" + "sync" "testing" "time" @@ -15,6 +17,7 @@ import ( "github.com/pkg/errors" "google.golang.org/protobuf/proto" + "github.com/daos-stack/daos/src/control/common/proto/mgmt" mgmtpb "github.com/daos-stack/daos/src/control/common/proto/mgmt" srvpb "github.com/daos-stack/daos/src/control/common/proto/srv" "github.com/daos-stack/daos/src/control/common/test" @@ -52,10 +55,7 @@ func TestEngineInstance_NotifyDrpcReady(t *testing.T) { instance.NotifyDrpcReady(req) - dc, err := instance.getDrpcClient() - if err != nil || dc == nil { - t.Fatal("Expected a dRPC client connection") - } + test.AssertEqual(t, req.DrpcListenerSock, instance.getDrpcSocket(), "expected socket value set") waitForEngineReady(t, instance) } @@ -64,6 +64,7 @@ func TestEngineInstance_CallDrpc(t *testing.T) { for name, tc := range map[string]struct { notStarted bool notReady bool + noSocket bool noClient bool resp *drpc.Response expErr error @@ -76,8 +77,8 @@ func TestEngineInstance_CallDrpc(t *testing.T) { notReady: true, expErr: errEngineNotReady, }, - "no client configured": { - noClient: true, + "drpc not ready": { + noSocket: true, expErr: errDRPCNotReady, }, "success": { @@ -94,11 +95,15 @@ func TestEngineInstance_CallDrpc(t *testing.T) { instance := NewEngineInstance(log, nil, nil, runner) instance.ready.Store(!tc.notReady) - if !tc.noClient { - cfg := &mockDrpcClientConfig{ - SendMsgResponse: tc.resp, - } - instance.setDrpcClient(newMockDrpcClient(cfg)) + if !tc.noSocket { + instance.setDrpcSocket("/something") + } + + cfg := &mockDrpcClientConfig{ + SendMsgResponse: tc.resp, + } + instance.getDrpcClientFn = func(s string) drpc.DomainSocketClient { + return newMockDrpcClient(cfg) } _, err := instance.CallDrpc(test.Context(t), @@ -108,6 +113,108 @@ func TestEngineInstance_CallDrpc(t *testing.T) { } } +type sendMsgDrpcClient struct { + sync.Mutex + sendMsgFn func(context.Context, *drpc.Call) (*drpc.Response, error) +} + +func (c *sendMsgDrpcClient) IsConnected() bool { + return true +} + +func (c *sendMsgDrpcClient) Connect(_ context.Context) error { + return nil +} + +func (c *sendMsgDrpcClient) Close() error { + return nil +} + +func (c *sendMsgDrpcClient) SendMsg(ctx context.Context, call *drpc.Call) (*drpc.Response, error) { + return c.sendMsgFn(ctx, call) +} + +func (c *sendMsgDrpcClient) GetSocketPath() string { + return "" +} + +func TestEngineInstance_CallDrpc_Parallel(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + // This test starts with one long-running drpc client that should remain in the SendMsg + // function until all other clients complete, demonstrating a single dRPC call cannot + // block the channel. + + numClients := 100 + numFastClients := numClients - 1 + + doneCh := make(chan struct{}, numFastClients) + longClient := &sendMsgDrpcClient{ + sendMsgFn: func(ctx context.Context, _ *drpc.Call) (*drpc.Response, error) { + numDone := 0 + + for numDone < numFastClients { + select { + case <-ctx.Done(): + t.Fatalf("context done before test finished: %s", ctx.Err()) + case <-doneCh: + numDone++ + t.Logf("%d/%d finished", numDone, numFastClients) + } + } + t.Log("long running client finished") + return &drpc.Response{}, nil + }, + } + + clientCh := make(chan drpc.DomainSocketClient, numClients) + go func(t *testing.T) { + t.Log("starting client producer thread...") + t.Log("adding long-running client") + clientCh <- longClient + for i := 0; i < numFastClients; i++ { + t.Logf("adding client %d", i) + clientCh <- &sendMsgDrpcClient{ + sendMsgFn: func(ctx context.Context, _ *drpc.Call) (*drpc.Response, error) { + doneCh <- struct{}{} + return &drpc.Response{}, nil + }, + } + } + t.Log("closing client channel") + close(clientCh) + }(t) + + t.Log("setting up engine...") + trc := engine.TestRunnerConfig{} + trc.Running.Store(true) + runner := engine.NewTestRunner(&trc, engine.MockConfig()) + instance := NewEngineInstance(log, nil, nil, runner) + instance.ready.Store(true) + + instance.getDrpcClientFn = func(s string) drpc.DomainSocketClient { + t.Log("fetching drpc client") + cli := <-clientCh + t.Log("got drpc client") + return cli + } + + var wg sync.WaitGroup + wg.Add(numClients) + for i := 0; i < numClients; i++ { + go func(t *testing.T, j int) { + t.Logf("%d: CallDrpc", j) + _, err := instance.CallDrpc(test.Context(t), drpc.MethodPoolCreate, &mgmt.PoolCreateReq{}) + if err != nil { + t.Logf("%d: error: %s", j, err.Error()) + } + wg.Done() + }(t, i) + } + wg.Wait() +} + func TestEngineInstance_DrespToRankResult(t *testing.T) { dRank := Rank(1) diff --git a/src/control/server/instance_storage.go b/src/control/server/instance_storage.go index 8cc36304228..71dc0ca181e 100644 --- a/src/control/server/instance_storage.go +++ b/src/control/server/instance_storage.go @@ -8,6 +8,7 @@ package server import ( "context" + "fmt" "os" "github.com/dustin/go-humanize" @@ -15,6 +16,8 @@ import ( "github.com/daos-stack/daos/src/control/build" "github.com/daos-stack/daos/src/control/events" + "github.com/daos-stack/daos/src/control/fault" + "github.com/daos-stack/daos/src/control/fault/code" "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/server/storage" ) @@ -26,13 +29,17 @@ func (ei *EngineInstance) GetStorage() *storage.Provider { // MountMetadata mounts the configured control metadata location. func (ei *EngineInstance) MountMetadata() error { - ei.log.Debug("checking if metadata is mounted") + msgIdx := fmt.Sprintf("instance %d", ei.Index()) + + msgCheck := fmt.Sprintf("%s: checking if metadata is mounted", msgIdx) + ei.log.Trace(msgCheck) + isMounted, err := ei.storage.ControlMetadataIsMounted() if err != nil { - return errors.Wrap(err, "checking if metadata is mounted") + return errors.WithMessage(err, msgCheck) } - ei.log.Debugf("IsMounted: %v", isMounted) + ei.log.Debugf("%s: IsMounted: %v", msgIdx, isMounted) if isMounted { return nil } @@ -44,10 +51,17 @@ func (ei *EngineInstance) MountMetadata() error { // at the mountpoint specified in the configuration. If the device is already // mounted, the function returns nil, indicating success. func (ei *EngineInstance) MountScm() error { + msgIdx := fmt.Sprintf("instance %d", ei.Index()) + + msgCheck := fmt.Sprintf("%s: checking if scm is mounted", msgIdx) + ei.log.Trace(msgCheck) + isMounted, err := ei.storage.ScmIsMounted() if err != nil && !os.IsNotExist(errors.Cause(err)) { - return errors.WithMessage(err, "failed to check SCM mount") + return errors.WithMessage(err, msgCheck) } + + ei.log.Debugf("%s: IsMounted: %v", msgIdx, isMounted) if isMounted { return nil } @@ -75,58 +89,82 @@ func createPublishFormatRequiredFunc(publish func(*events.RASEvent), hostname st } } +func (ei *EngineInstance) checkScmNeedFormat() (bool, error) { + msgIdx := fmt.Sprintf("instance %d", ei.Index()) + + if ei.storage.ControlMetadataPathConfigured() { + cfg, err := ei.storage.GetScmConfig() + if err != nil { + return false, err + } + if cfg.Class != "ram" { + return false, storage.FaultBdevConfigRolesWithDCPM + } + if !ei.storage.BdevRoleMetaConfigured() { + return false, storage.FaultBdevConfigControlMetadataNoRoles + } + ei.log.Debugf("scm class is ram and bdev role meta configured") + + return true, nil + } + + needsScmFormat, err := ei.storage.ScmNeedsFormat() + if err != nil { + if fault.IsFaultCode(err, code.StorageDeviceWithFsNoMountpoint) { + return false, err + } + ei.log.Errorf("%s: failed to check storage formatting: %s", msgIdx, err) + + return true, nil + } + + return needsScmFormat, nil +} + // awaitStorageReady blocks until instance has storage available and ready to be used. func (ei *EngineInstance) awaitStorageReady(ctx context.Context) error { idx := ei.Index() + msgIdx := fmt.Sprintf("instance %d", idx) if ei.IsStarted() { - return errors.Errorf("can't wait for storage: instance %d already started", idx) + return errors.Errorf("can't wait for storage: %s already started", msgIdx) } - ei.log.Infof("Checking %s instance %d storage ...", build.DataPlaneName, idx) + ei.log.Infof("Checking %s %s storage ...", build.DataPlaneName, msgIdx) needsMetaFormat, err := ei.storage.ControlMetadataNeedsFormat() if err != nil { - ei.log.Errorf("failed to check control metadata storage formatting: %s", err) + ei.log.Errorf("%s: failed to check control metadata storage formatting: %s", + msgIdx, err) needsMetaFormat = true } - ei.log.Debugf("needsMetaFormat: %t", needsMetaFormat) + ei.log.Debugf("%s: needsMetaFormat: %t", msgIdx, needsMetaFormat) - needsScmFormat, err := ei.storage.ScmNeedsFormat() + needsScmFormat, err := ei.checkScmNeedFormat() if err != nil { - ei.log.Errorf("instance %d: failed to check storage formatting: %s", idx, err) - needsScmFormat = true + return err } - ei.log.Debugf("needsScmFormat: %t", needsScmFormat) + ei.log.Debugf("%s: needsScmFormat: %t", msgIdx, needsScmFormat) - if !needsMetaFormat && ei.storage.ControlMetadataPathConfigured() { - cfg, err := ei.storage.GetScmConfig() - if err != nil { - return err - } - if (cfg.Class == "ram") && ei.storage.BdevRoleMetaConfigured() { - ei.log.Debugf("scm class is ram and bdev role meta configured") - err := ei.storage.FormatScm(true) - if err != nil { - ei.log.Errorf("instance %d: failed to setup the scm: %s", idx, err) - } else { - ei.log.Debugf("remounted scm") - needsScmFormat = false - } + // Always reformat ramdisk in MD-on-SSD mode if control metadata intact. + if ei.storage.ControlMetadataPathConfigured() && !needsMetaFormat { + if err := ei.storage.FormatScm(true); err != nil { + return errors.Wrapf(err, "%s: format ramdisk", msgIdx) } + needsScmFormat = false } if !needsMetaFormat && !needsScmFormat { - ei.log.Debugf("instance %d: no SCM format required; checking for superblock", idx) + ei.log.Debugf("%s: no SCM format required; checking for superblock", msgIdx) needsSuperblock, err := ei.needsSuperblock() if err != nil { - ei.log.Errorf("instance %d: failed to check instance superblock: %s", idx, err) + ei.log.Errorf("%s: failed to check instance superblock: %s", msgIdx, err) } if !needsSuperblock { - ei.log.Debugf("instance %d: superblock not needed", idx) + ei.log.Debugf("%s: superblock not needed", msgIdx) return nil } - ei.log.Debugf("instance %d: superblock needed", idx) + ei.log.Debugf("%s: superblock needed", msgIdx) } // by this point we need superblock and possibly scm format @@ -134,7 +172,7 @@ func (ei *EngineInstance) awaitStorageReady(ctx context.Context) error { if !needsScmFormat { formatType = "Metadata" } - ei.log.Infof("%s format required on instance %d", formatType, idx) + ei.log.Infof("%s format required on %s", formatType, msgIdx) ei.waitFormat.SetTrue() // After we know that the instance is awaiting format, fire off @@ -147,9 +185,9 @@ func (ei *EngineInstance) awaitStorageReady(ctx context.Context) error { select { case <-ctx.Done(): - ei.log.Infof("%s instance %d storage not ready: %s", build.DataPlaneName, idx, ctx.Err()) + ei.log.Infof("%s %s storage not ready: %s", build.DataPlaneName, msgIdx, ctx.Err()) case <-ei.storageReady: - ei.log.Infof("%s instance %d storage ready", build.DataPlaneName, idx) + ei.log.Infof("%s %s storage ready", build.DataPlaneName, msgIdx) } ei.waitFormat.SetFalse() diff --git a/src/control/server/instance_storage_test.go b/src/control/server/instance_storage_test.go index 4d3db1f734c..3685ae14d8c 100644 --- a/src/control/server/instance_storage_test.go +++ b/src/control/server/instance_storage_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2023 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -10,6 +10,7 @@ import ( "context" "fmt" "os" + "path/filepath" "sync" "testing" "time" @@ -27,23 +28,30 @@ import ( "github.com/daos-stack/daos/src/control/server/storage/scm" ) -func TestIOEngineInstance_MountControlMetadata(t *testing.T) { - cfg := &storage.Config{ - ControlMetadata: storage.ControlMetadata{ - Path: "/dontcare", - DevicePath: "/dev/dontcare", - }, - Tiers: storage.TierConfigs{ - { - Class: storage.ClassRam, - Scm: storage.ScmConfig{ - MountPoint: defaultStoragePath, - RamdiskSize: 1, - }, +var mockRamCfg = storage.Config{ + ControlMetadata: storage.ControlMetadata{ + Path: "/dontcare", + DevicePath: "/dev/dontcare", + }, + Tiers: storage.TierConfigs{ + { + Class: storage.ClassRam, + Scm: storage.ScmConfig{ + MountPoint: defaultStoragePath, + RamdiskSize: 1, }, }, - } + { + Class: storage.ClassNvme, + Bdev: storage.BdevConfig{ + DeviceList: new(storage.BdevDeviceList), + DeviceRoles: storage.BdevRoles{storage.BdevRoleAll}, + }, + }, + }, +} +func TestIOEngineInstance_MountControlMetadata(t *testing.T) { for name, tc := range map[string]struct { meta *storage.MockMetadataProvider sysCfg *system.MockSysConfig @@ -84,11 +92,12 @@ func TestIOEngineInstance_MountControlMetadata(t *testing.T) { } ec := engine.MockConfig(). - WithStorageControlMetadataPath(cfg.ControlMetadata.Path). - WithStorageControlMetadataDevice(cfg.ControlMetadata.DevicePath) + WithStorageControlMetadataPath(mockRamCfg.ControlMetadata.Path). + WithStorageControlMetadataDevice(mockRamCfg.ControlMetadata.DevicePath) runner := engine.NewRunner(log, ec) sysProv := system.NewMockSysProvider(log, tc.sysCfg) - provider := storage.MockProvider(log, 0, cfg, sysProv, nil, nil, tc.meta) + provider := storage.MockProvider(log, 0, &mockRamCfg, sysProv, nil, nil, + tc.meta) instance := NewEngineInstance(log, provider, nil, runner) gotErr := instance.MountMetadata() @@ -205,6 +214,7 @@ func TestIOEngineInstance_MountScmDevice(t *testing.T) { func TestEngineInstance_NeedsScmFormat(t *testing.T) { const ( + dev = "/dev/foo" goodMountPoint = "/mnt/daos" ) var ( @@ -218,7 +228,7 @@ func TestEngineInstance_NeedsScmFormat(t *testing.T) { storage.NewTierConfig(). WithStorageClass(storage.ClassDcpm.String()). WithScmMountPoint(goodMountPoint). - WithScmDeviceList("/dev/foo"), + WithScmDeviceList(dev), ) ) @@ -284,6 +294,7 @@ func TestEngineInstance_NeedsScmFormat(t *testing.T) { IsMountedErr: os.ErrNotExist, GetfsStr: "ext4", }, + expErr: storage.FaultDeviceWithFsNoMountpoint(dev, goodMountPoint), expNeedsFormat: false, }, "check dcpm fails (IsMounted fails)": { @@ -350,39 +361,122 @@ func (tly *tally) fakePublish(evt *events.RASEvent) { func TestIOEngineInstance_awaitStorageReady(t *testing.T) { errStarted := errors.New("already started") + dev := "/dev/foo" + mnt := "/mnt/test" dcpmCfg := engine.MockConfig().WithStorage( storage.NewTierConfig(). WithStorageClass(storage.ClassDcpm.String()). - WithScmMountPoint("/mnt/test"). - WithScmDeviceList("/dev/foo"), + WithScmMountPoint(mnt). + WithScmDeviceList(dev), ) for name, tc := range map[string]struct { - engineStarted bool - needsScmFormat bool - hasSB bool - engineIndex uint32 - expFmtType string - expErr error + engineStarted bool + storageCfg *storage.Config + fsStr string + fsErr error + sbSet bool + engineIndex uint32 + isMounted bool + isMountedErr error + mountErr error + unmountErr error + readErr error + metaNeedsFmt bool + metaNeedsFmtErr error + expNoWait bool + expFmtType string + expErr error }{ "already started": { engineStarted: true, + fsStr: "ext4", expErr: errStarted, }, - "no need to format and existing superblock": { - hasSB: true, + "no need to format and superblock set": { + sbSet: true, + fsStr: "ext4", + isMounted: true, + expNoWait: true, }, - "needs scm format": { - needsScmFormat: true, - expFmtType: "SCM", + "no need to format and superblock set; mount fails": { + fsStr: "ext4", + mountErr: errors.New("bad mount"), + expFmtType: "Metadata", + }, + "filesystem exists and superblock set; mountpoint missing": { + sbSet: true, + fsStr: "ext4", + isMountedErr: os.ErrNotExist, + expErr: storage.FaultDeviceWithFsNoMountpoint(dev, mnt), + }, + "mount check fails": { + sbSet: true, + fsStr: "ext4", + isMountedErr: errors.New("fail"), + expFmtType: "SCM", + }, + "no need to format: superblock exists but not yet set": { + fsStr: "ext4", + isMounted: true, + expNoWait: true, }, - "needs metadata format": { + "superblock not found; needs metadata format": { + fsStr: "ext4", + readErr: os.ErrNotExist, expFmtType: "Metadata", }, - "engine index 1": { + "needs scm format": { + fsStr: "none", + expFmtType: "SCM", + }, + "engine index 1; needs metadata format": { engineIndex: 1, + fsStr: "ext4", + readErr: os.ErrNotExist, expFmtType: "Metadata", }, + "metadata path with dcpm": { + storageCfg: &storage.Config{ + ControlMetadata: storage.ControlMetadata{ + Path: "/dontcare", + DevicePath: "/dev/dontcare", + }, + Tiers: storage.TierConfigs{ + {Class: storage.ClassDcpm}, + }, + }, + expErr: storage.FaultBdevConfigRolesWithDCPM, + }, + "no device roles with metadata path configured": { + storageCfg: &storage.Config{ + ControlMetadata: storage.ControlMetadata{ + Path: "/dontcare", + DevicePath: "/dev/dontcare", + }, + Tiers: storage.TierConfigs{ + {Class: storage.ClassRam}, + }, + }, + expErr: storage.FaultBdevConfigControlMetadataNoRoles, + }, + "metadata path; no metadata format needed; scm format fails": { + storageCfg: &mockRamCfg, + isMounted: true, + unmountErr: errors.New("ramdisk busy"), + expErr: errors.New("format ramdisk: failed to clear existing mount: failed to unmount"), + }, + "metadata path; no metadata format needed; scm format succeeds; superblock intact": { + storageCfg: &mockRamCfg, + isMounted: true, + expNoWait: true, + }, + "metadata path; no metadata format needed; scm format succeeds; no superblock": { + storageCfg: &mockRamCfg, + isMounted: true, + readErr: os.ErrNotExist, + expFmtType: "Metadata", + }, } { t.Run(name, func(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) @@ -394,22 +488,43 @@ func TestIOEngineInstance_awaitStorageReady(t *testing.T) { } runner := engine.NewTestRunner(trc, dcpmCfg) - fs := "none" - if !tc.needsScmFormat { - fs = "ext4" + if tc.storageCfg == nil { + tc.storageCfg = &dcpmCfg.Storage + } + + msc := system.MockSysConfig{ + IsMountedBool: tc.isMounted, + IsMountedErr: tc.isMountedErr, + MountErr: tc.mountErr, + UnmountErr: tc.unmountErr, + GetfsStr: tc.fsStr, + GetfsErr: tc.fsErr, + } + if tc.readErr != nil { + storagePath := mnt + ctlMD := tc.storageCfg.ControlMetadata + if ctlMD.HasPath() { + storagePath = ctlMD.EngineDirectory(uint(tc.engineIndex)) + } + sbPath := filepath.Join(storagePath, "superblock") + t.Logf("setting readfile err %s for path %s", tc.readErr.Error(), sbPath) + msc.ReadFileErrors = map[string]error{sbPath: tc.readErr} + } + smbc := scm.MockBackendConfig{} + mmp := &storage.MockMetadataProvider{ + NeedsFormatRes: tc.metaNeedsFmt, + NeedsFormatErr: tc.metaNeedsFmtErr, } - msc := system.MockSysConfig{GetfsStr: fs} - mbc := scm.MockBackendConfig{} - mp := storage.NewProvider(log, 0, &dcpmCfg.Storage, + mp := storage.NewProvider(log, 0, tc.storageCfg, system.NewMockSysProvider(log, &msc), - scm.NewMockProvider(log, &mbc, &msc), - nil, nil) + scm.NewMockProvider(log, &smbc, &msc), + nil, mmp) engine := NewEngineInstance(log, mp, nil, runner) engine.setIndex(tc.engineIndex) - if tc.hasSB { + if tc.sbSet { engine.setSuperblock(&Superblock{ Rank: ranklist.NewRankPtr(0), ValidRank: true, }) @@ -425,7 +540,7 @@ func TestIOEngineInstance_awaitStorageReady(t *testing.T) { gotErr := engine.awaitStorageReady(ctx) test.CmpErr(t, tc.expErr, gotErr) - if tc.expErr == errStarted || tc.hasSB == true { + if tc.expErr != nil || tc.expNoWait == true { return } diff --git a/src/control/server/instance_superblock.go b/src/control/server/instance_superblock.go index cf4432969bb..624d2e40613 100644 --- a/src/control/server/instance_superblock.go +++ b/src/control/server/instance_superblock.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -7,7 +7,7 @@ package server import ( - "io/ioutil" + "fmt" "os" "path/filepath" @@ -163,18 +163,26 @@ func (ei *EngineInstance) WriteSuperblock() error { return WriteSuperblock(ei.superblockPath(), ei.getSuperblock()) } -// ReadSuperblock reads the instance's superblock -// from storage. +// ReadSuperblock reads the instance's superblock from storage. func (ei *EngineInstance) ReadSuperblock() error { if err := ei.MountMetadata(); err != nil { return errors.Wrap(err, "failed to mount control metadata device") } - sb, err := ReadSuperblock(ei.superblockPath()) + msgIdx := fmt.Sprintf("instance %d", ei.Index()) + sbPath := ei.superblockPath() + ei.log.Tracef("%s: read sb: %q", msgIdx, sbPath) + + data, err := ei.storage.Sys.ReadFile(sbPath) if err != nil { - ei.log.Debugf("instance %d: failed to read superblock at %s: %s", ei.Index(), ei.superblockPath(), err) - return errors.Wrap(err, "failed to read instance superblock") + return errors.Wrapf(err, "%s: failed to read Superblock from %s", msgIdx, sbPath) + } + + sb := &Superblock{} + if err := sb.Unmarshal(data); err != nil { + return err } + ei.setSuperblock(sb) return nil @@ -198,18 +206,3 @@ func WriteSuperblock(sbPath string, sb *Superblock) error { return errors.Wrapf(common.WriteFileAtomic(sbPath, data, 0600), "Failed to write Superblock to %s", sbPath) } - -// ReadSuperblock reads a Superblock from storage. -func ReadSuperblock(sbPath string) (*Superblock, error) { - data, err := ioutil.ReadFile(sbPath) - if err != nil { - return nil, errors.Wrapf(err, "Failed to read Superblock from %s", sbPath) - } - - sb := &Superblock{} - if err := sb.Unmarshal(data); err != nil { - return nil, err - } - - return sb, nil -} diff --git a/src/control/server/instance_superblock_test.go b/src/control/server/instance_superblock_test.go index c01196e1a69..4dee3eceefb 100644 --- a/src/control/server/instance_superblock_test.go +++ b/src/control/server/instance_superblock_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2023 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -44,9 +44,12 @@ func TestServer_Instance_createSuperblock(t *testing.T) { r := engine.NewRunner(log, cfg) msc := &sysprov.MockSysConfig{ IsMountedBool: true, + RealReadFile: true, } mbc := &scm.MockBackendConfig{} - mp := storage.NewProvider(log, 0, &cfg.Storage, sysprov.NewMockSysProvider(log, msc), scm.NewMockProvider(log, mbc, msc), nil, nil) + mp := storage.NewProvider(log, 0, &cfg.Storage, + sysprov.NewMockSysProvider(log, msc), + scm.NewMockProvider(log, mbc, msc), nil, nil) ei := NewEngineInstance(log, mp, nil, r). WithHostFaultDomain(system.MustCreateFaultDomainFromString("/host1")) ei.fsRoot = testDir diff --git a/src/control/server/instance_test.go b/src/control/server/instance_test.go index e4ec0ad84fb..d4eb86cce75 100644 --- a/src/control/server/instance_test.go +++ b/src/control/server/instance_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -25,8 +25,10 @@ import ( "github.com/daos-stack/daos/src/control/lib/atm" "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/logging" + sysprov "github.com/daos-stack/daos/src/control/provider/system" "github.com/daos-stack/daos/src/control/server/engine" "github.com/daos-stack/daos/src/control/server/storage" + "github.com/daos-stack/daos/src/control/server/storage/scm" "github.com/daos-stack/daos/src/control/system" ) @@ -113,26 +115,40 @@ func TestServer_Instance_updateFaultDomainInSuperblock(t *testing.T) { testDir, cleanupDir := test.CreateTestDir(t) defer cleanupDir() - inst := getTestEngineInstance(log).WithHostFaultDomain(tc.newDomain) - inst.fsRoot = testDir - inst._superblock = tc.superblock - - sbPath := inst.superblockPath() + // Use real os.ReadFile in MockSysProvider to test superblock logic. + cfg := engine.MockConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("ram"). + WithScmMountPoint("/foo/bar"), + ) + runner := engine.NewRunner(log, cfg) + sysCfg := sysprov.MockSysConfig{RealReadFile: true} + sysProv := sysprov.NewMockSysProvider(log, &sysCfg) + scmProv := scm.NewMockProvider(log, &scm.MockBackendConfig{}, &sysCfg) + storage := storage.MockProvider(log, 0, &cfg.Storage, sysProv, scmProv, + nil, nil) + + ei := NewEngineInstance(log, storage, nil, runner). + WithHostFaultDomain(tc.newDomain) + ei.fsRoot = testDir + ei._superblock = tc.superblock + + sbPath := ei.superblockPath() if err := os.MkdirAll(filepath.Dir(sbPath), 0755); err != nil { t.Fatalf("failed to make test superblock dir: %s", err.Error()) } - err := inst.updateFaultDomainInSuperblock() - + err := ei.updateFaultDomainInSuperblock() test.CmpErr(t, tc.expErr, err) // Ensure the newer value in the instance was written to the superblock - newSB, err := ReadSuperblock(sbPath) + err = ei.ReadSuperblock() if tc.expWritten { if err != nil { t.Fatalf("can't read expected superblock: %s", err.Error()) } + newSB := ei.getSuperblock() if newSB == nil { t.Fatalf("expected non-nil superblock") } diff --git a/src/control/server/mgmt_check_test.go b/src/control/server/mgmt_check_test.go index b59daad699e..16f25cc3e69 100644 --- a/src/control/server/mgmt_check_test.go +++ b/src/control/server/mgmt_check_test.go @@ -131,7 +131,7 @@ func TestServer_mgmtSvc_SystemCheckStart(t *testing.T) { for name, tc := range map[string]struct { createMS func(*testing.T, logging.Logger) *mgmtSvc - setupDrpc func(*testing.T, *mgmtSvc) + getMockDrpc func() *mockDrpcClient req *mgmtpb.CheckStartReq expResp *mgmtpb.CheckStartResp expErr error @@ -170,8 +170,8 @@ func TestServer_mgmtSvc_SystemCheckStart(t *testing.T) { expErr: errors.New("unmarshal checker policies"), }, "dRPC fails": { - setupDrpc: func(t *testing.T, ms *mgmtSvc) { - setupMockDrpcClient(ms, nil, errors.New("mock dRPC")) + getMockDrpc: func() *mockDrpcClient { + return getMockDrpcClient(nil, errors.New("mock dRPC")) }, req: &mgmtpb.CheckStartReq{ Sys: "daos_server", @@ -181,8 +181,8 @@ func TestServer_mgmtSvc_SystemCheckStart(t *testing.T) { expPolicies: testPolicies, }, "bad resp": { - setupDrpc: func(t *testing.T, ms *mgmtSvc) { - setupMockDrpcClientBytes(ms, []byte("garbage"), nil) + getMockDrpc: func() *mockDrpcClient { + return getMockDrpcClientBytes([]byte("garbage"), nil) }, req: &mgmtpb.CheckStartReq{ Sys: "daos_server", @@ -192,8 +192,8 @@ func TestServer_mgmtSvc_SystemCheckStart(t *testing.T) { expPolicies: testPolicies, }, "request failed": { - setupDrpc: func(t *testing.T, ms *mgmtSvc) { - setupMockDrpcClient(ms, &mgmt.CheckStartResp{Status: int32(daos.MiscError)}, nil) + getMockDrpc: func() *mockDrpcClient { + return getMockDrpcClient(&mgmt.CheckStartResp{Status: int32(daos.MiscError)}, nil) }, req: &mgmtpb.CheckStartReq{ Sys: "daos_server", @@ -215,9 +215,9 @@ func TestServer_mgmtSvc_SystemCheckStart(t *testing.T) { Sys: "daos_server", Flags: uint32(chkpb.CheckFlag_CF_RESET), }, - setupDrpc: func(t *testing.T, ms *mgmtSvc) { + getMockDrpc: func() *mockDrpcClient { // engine returns status > 0 to indicate reset - setupMockDrpcClient(ms, &mgmt.CheckStartResp{Status: 1}, nil) + return getMockDrpcClient(&mgmt.CheckStartResp{Status: 1}, nil) }, expResp: &mgmtpb.CheckStartResp{}, expFindings: []*checker.Finding{}, @@ -229,9 +229,9 @@ func TestServer_mgmtSvc_SystemCheckStart(t *testing.T) { Flags: uint32(chkpb.CheckFlag_CF_RESET), Uuids: []string{uuids[0], uuids[2]}, }, - setupDrpc: func(t *testing.T, ms *mgmtSvc) { + getMockDrpc: func() *mockDrpcClient { // engine returns status > 0 to indicate reset - setupMockDrpcClient(ms, &mgmt.CheckStartResp{Status: 1}, nil) + return getMockDrpcClient(&mgmt.CheckStartResp{Status: 1}, nil) }, expResp: &mgmtpb.CheckStartResp{}, expFindings: []*checker.Finding{ @@ -283,12 +283,13 @@ func TestServer_mgmtSvc_SystemCheckStart(t *testing.T) { } svc := tc.createMS(t, log) - if tc.setupDrpc == nil { - tc.setupDrpc = func(t *testing.T, ms *mgmtSvc) { - setupMockDrpcClient(ms, &mgmtpb.CheckStartResp{}, nil) + if tc.getMockDrpc == nil { + tc.getMockDrpc = func() *mockDrpcClient { + return getMockDrpcClient(&mgmtpb.CheckStartResp{}, nil) } } - tc.setupDrpc(t, svc) + mockDrpc := tc.getMockDrpc() + setupSvcDrpcClient(svc, 0, mockDrpc) resp, err := svc.SystemCheckStart(test.Context(t), tc.req) @@ -310,15 +311,6 @@ func TestServer_mgmtSvc_SystemCheckStart(t *testing.T) { } // Check contents of drpc payload - ei, ok := svc.harness.instances[0].(*EngineInstance) - if !ok { - t.Fatalf("bad engine instance type %T", svc.harness.instances[0]) - } - mockDrpc, ok := ei._drpcClient.(*mockDrpcClient) - if !ok { - t.Fatalf("bad drpc client type type %T", ei._drpcClient) - } - drpcInput := new(mgmtpb.CheckStartReq) calls := mockDrpc.calls.get() if len(calls) == 0 { diff --git a/src/control/server/mgmt_cont_test.go b/src/control/server/mgmt_cont_test.go index f4096d7fd14..5542fe40c52 100644 --- a/src/control/server/mgmt_cont_test.go +++ b/src/control/server/mgmt_cont_test.go @@ -92,7 +92,7 @@ func TestMgmt_ListContainers(t *testing.T) { }, "drpc error": { setupDrpc: func(t *testing.T, svc *mgmtSvc) { - setupMockDrpcClient(svc, nil, errors.New("mock drpc")) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(nil, errors.New("mock drpc"))) }, req: validListContReq(), expErr: errors.New("mock drpc"), @@ -100,23 +100,24 @@ func TestMgmt_ListContainers(t *testing.T) { "bad drpc resp": { setupDrpc: func(t *testing.T, svc *mgmtSvc) { badBytes := makeBadBytes(16) - setupMockDrpcClientBytes(svc, badBytes, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, nil)) }, req: validListContReq(), expErr: errors.New("unmarshal"), }, "success; zero containers": { setupDrpc: func(t *testing.T, svc *mgmtSvc) { - setupMockDrpcClient(svc, &mgmtpb.ListContResp{}, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(&mgmtpb.ListContResp{}, nil)) }, req: validListContReq(), expResp: &mgmtpb.ListContResp{}, }, "success; multiple containers": { setupDrpc: func(t *testing.T, svc *mgmtSvc) { - setupMockDrpcClient(svc, &mgmtpb.ListContResp{ - Containers: multiConts, - }, nil) + setupSvcDrpcClient(svc, 0, + getMockDrpcClient(&mgmtpb.ListContResp{ + Containers: multiConts, + }, nil)) }, req: validListContReq(), expResp: &mgmtpb.ListContResp{ @@ -191,7 +192,7 @@ func TestMgmt_ContSetOwner(t *testing.T) { }, "drpc error": { setupDrpc: func(t *testing.T, svc *mgmtSvc) { - setupMockDrpcClient(svc, nil, errors.New("mock drpc")) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(nil, errors.New("mock drpc"))) }, req: validContSetOwnerReq(), expErr: errors.New("mock drpc"), @@ -199,14 +200,14 @@ func TestMgmt_ContSetOwner(t *testing.T) { "bad drpc resp": { setupDrpc: func(t *testing.T, svc *mgmtSvc) { badBytes := makeBadBytes(16) - setupMockDrpcClientBytes(svc, badBytes, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, nil)) }, req: validContSetOwnerReq(), expErr: errors.New("unmarshal"), }, "success": { setupDrpc: func(t *testing.T, svc *mgmtSvc) { - setupMockDrpcClient(svc, &mgmtpb.ContSetOwnerResp{}, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(&mgmtpb.ContSetOwnerResp{}, nil)) }, req: validContSetOwnerReq(), expResp: &mgmtpb.ContSetOwnerResp{ diff --git a/src/control/server/mgmt_pool.go b/src/control/server/mgmt_pool.go index 3ee573d14f2..540566be5be 100644 --- a/src/control/server/mgmt_pool.go +++ b/src/control/server/mgmt_pool.go @@ -299,7 +299,7 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq return nil, errors.Wrap(err, "query on already-created pool failed") } - resp.SvcLdr = qr.Leader + resp.SvcLdr = qr.SvcLdr resp.SvcReps = ranklist.RanksToUint32(ps.Replicas) resp.TgtRanks = ranklist.RanksToUint32(ps.Storage.CreationRanks()) resp.TierBytes = ps.Storage.PerRankTierStorage @@ -929,6 +929,10 @@ func (svc *mgmtSvc) PoolQuery(ctx context.Context, req *mgmtpb.PoolQueryReq) (*m return nil, err } + if req.QueryMask == 0 { + req.QueryMask = uint64(daos.DefaultPoolQueryMask) + } + dresp, err := svc.makePoolServiceCall(ctx, drpc.MethodPoolQuery, req) if err != nil { return nil, err @@ -939,6 +943,9 @@ func (svc *mgmtSvc) PoolQuery(ctx context.Context, req *mgmtpb.PoolQueryReq) (*m return nil, errors.Wrap(err, "unmarshal PoolQuery response") } + // Preserve compatibility with pre-2.6 callers. + resp.Leader = resp.SvcLdr + return resp, nil } diff --git a/src/control/server/mgmt_pool_test.go b/src/control/server/mgmt_pool_test.go index 25293a085ba..a85b990c4c5 100644 --- a/src/control/server/mgmt_pool_test.go +++ b/src/control/server/mgmt_pool_test.go @@ -117,7 +117,7 @@ func TestServer_MgmtSvc_PoolCreateAlreadyExists(t *testing.T) { "ready": { state: system.PoolServiceStateReady, queryResp: &mgmtpb.PoolQueryResp{ - Leader: 1, + SvcLdr: 1, }, expResp: &mgmtpb.PoolCreateResp{ SvcLdr: 1, @@ -143,7 +143,8 @@ func TestServer_MgmtSvc_PoolCreateAlreadyExists(t *testing.T) { defer test.ShowBufferOnFailure(t, buf) svc := newTestMgmtSvc(t, log) - setupMockDrpcClient(svc, tc.queryResp, tc.queryErr) + mdc := getMockDrpcClient(tc.queryResp, tc.queryErr) + setupSvcDrpcClient(svc, 0, mdc) if _, err := svc.membership.Add(system.MockMember(t, 1, system.MemberStateJoined)); err != nil { t.Fatal(err) } @@ -404,7 +405,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(42) - setupMockDrpcClientBytes(svc, badBytes, err) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, err)) }, expErr: errors.New("unmarshal"), }, @@ -569,6 +570,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { mp := storage.NewProvider(log, 0, &engineCfg.Storage, nil, nil, nil, nil) srv := NewEngineInstance(log, mp, nil, r) + srv.setDrpcSocket("/dontcare") srv.ready.SetTrue() harness := NewEngineHarness(log) @@ -597,7 +599,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { if tc.setupMockDrpc == nil { tc.setupMockDrpc = func(svc *mgmtSvc, err error) { - setupMockDrpcClient(tc.mgmtSvc, tc.drpcRet, tc.expErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(tc.drpcRet, tc.expErr)) } } tc.setupMockDrpc(tc.mgmtSvc, tc.expErr) @@ -641,7 +643,7 @@ func TestServer_MgmtSvc_PoolCreateDownRanks(t *testing.T) { dc := newMockDrpcClient(&mockDrpcClientConfig{IsConnectedBool: true}) dc.cfg.setSendMsgResponse(drpc.Status_SUCCESS, nil, nil) - mgmtSvc.harness.instances[0].(*EngineInstance)._drpcClient = dc + mgmtSvc.harness.instances[0].(*EngineInstance).getDrpcClientFn = func(s string) drpc.DomainSocketClient { return dc } for _, m := range []*system.Member{ system.MockMember(t, 0, system.MemberStateJoined), @@ -1209,8 +1211,8 @@ func TestServer_MgmtSvc_PoolDestroy(t *testing.T) { cfg.setSendMsgResponseList(t, mock) } } - ei := mgmtSvc.harness.instances[0].(*EngineInstance) - ei.setDrpcClient(newMockDrpcClient(cfg)) + mdc := newMockDrpcClient(cfg) + setupSvcDrpcClient(mgmtSvc, 0, mdc) if tc.req != nil && tc.req.Sys == "" { tc.req.Sys = build.DefaultSystemName @@ -1227,7 +1229,7 @@ func TestServer_MgmtSvc_PoolDestroy(t *testing.T) { if tc.expDrpcReq != nil { gotReq := new(mgmtpb.PoolDestroyReq) - if err := proto.Unmarshal(getLastMockCall(mgmtSvc).Body, gotReq); err != nil { + if err := proto.Unmarshal(getLastMockCall(mdc).Body, gotReq); err != nil { t.Fatal(err) } if diff := cmp.Diff(tc.expDrpcReq, gotReq, cmpOpts...); diff != "" { @@ -1236,7 +1238,7 @@ func TestServer_MgmtSvc_PoolDestroy(t *testing.T) { } if tc.expDrpcEvReq != nil { gotReq := new(mgmtpb.PoolEvictReq) - if err := proto.Unmarshal(getLastMockCall(mgmtSvc).Body, gotReq); err != nil { + if err := proto.Unmarshal(getLastMockCall(mdc).Body, gotReq); err != nil { t.Fatal(err) } if diff := cmp.Diff(tc.expDrpcEvReq, gotReq, cmpOpts...); diff != "" { @@ -1245,7 +1247,7 @@ func TestServer_MgmtSvc_PoolDestroy(t *testing.T) { } if tc.expDrpcListContReq != nil { gotReq := new(mgmtpb.ListContReq) - if err := proto.Unmarshal(getLastMockCall(mgmtSvc).Body, gotReq); err != nil { + if err := proto.Unmarshal(getLastMockCall(mdc).Body, gotReq); err != nil { t.Fatal(err) } if diff := cmp.Diff(tc.expDrpcListContReq, gotReq, cmpOpts...); diff != "" { @@ -1331,7 +1333,7 @@ func TestServer_MgmtSvc_PoolExtend(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(42) - setupMockDrpcClientBytes(svc, badBytes, err) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, err)) }, expErr: errors.New("unmarshal"), }, @@ -1357,7 +1359,7 @@ func TestServer_MgmtSvc_PoolExtend(t *testing.T) { if tc.setupMockDrpc == nil { tc.setupMockDrpc = func(svc *mgmtSvc, err error) { - setupMockDrpcClient(tc.mgmtSvc, tc.expResp, tc.expErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(tc.expResp, tc.expErr)) } } tc.setupMockDrpc(tc.mgmtSvc, tc.expErr) @@ -1433,7 +1435,7 @@ func TestServer_MgmtSvc_PoolDrain(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(42) - setupMockDrpcClientBytes(svc, badBytes, err) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, err)) }, expErr: errors.New("unmarshal"), }, @@ -1457,7 +1459,7 @@ func TestServer_MgmtSvc_PoolDrain(t *testing.T) { if tc.setupMockDrpc == nil { tc.setupMockDrpc = func(svc *mgmtSvc, err error) { - setupMockDrpcClient(tc.mgmtSvc, tc.expResp, tc.expErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(tc.expResp, tc.expErr)) } } tc.setupMockDrpc(tc.mgmtSvc, tc.expErr) @@ -1525,7 +1527,7 @@ func TestServer_MgmtSvc_PoolEvict(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(42) - setupMockDrpcClientBytes(svc, badBytes, err) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, err)) }, expErr: errors.New("unmarshal"), }, @@ -1549,7 +1551,7 @@ func TestServer_MgmtSvc_PoolEvict(t *testing.T) { if tc.setupMockDrpc == nil { tc.setupMockDrpc = func(svc *mgmtSvc, err error) { - setupMockDrpcClient(tc.mgmtSvc, tc.expResp, tc.expErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(tc.expResp, tc.expErr)) } } tc.setupMockDrpc(tc.mgmtSvc, tc.expErr) @@ -1684,7 +1686,7 @@ func TestPoolGetACL_Success(t *testing.T) { Entries: []string{"A::OWNER@:rw", "A:g:GROUP@:r"}, }, } - setupMockDrpcClient(svc, expectedResp, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(expectedResp, nil)) resp, err := svc.PoolGetACL(test.Context(t), newTestGetACLReq()) @@ -1705,7 +1707,7 @@ func TestPoolGetACL_DrpcFailed(t *testing.T) { svc := newTestMgmtSvc(t, log) addTestPools(t, svc.sysdb, mockUUID) expectedErr := errors.New("mock error") - setupMockDrpcClient(svc, nil, expectedErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(nil, expectedErr)) resp, err := svc.PoolGetACL(test.Context(t), newTestGetACLReq()) @@ -1725,7 +1727,7 @@ func TestPoolGetACL_BadDrpcResp(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(12) - setupMockDrpcClientBytes(svc, badBytes, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, nil)) resp, err := svc.PoolGetACL(test.Context(t), newTestGetACLReq()) @@ -1768,7 +1770,7 @@ func TestPoolOverwriteACL_DrpcFailed(t *testing.T) { svc := newTestMgmtSvc(t, log) addTestPools(t, svc.sysdb, mockUUID) expectedErr := errors.New("mock error") - setupMockDrpcClient(svc, nil, expectedErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(nil, expectedErr)) resp, err := svc.PoolOverwriteACL(test.Context(t), newTestModifyACLReq()) @@ -1788,7 +1790,7 @@ func TestPoolOverwriteACL_BadDrpcResp(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(16) - setupMockDrpcClientBytes(svc, badBytes, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, nil)) resp, err := svc.PoolOverwriteACL(test.Context(t), newTestModifyACLReq()) @@ -1812,7 +1814,7 @@ func TestPoolOverwriteACL_Success(t *testing.T) { Entries: []string{"A::OWNER@:rw", "A:g:GROUP@:r"}, }, } - setupMockDrpcClient(svc, expectedResp, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(expectedResp, nil)) resp, err := svc.PoolOverwriteACL(test.Context(t), newTestModifyACLReq()) @@ -1848,7 +1850,7 @@ func TestPoolUpdateACL_DrpcFailed(t *testing.T) { svc := newTestMgmtSvc(t, log) addTestPools(t, svc.sysdb, mockUUID) expectedErr := errors.New("mock error") - setupMockDrpcClient(svc, nil, expectedErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(nil, expectedErr)) resp, err := svc.PoolUpdateACL(test.Context(t), newTestModifyACLReq()) @@ -1868,7 +1870,7 @@ func TestPoolUpdateACL_BadDrpcResp(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(16) - setupMockDrpcClientBytes(svc, badBytes, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, nil)) resp, err := svc.PoolUpdateACL(test.Context(t), newTestModifyACLReq()) @@ -1892,7 +1894,7 @@ func TestPoolUpdateACL_Success(t *testing.T) { Entries: []string{"A::OWNER@:rw", "A:g:GROUP@:r"}, }, } - setupMockDrpcClient(svc, expectedResp, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(expectedResp, nil)) resp, err := svc.PoolUpdateACL(test.Context(t), newTestModifyACLReq()) @@ -1936,7 +1938,7 @@ func TestPoolDeleteACL_DrpcFailed(t *testing.T) { svc := newTestMgmtSvc(t, log) addTestPools(t, svc.sysdb, mockUUID) expectedErr := errors.New("mock error") - setupMockDrpcClient(svc, nil, expectedErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(nil, expectedErr)) resp, err := svc.PoolDeleteACL(test.Context(t), newTestDeleteACLReq()) @@ -1956,7 +1958,7 @@ func TestPoolDeleteACL_BadDrpcResp(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(16) - setupMockDrpcClientBytes(svc, badBytes, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, nil)) resp, err := svc.PoolDeleteACL(test.Context(t), newTestDeleteACLReq()) @@ -1980,7 +1982,7 @@ func TestPoolDeleteACL_Success(t *testing.T) { Entries: []string{"A::OWNER@:rw", "A:G:readers@:r"}, }, } - setupMockDrpcClient(svc, expectedResp, nil) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(expectedResp, nil)) resp, err := svc.PoolDeleteACL(test.Context(t), newTestDeleteACLReq()) @@ -2046,7 +2048,7 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(42) - setupMockDrpcClientBytes(svc, badBytes, err) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, err)) }, expErr: errors.New("unmarshal"), }, @@ -2066,6 +2068,25 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) { Uuid: mockUUID, }, }, + "successful query (includes pre-2.6 Leader field)": { + req: &mgmtpb.PoolQueryReq{ + Id: mockUUID, + }, + setupMockDrpc: func(svc *mgmtSvc, err error) { + resp := &mgmtpb.PoolQueryResp{ + State: mgmtpb.PoolServiceState_Ready, + Uuid: mockUUID, + SvcLdr: 42, + } + setupMockDrpcClient(svc, resp, nil) + }, + expResp: &mgmtpb.PoolQueryResp{ + State: mgmtpb.PoolServiceState_Ready, + Uuid: mockUUID, + SvcLdr: 42, + Leader: 42, + }, + }, } { t.Run(name, func(t *testing.T) { buf.Reset() @@ -2078,7 +2099,7 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) { if tc.setupMockDrpc == nil { tc.setupMockDrpc = func(svc *mgmtSvc, err error) { - setupMockDrpcClient(tc.mgmtSvc, tc.expResp, tc.expErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(tc.expResp, tc.expErr)) } } tc.setupMockDrpc(tc.mgmtSvc, tc.expErr) @@ -2101,22 +2122,17 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) { } } -func getLastMockCall(svc *mgmtSvc) *drpc.Call { - mi := svc.harness.instances[0].(*EngineInstance) - if mi == nil || mi._drpcClient == nil { - return nil - } - - return mi._drpcClient.(*mockDrpcClient).SendMsgInputCall +func getLastMockCall(mdc *mockDrpcClient) *drpc.Call { + return mdc.SendMsgInputCall } func TestServer_MgmtSvc_PoolSetProp(t *testing.T) { for name, tc := range map[string]struct { - setupMockDrpc func(_ *mgmtSvc, _ error) - drpcResp *mgmtpb.PoolSetPropResp - req *mgmtpb.PoolSetPropReq - expDrpcReq *mgmtpb.PoolSetPropReq - expErr error + getMockDrpc func(error) *mockDrpcClient + drpcResp *mgmtpb.PoolSetPropResp + req *mgmtpb.PoolSetPropReq + expDrpcReq *mgmtpb.PoolSetPropReq + expErr error }{ "wrong system": { req: &mgmtpb.PoolSetPropReq{Id: mockUUID, Sys: "bad"}, @@ -2132,11 +2148,11 @@ func TestServer_MgmtSvc_PoolSetProp(t *testing.T) { }, }, }, - setupMockDrpc: func(svc *mgmtSvc, err error) { + getMockDrpc: func(err error) *mockDrpcClient { // dRPC call returns junk in the message body badBytes := makeBadBytes(42) - setupMockDrpcClientBytes(svc, badBytes, err) + return getMockDrpcClientBytes(badBytes, err) }, expErr: errors.New("unmarshal"), }, @@ -2221,12 +2237,14 @@ func TestServer_MgmtSvc_PoolSetProp(t *testing.T) { if tc.req.Id != mockUUID { addTestPools(t, ms.sysdb, tc.req.Id) } - if tc.setupMockDrpc == nil { - tc.setupMockDrpc = func(svc *mgmtSvc, err error) { - setupMockDrpcClient(svc, tc.drpcResp, tc.expErr) + + if tc.getMockDrpc == nil { + tc.getMockDrpc = func(err error) *mockDrpcClient { + return getMockDrpcClient(tc.drpcResp, err) } } - tc.setupMockDrpc(ms, tc.expErr) + mdc := tc.getMockDrpc(tc.expErr) + setupSvcDrpcClient(ms, 0, mdc) if tc.req != nil && tc.req.Sys == "" { tc.req.Sys = build.DefaultSystemName @@ -2238,7 +2256,7 @@ func TestServer_MgmtSvc_PoolSetProp(t *testing.T) { } lastReq := new(mgmtpb.PoolSetPropReq) - if err := proto.Unmarshal(getLastMockCall(ms).Body, lastReq); err != nil { + if err := proto.Unmarshal(getLastMockCall(mdc).Body, lastReq); err != nil { t.Fatal(err) } if diff := cmp.Diff(tc.expDrpcReq, lastReq, test.DefaultCmpOpts()...); diff != "" { @@ -2273,7 +2291,7 @@ func TestServer_MgmtSvc_PoolGetProp(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(42) - setupMockDrpcClientBytes(svc, badBytes, err) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, err)) }, expErr: errors.New("unmarshal"), }, @@ -2306,7 +2324,7 @@ func TestServer_MgmtSvc_PoolGetProp(t *testing.T) { } if tc.setupMockDrpc == nil { tc.setupMockDrpc = func(svc *mgmtSvc, err error) { - setupMockDrpcClient(svc, tc.drpcResp, tc.expErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(tc.drpcResp, tc.expErr)) } } tc.setupMockDrpc(ms, tc.expErr) @@ -2368,7 +2386,7 @@ func TestServer_MgmtSvc_PoolUpgrade(t *testing.T) { // dRPC call returns junk in the message body badBytes := makeBadBytes(42) - setupMockDrpcClientBytes(svc, badBytes, err) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(badBytes, err)) }, expErr: errors.New("unmarshal"), }, @@ -2392,7 +2410,7 @@ func TestServer_MgmtSvc_PoolUpgrade(t *testing.T) { if tc.setupMockDrpc == nil { tc.setupMockDrpc = func(svc *mgmtSvc, err error) { - setupMockDrpcClient(tc.mgmtSvc, tc.expResp, tc.expErr) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(tc.expResp, tc.expErr)) } } tc.setupMockDrpc(tc.mgmtSvc, tc.expErr) diff --git a/src/control/server/mgmt_system_test.go b/src/control/server/mgmt_system_test.go index 8632fb45ffe..300e5f7ff38 100644 --- a/src/control/server/mgmt_system_test.go +++ b/src/control/server/mgmt_system_test.go @@ -198,7 +198,10 @@ func TestServer_MgmtSvc_GetAttachInfo(t *testing.T) { if err := harness.AddInstance(srv); err != nil { t.Fatal(err) } - srv.setDrpcClient(newMockDrpcClient(nil)) + + srv.getDrpcClientFn = func(s string) drpc.DomainSocketClient { + return newMockDrpcClient(nil) + } harness.started.SetTrue() db := raft.MockDatabaseWithAddr(t, log, msReplica.Addr) @@ -2153,7 +2156,8 @@ func TestServer_MgmtSvc_Join(t *testing.T) { } peerCtx := peer.NewContext(test.Context(t), &peer.Peer{Addr: peerAddr}) - setupMockDrpcClient(svc, tc.guResp, nil) + mdc := getMockDrpcClient(tc.guResp, nil) + setupSvcDrpcClient(svc, 0, mdc) gotResp, gotErr := svc.Join(peerCtx, tc.req) test.CmpErr(t, tc.expErr, gotErr) @@ -2169,8 +2173,6 @@ func TestServer_MgmtSvc_Join(t *testing.T) { return } - ei := svc.harness.instances[0].(*EngineInstance) - mdc := ei._drpcClient.(*mockDrpcClient) gotGuReq := new(mgmtpb.GroupUpdateReq) calls := mdc.calls.get() // wait for GroupUpdate @@ -2301,7 +2303,7 @@ func TestServer_MgmtSvc_doGroupUpdate(t *testing.T) { } svc := tc.getSvc(t, log) mockDrpc := getMockDrpcClient(tc.drpcResp, tc.drpcErr) - setMockDrpcClient(svc, mockDrpc) + setupSvcDrpcClient(svc, 0, mockDrpc) err := svc.doGroupUpdate(test.Context(t), tc.force) diff --git a/src/control/server/storage/config.go b/src/control/server/storage/config.go index 23ab0305284..180b9663b31 100644 --- a/src/control/server/storage/config.go +++ b/src/control/server/storage/config.go @@ -587,17 +587,17 @@ func (sc *ScmConfig) Validate(class Class) error { switch class { case ClassDcpm: if sc.RamdiskSize > 0 { - return errors.New("scm_size may not be set when scm_class is dcpm") + return errors.New("scm_size may not be set when class is dcpm") } if len(sc.DeviceList) == 0 { - return errors.New("scm_list must be set when scm_class is dcpm") + return errors.New("scm_list must be set when class is dcpm") } if sc.DisableHugepages { - return errors.New("scm_hugepages_disabled may not be set when scm_class is dcpm") + return errors.New("scm_hugepages_disabled may not be set when class is dcpm") } case ClassRam: if len(sc.DeviceList) > 0 { - return errors.New("scm_list may not be set when scm_class is ram") + return errors.New("scm_list may not be set when class is ram") } // Note: RAM-disk size can be auto-sized so allow if zero. if sc.RamdiskSize != 0 { @@ -954,8 +954,7 @@ type BdevConfig struct { func (bc *BdevConfig) checkNonZeroDevFileSize(class Class) error { if bc.FileSize == 0 { - return errors.Errorf("bdev_class %s requires non-zero bdev_size", - class) + return errors.Errorf("class %s requires non-zero bdev_size", class) } return nil @@ -963,8 +962,7 @@ func (bc *BdevConfig) checkNonZeroDevFileSize(class Class) error { func (bc *BdevConfig) checkNonEmptyDevList(class Class) error { if bc.DeviceList == nil || bc.DeviceList.Len() == 0 { - return errors.Errorf("bdev_class %s requires non-empty bdev_list", - class) + return errors.Errorf("class %s requires non-empty bdev_list", class) } return nil @@ -991,10 +989,10 @@ func (bc *BdevConfig) Validate(class Class) error { case ClassNvme: // NB: We are specifically checking that the embedded PCIAddressSet is non-empty. if bc.DeviceList == nil || bc.DeviceList.PCIAddressSet.Len() == 0 { - return errors.New("bdev_class nvme requires valid PCI addresses in bdev_list") + return errors.New("class nvme requires valid PCI addresses in bdev_list") } default: - return errors.Errorf("bdev_class value %q not supported (valid: nvme/kdev/file)", class) + return errors.Errorf("class value %q not supported (valid: nvme/kdev/file)", class) } return nil diff --git a/src/control/server/storage/faults.go b/src/control/server/storage/faults.go index 1bb42c20d2b..12e1971488b 100644 --- a/src/control/server/storage/faults.go +++ b/src/control/server/storage/faults.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2022-2023 Intel Corporation. +// (C) Copyright 2022-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -102,7 +102,24 @@ func FaultConfigRamdiskUnderMinMem(confSize, memRamdiskMin uint64) *fault.Fault ) } +// FaultDeviceWithFsNoMountpoint creates a Fault for the case where a mount device is missing +// respective target location. +func FaultDeviceWithFsNoMountpoint(dev, tgt string) *fault.Fault { + return storageFault( + code.StorageDeviceWithFsNoMountpoint, + fmt.Sprintf("filesystem exists on device %s but mount-point path %s does not exist", + dev, tgt), + "check the mount-point path exists and if not create it before trying again", + ) +} + var ( + // FaultTargetAlreadyMounted represents an error where the target was already mounted. + FaultTargetAlreadyMounted = storageFault( + code.StorageTargetAlreadyMounted, + "request included already-mounted mount target (cannot double-mount)", + "unmount the target and retry the operation") + // FaultScmNoPMem represents an error where no PMem modules exist. FaultScmNoPMem = storageFault( code.ScmNoPMem, @@ -236,12 +253,6 @@ var ( code.BdevNoIOMMU, "IOMMU capability is required to access NVMe devices but no IOMMU capability detected", "enable IOMMU per the DAOS Admin Guide") - - // FaultTargetAlreadyMounted represents an error where the target was already mounted. - FaultTargetAlreadyMounted = storageFault( - code.StorageTargetAlreadyMounted, - "request included already-mounted mount target (cannot double-mount)", - "unmount the target and retry the operation") ) // FaultPathAccessDenied represents an error where a mount point or device path for diff --git a/src/control/server/storage/metadata/provider.go b/src/control/server/storage/metadata/provider.go index e060604e529..ba3b3110ee8 100644 --- a/src/control/server/storage/metadata/provider.go +++ b/src/control/server/storage/metadata/provider.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2022-2023 Intel Corporation. +// (C) Copyright 2022-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -11,32 +11,35 @@ import ( "path/filepath" "strings" + "github.com/pkg/errors" + "github.com/daos-stack/daos/src/control/logging" "github.com/daos-stack/daos/src/control/provider/system" "github.com/daos-stack/daos/src/control/server/storage" "github.com/daos-stack/daos/src/control/server/storage/mount" - "github.com/pkg/errors" ) -// SystemProvider provides operating system capabilities. -type SystemProvider interface { - system.IsMountedProvider - system.MountProvider - Mkfs(req system.MkfsReq) error - Getfs(device string) (string, error) - Chown(string, int, int) error - Stat(string) (os.FileInfo, error) - GetFsType(path string) (*system.FsType, error) -} - const defaultDevFS = "ext4" -// Provider provides management functionality for metadata storage. -type Provider struct { - log logging.Logger - sys SystemProvider - mounter storage.MountProvider -} +type ( + // SystemProvider provides operating system capabilities. + SystemProvider interface { + Chown(string, int, int) error + Getfs(device string) (string, error) + GetfsType(path string) (*system.FsType, error) + Mkdir(string, os.FileMode) error + Mkfs(req system.MkfsReq) error + RemoveAll(string) error + Stat(string) (os.FileInfo, error) + } + + // Provider provides management functionality for metadata storage. + Provider struct { + log logging.Logger + sys SystemProvider + mounter storage.MountProvider + } +) // Format formats the storage used for control metadata, if it is a separate device. // If the storage location is on an existing partition, the format of the existing filesystem is @@ -119,7 +122,7 @@ func (p *Provider) setupMountPoint(req storage.MetadataFormatRequest) error { func (p *Provider) setupRootDir(req storage.MetadataFormatRequest) error { fsPath := req.RootPath for { - fsType, err := p.sys.GetFsType(fsPath) + fsType, err := p.sys.GetfsType(fsPath) if err == nil { if !p.isUsableFS(fsType, fsPath) { return FaultBadFilesystem(fsType) @@ -175,11 +178,11 @@ func (p *Provider) isUsableFS(fs *system.FsType, path string) bool { } func (p *Provider) setupDataDir(req storage.MetadataFormatRequest) error { - if err := os.RemoveAll(req.DataPath); err != nil { + if err := p.sys.RemoveAll(req.DataPath); err != nil { return errors.Wrap(err, "removing old control metadata subdirectory") } - if err := os.Mkdir(req.DataPath, 0755); err != nil { + if err := p.sys.Mkdir(req.DataPath, 0755); err != nil { return errors.Wrap(err, "creating control metadata subdirectory") } @@ -189,7 +192,7 @@ func (p *Provider) setupDataDir(req storage.MetadataFormatRequest) error { for _, idx := range req.EngineIdxs { engPath := storage.ControlMetadataEngineDir(req.DataPath, idx) - if err := os.Mkdir(engPath, 0755); err != nil { + if err := p.sys.Mkdir(engPath, 0755); err != nil { return errors.Wrapf(err, "creating control metadata engine %d subdirectory", idx) } diff --git a/src/control/server/storage/metadata/provider_test.go b/src/control/server/storage/metadata/provider_test.go index 645fad9fbbf..a3262f78f41 100644 --- a/src/control/server/storage/metadata/provider_test.go +++ b/src/control/server/storage/metadata/provider_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2022 Intel Corporation. +// (C) Copyright 2022-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -69,17 +69,17 @@ func TestMetadata_Provider_Format(t *testing.T) { }, expErr: errors.New("not a subdirectory"), }, - "GetFsType fails": { + "GetfsType fails": { req: pathReq, sysCfg: &system.MockSysConfig{ - GetFsTypeErr: []error{errors.New("mock GetFsType")}, + GetfsTypeErr: []error{errors.New("mock GetfsType")}, }, - expErr: errors.New("mock GetFsType"), + expErr: errors.New("mock GetfsType"), }, - "GetFsType returns nosuid flag": { + "GetfsType returns nosuid flag": { req: pathReq, sysCfg: &system.MockSysConfig{ - GetFsTypeRes: &system.FsType{ + GetfsTypeRes: &system.FsType{ Name: system.FsTypeExt4, NoSUID: true, }, @@ -89,30 +89,30 @@ func TestMetadata_Provider_Format(t *testing.T) { NoSUID: true, }), }, - "GetFsType returns nfs": { + "GetfsType returns nfs": { req: pathReq, sysCfg: &system.MockSysConfig{ - GetFsTypeRes: &system.FsType{Name: system.FsTypeNfs}, + GetfsTypeRes: &system.FsType{Name: system.FsTypeNfs}, }, expErr: FaultBadFilesystem(&system.FsType{Name: system.FsTypeNfs}), }, - "GetFsType returns unknown": { + "GetfsType returns unknown": { req: pathReq, sysCfg: &system.MockSysConfig{ - GetFsTypeRes: &system.FsType{Name: system.FsTypeUnknown}, + GetfsTypeRes: &system.FsType{Name: system.FsTypeUnknown}, }, }, - "GetFsType skipped with device": { + "GetfsType skipped with device": { req: deviceReq, sysCfg: &system.MockSysConfig{ - GetFsTypeErr: []error{errors.New("mock GetFsType")}, + GetfsTypeErr: []error{errors.New("mock GetfsType")}, }, }, - "GetFsType retries with parent if dir doesn't exist": { + "GetfsType retries with parent if dir doesn't exist": { req: pathReq, sysCfg: &system.MockSysConfig{ - GetFsTypeRes: &system.FsType{Name: system.FsTypeExt4}, - GetFsTypeErr: []error{os.ErrNotExist, os.ErrNotExist, nil}, + GetfsTypeRes: &system.FsType{Name: system.FsTypeExt4}, + GetfsTypeErr: []error{os.ErrNotExist, os.ErrNotExist, nil}, }, }, "ClearMountpoint fails": { @@ -201,7 +201,7 @@ func TestMetadata_Provider_Format(t *testing.T) { req: pathReq, sysCfg: &system.MockSysConfig{ MkfsErr: errors.New("mkfs was called!"), - GetFsTypeRes: &system.FsType{Name: system.FsTypeExt4}, + GetfsTypeRes: &system.FsType{Name: system.FsTypeExt4}, }, mountCfg: &storage.MockMountProviderConfig{ MountErr: errors.New("mount was called!"), @@ -215,6 +215,12 @@ func TestMetadata_Provider_Format(t *testing.T) { testDir, cleanupTestDir := test.CreateTestDir(t) defer cleanupTestDir() + if tc.sysCfg == nil { + tc.sysCfg = new(system.MockSysConfig) + } + tc.sysCfg.RealMkdir = true + tc.sysCfg.RealRemoveAll = true + // Point the paths at the testdir if tc.req.RootPath != "" { tc.req.RootPath = filepath.Join(testDir, tc.req.RootPath) @@ -239,7 +245,8 @@ func TestMetadata_Provider_Format(t *testing.T) { var p *Provider if !tc.nilProv { - p = NewProvider(log, system.NewMockSysProvider(log, tc.sysCfg), storage.NewMockMountProvider(tc.mountCfg)) + p = NewProvider(log, system.NewMockSysProvider(log, tc.sysCfg), + storage.NewMockMountProvider(tc.mountCfg)) } err := p.Format(tc.req) diff --git a/src/control/server/storage/mount/provider.go b/src/control/server/storage/mount/provider.go index d27d1bf17c7..c14c7485ff2 100644 --- a/src/control/server/storage/mount/provider.go +++ b/src/control/server/storage/mount/provider.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2022 Intel Corporation. +// (C) Copyright 2022-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -24,13 +24,17 @@ const ( ) type ( - // SystemProvider defines a set of methods to be implemented by a provider - // of system capabilities. + // SystemProvider provides operating system capabilities. SystemProvider interface { system.IsMountedProvider system.MountProvider system.UnmountProvider Chmod(string, os.FileMode) error + Chown(string, int, int) error + Getegid() int + Geteuid() int + Mkdir(string, os.FileMode) error + RemoveAll(string) error Stat(string) (os.FileInfo, error) } @@ -140,7 +144,7 @@ func (p *Provider) ClearMountpoint(mntpt string) error { } } - if err := os.RemoveAll(mntpt); err != nil && !os.IsNotExist(err) { + if err := p.sys.RemoveAll(mntpt); err != nil && !os.IsNotExist(err) { return errors.Wrapf(err, "failed to remove %s", mntpt) } @@ -169,12 +173,12 @@ func (p *Provider) MakeMountPath(path string, tgtUID, tgtGID int) error { switch { case os.IsNotExist(err): // subdir missing, attempt to create and chown - if err := os.Mkdir(ps, defaultMountPointPerms); err != nil { + if err := p.sys.Mkdir(ps, defaultMountPointPerms); err != nil { return errors.Wrapf(err, "failed to create directory %q", ps) } - if err := os.Chown(ps, tgtUID, tgtGID); err != nil { + if err := p.sys.Chown(ps, tgtUID, tgtGID); err != nil { return errors.Wrapf(err, "failed to set ownership of %s to %d.%d from %d.%d", - ps, tgtUID, tgtGID, os.Geteuid(), os.Getegid()) + ps, tgtUID, tgtGID, p.sys.Geteuid(), p.sys.Getegid()) } case err != nil: return errors.Wrapf(err, "unable to stat %q", ps) diff --git a/src/control/server/storage/mount/provider_test.go b/src/control/server/storage/mount/provider_test.go index df5d8cf3c16..952b5229c0a 100644 --- a/src/control/server/storage/mount/provider_test.go +++ b/src/control/server/storage/mount/provider_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2022 Intel Corporation. +// (C) Copyright 2022-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -270,6 +270,11 @@ func TestProvider_ClearMountpoint(t *testing.T) { testDir, cleanupDir := test.CreateTestDir(t) defer cleanupDir() + if tc.msc == nil { + tc.msc = new(system.MockSysConfig) + } + tc.msc.RealRemoveAll = true + if tc.setup == nil { tc.setup = func(t *testing.T, testDir string) func(*testing.T) { t.Helper() @@ -374,6 +379,7 @@ func TestProvider_MakeMountPath(t *testing.T) { msc := &system.MockSysConfig{ StatErrors: make(map[string]error), + RealMkdir: true, } for mp, err := range tc.statErrs { // mocked stat return errors updated for paths diff --git a/src/control/server/storage/provider.go b/src/control/server/storage/provider.go index a8cad7e861b..ecfa0535a2f 100644 --- a/src/control/server/storage/provider.go +++ b/src/control/server/storage/provider.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2021-2023 Intel Corporation. +// (C) Copyright 2021-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -26,9 +26,8 @@ const defaultMetadataPath = "/mnt/daos" // SystemProvider provides operating system capabilities. type SystemProvider interface { system.IsMountedProvider - system.MountProvider GetfsUsage(string) (uint64, uint64, error) - Mkfs(system.MkfsReq) error + ReadFile(string) ([]byte, error) } // Provider provides storage specific capabilities. @@ -185,11 +184,11 @@ func (p *Provider) ControlMetadataIsMounted() (bool, error) { return false, errors.New("nil provider") } - p.log.Debugf("control metadata config: %+v", p.engineStorage.ControlMetadata) if !p.engineStorage.ControlMetadata.HasPath() { // If there's no control metadata path, we use SCM for control metadata return p.ScmIsMounted() } + p.log.Debugf("control metadata config: %+v", p.engineStorage.ControlMetadata) if p.engineStorage.ControlMetadata.DevicePath == "" { p.log.Debug("no metadata device defined") @@ -283,7 +282,7 @@ func (p *Provider) MountScm() error { return errors.New(ScmMsgClassNotSupported) } - p.log.Debugf("attempting to mount existing SCM dir %s\n", cfg.Scm.MountPoint) + p.log.Debugf("attempting to mount SCM dir %s\n", cfg.Scm.MountPoint) res, err := p.scm.Mount(req) if err != nil { diff --git a/src/control/server/storage/scm/mocks.go b/src/control/server/storage/scm/mocks.go index c452f110676..627f6272cd0 100644 --- a/src/control/server/storage/scm/mocks.go +++ b/src/control/server/storage/scm/mocks.go @@ -428,12 +428,16 @@ func DefaultMockBackend() *MockBackend { return NewMockBackend(nil) } +// NewMockProvider stubs os calls by mocking system and mount providers. scm provider functions +// call into system and mount providers for any os access. func NewMockProvider(log logging.Logger, mbc *MockBackendConfig, msc *system.MockSysConfig) *Provider { sysProv := system.NewMockSysProvider(log, msc) mountProv := mount.NewProvider(log, sysProv) return NewProvider(log, NewMockBackend(mbc), sysProv, mountProv) } +// DefaultMockProvider stubs os calls by mocking system and mount providers. scm provider functions +// call into system and mount providers for any os access. func DefaultMockProvider(log logging.Logger) *Provider { sysProv := system.DefaultMockSysProvider(log) mountProv := mount.NewProvider(log, sysProv) diff --git a/src/control/server/storage/scm/provider.go b/src/control/server/storage/scm/provider.go index 1c471c75873..aac7d9ea5da 100644 --- a/src/control/server/storage/scm/provider.go +++ b/src/control/server/storage/scm/provider.go @@ -1,5 +1,4 @@ -// -// (C) Copyright 2019-2022 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -44,14 +43,11 @@ type ( UpdateFirmware(deviceUID string, firmwarePath string) error } - // SystemProvider defines a set of methods to be implemented by a provider - // of SCM-specific system capabilities. + // SystemProvider provides operating system capabilities. SystemProvider interface { - Mkfs(system.MkfsReq) error - Getfs(device string) (string, error) - Stat(string) (os.FileInfo, error) - Chmod(string, os.FileMode) error Chown(string, int, int) error + Getfs(string) (string, error) + Mkfs(system.MkfsReq) error } // Provider encapsulates configuration and logic for @@ -173,7 +169,7 @@ func (p *Provider) prepare(req storage.ScmPrepareRequest, scan scanFn) (*storage if len(scanResp.Namespaces) > 0 { for _, ns := range scanResp.Namespaces { nsDev := "/dev/" + ns.BlockDevice - isMounted, err := p.IsMounted(nsDev) + isMounted, err := p.mounter.IsMounted(nsDev) if err != nil { if os.IsNotExist(errors.Cause(err)) { continue @@ -219,9 +215,14 @@ func (p *Provider) CheckFormat(req storage.ScmFormatRequest) (*storage.ScmFormat Formatted: true, } - isMounted, err := p.IsMounted(req.Mountpoint) - if err != nil && !os.IsNotExist(err) { - return nil, err + mntptMissing := false + + isMounted, err := p.mounter.IsMounted(req.Mountpoint) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + mntptMissing = true } if isMounted { res.Mounted = true @@ -246,6 +247,10 @@ func (p *Provider) CheckFormat(req storage.ScmFormatRequest) (*storage.ScmFormat switch fsType { case system.FsTypeExt4: + if mntptMissing { + return nil, storage.FaultDeviceWithFsNoMountpoint(req.Dcpm.Device, + req.Mountpoint) + } res.Mountable = true case system.FsTypeNone: res.Formatted = false @@ -468,14 +473,3 @@ func (p *Provider) Unmount(req storage.ScmMountRequest) (*storage.MountResponse, Target: req.Target, }) } - -// Stat probes the specified path and returns os level file info. -func (p *Provider) Stat(path string) (os.FileInfo, error) { - return p.sys.Stat(path) -} - -// IsMounted checks to see if the target device or directory is mounted and -// returns flag to specify whether mounted or a relevant fault. -func (p *Provider) IsMounted(target string) (bool, error) { - return p.mounter.IsMounted(target) -} diff --git a/src/control/server/storage/scm/provider_test.go b/src/control/server/storage/scm/provider_test.go index fab72a0aae8..dd6e473a96a 100644 --- a/src/control/server/storage/scm/provider_test.go +++ b/src/control/server/storage/scm/provider_test.go @@ -233,7 +233,7 @@ func TestProvider_Prepare(t *testing.T) { // Verify namespaces get unmounted on reset. for _, ns := range tc.scanResp.Namespaces { - isMounted, err := p.IsMounted("/dev/" + ns.BlockDevice) + isMounted, err := p.mounter.IsMounted("/dev/" + ns.BlockDevice) if err != nil { t.Fatal(err) } @@ -293,6 +293,7 @@ func TestProvider_CheckFormat(t *testing.T) { "isMounted fails": { mountPoint: goodMountPoint, isMountedErr: errors.New("is mounted check failed"), + expErr: errors.New("is mounted check failed"), }, "mountpoint doesn't exist": { mountPoint: goodMountPoint, @@ -302,6 +303,18 @@ func TestProvider_CheckFormat(t *testing.T) { Formatted: false, }, }, + "mountpoint doesn't exist; dcpm has expected fs": { + request: &storage.ScmFormatRequest{ + Mountpoint: "/missing/dir", + Dcpm: &storage.DeviceParams{ + Device: goodDevice, + }, + }, + getFsStr: system.FsTypeExt4, + isMountedErr: os.ErrNotExist, + expErr: storage.FaultDeviceWithFsNoMountpoint(goodDevice, + "/missing/dir"), + }, "already mounted": { mountPoint: goodMountPoint, alreadyMounted: true, @@ -319,6 +332,7 @@ func TestProvider_CheckFormat(t *testing.T) { }, }, getFsErr: errors.New("getfs failed"), + expErr: errors.New("getfs failed"), }, "already formatted; not mountable": { request: &storage.ScmFormatRequest{ @@ -390,15 +404,11 @@ func TestProvider_CheckFormat(t *testing.T) { } } res, err := p.CheckFormat(*req) - if err != nil { - switch errors.Cause(err) { - case tc.isMountedErr, tc.getFsErr, - tc.expErr: - return - default: - t.Fatal(err) - } + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return } + cmpRes(t, tc.expResponse, res) }) } diff --git a/src/control/server/util_test.go b/src/control/server/util_test.go index f2d54c8e585..adcee6585d0 100644 --- a/src/control/server/util_test.go +++ b/src/control/server/util_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -171,7 +171,7 @@ func newMockDrpcClient(cfg *mockDrpcClientConfig) *mockDrpcClient { // setupMockDrpcClientBytes sets up the dRPC client for the mgmtSvc to return // a set of bytes as a response. func setupMockDrpcClientBytes(svc *mgmtSvc, respBytes []byte, err error) { - setMockDrpcClient(svc, getMockDrpcClientBytes(respBytes, err)) + setupSvcDrpcClient(svc, 0, getMockDrpcClientBytes(respBytes, err)) } func getMockDrpcClientBytes(respBytes []byte, err error) *mockDrpcClient { @@ -183,17 +183,19 @@ func getMockDrpcClientBytes(respBytes []byte, err error) *mockDrpcClient { // setupMockDrpcClient sets up the dRPC client for the mgmtSvc to return // a valid protobuf message as a response. func setupMockDrpcClient(svc *mgmtSvc, resp proto.Message, err error) { - setMockDrpcClient(svc, getMockDrpcClient(resp, err)) + setupSvcDrpcClient(svc, 0, getMockDrpcClient(resp, err)) } +// getMockDrpcClient sets up the dRPC client to return a valid protobuf message as a response. func getMockDrpcClient(resp proto.Message, err error) *mockDrpcClient { respBytes, _ := proto.Marshal(resp) return getMockDrpcClientBytes(respBytes, err) } -func setMockDrpcClient(svc *mgmtSvc, mdc *mockDrpcClient) { - mi := svc.harness.instances[0] - mi.(*EngineInstance).setDrpcClient(mdc) +func setupSvcDrpcClient(svc *mgmtSvc, engineIdx int, mdc *mockDrpcClient) { + svc.harness.instances[engineIdx].(*EngineInstance).getDrpcClientFn = func(_ string) drpc.DomainSocketClient { + return mdc + } } // newTestEngine returns an EngineInstance configured for testing. @@ -213,6 +215,7 @@ func newTestEngine(log logging.Logger, isAP bool, provider *storage.Provider, en r := engine.NewTestRunner(rCfg, engineCfg[0]) e := NewEngineInstance(log, provider, nil, r) + e.setDrpcSocket("/dontcare") e.setSuperblock(&Superblock{ Rank: ranklist.NewRankPtr(0), }) diff --git a/src/control/system/pool.go b/src/control/system/pool.go index f029f0628da..9720e792557 100644 --- a/src/control/system/pool.go +++ b/src/control/system/pool.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2022-2023 Intel Corporation. +// (C) Copyright 2022-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -8,61 +8,57 @@ package system import ( "fmt" - "strconv" - "strings" "sync" "time" "github.com/dustin/go-humanize" "github.com/google/uuid" - "github.com/pkg/errors" - mgmtpb "github.com/daos-stack/daos/src/control/common/proto/mgmt" - . "github.com/daos-stack/daos/src/control/lib/ranklist" -) - -const ( - // PoolServiceStateCreating indicates that the pool service is being created - PoolServiceStateCreating PoolServiceState = iota - // PoolServiceStateReady indicates that the pool service is ready to be used - PoolServiceStateReady - // PoolServiceStateDestroying indicates that the pool service is being destroyed - PoolServiceStateDestroying - // PoolServiceStateDegraded indicates that the pool service is being Degraded - PoolServiceStateDegraded - // PoolServiceStateUnknown indicates that the pool service is Unknown state - PoolServiceStateUnknown + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" ) type ( - // PoolServiceState is used to represent the state of the pool service - PoolServiceState uint - // PoolServiceStorage holds information about the pool storage. PoolServiceStorage struct { sync.Mutex - CreationRankStr string // string rankset set at creation - creationRanks *RankSet // used to reconstitute the rankset - CurrentRankStr string // string rankset representing current ranks - currentRanks *RankSet // used to reconstitute the rankset - PerRankTierStorage []uint64 // storage allocated to each tier on a rank + CreationRankStr string // string rankset set at creation + creationRanks *ranklist.RankSet // used to reconstitute the rankset + CurrentRankStr string // string rankset representing current ranks + currentRanks *ranklist.RankSet // used to reconstitute the rankset + PerRankTierStorage []uint64 // storage allocated to each tier on a rank } + // PoolServiceState is a local type alias for daos.PoolServiceState. + // NB: We use this to insulate the system DB from any incompatible + // changes made to the daos.PoolServiceState type. + PoolServiceState daos.PoolServiceState + // PoolService represents a pool service created to manage metadata // for a DAOS Pool. PoolService struct { PoolUUID uuid.UUID PoolLabel string State PoolServiceState - Replicas []Rank + Replicas []ranklist.Rank Storage *PoolServiceStorage LastUpdate time.Time } ) +const ( + PoolServiceStateCreating = PoolServiceState(daos.PoolServiceStateCreating) + PoolServiceStateReady = PoolServiceState(daos.PoolServiceStateReady) + PoolServiceStateDestroying = PoolServiceState(daos.PoolServiceStateDestroying) +) + +func (pss PoolServiceState) String() string { + return daos.PoolServiceState(pss).String() +} + // NewPoolService returns a properly-initialized *PoolService. -func NewPoolService(uuid uuid.UUID, tierStorage []uint64, ranks []Rank) *PoolService { - rs := RankSetFromRanks(ranks) +func NewPoolService(uuid uuid.UUID, tierStorage []uint64, ranks []ranklist.Rank) *PoolService { + rs := ranklist.RankSetFromRanks(ranks) return &PoolService{ PoolUUID: uuid, State: PoolServiceStateCreating, @@ -76,13 +72,13 @@ func NewPoolService(uuid uuid.UUID, tierStorage []uint64, ranks []Rank) *PoolSer // CreationRanks returns the set of target ranks associated // with the pool's creation. -func (pss *PoolServiceStorage) CreationRanks() []Rank { +func (pss *PoolServiceStorage) CreationRanks() []ranklist.Rank { pss.Lock() defer pss.Unlock() if pss.creationRanks == nil { var err error - pss.creationRanks, err = CreateRankSet(pss.CreationRankStr) + pss.creationRanks, err = ranklist.CreateRankSet(pss.CreationRankStr) if err != nil { return nil } @@ -92,13 +88,13 @@ func (pss *PoolServiceStorage) CreationRanks() []Rank { // CurrentRanks returns the set of target ranks associated // with the pool's current. -func (pss *PoolServiceStorage) CurrentRanks() []Rank { +func (pss *PoolServiceStorage) CurrentRanks() []ranklist.Rank { pss.Lock() defer pss.Unlock() if pss.currentRanks == nil { var err error - pss.currentRanks, err = CreateRankSet(pss.CurrentRankStr) + pss.currentRanks, err = ranklist.CreateRankSet(pss.CurrentRankStr) if err != nil { return nil } @@ -138,43 +134,3 @@ func (pss *PoolServiceStorage) String() string { humanize.Bytes(pss.TotalSCM()), humanize.Bytes(pss.TotalNVMe())) } - -func (pss PoolServiceState) String() string { - return [...]string{ - "Creating", - "Ready", - "Destroying", - "Degraded", - "Unknown", - }[pss] -} - -func (pss PoolServiceState) MarshalJSON() ([]byte, error) { - stateStr, ok := mgmtpb.PoolServiceState_name[int32(pss)] - if !ok { - return nil, errors.Errorf("invalid Pool Service state %d", pss) - } - return []byte(`"` + stateStr + `"`), nil -} - -func (pss *PoolServiceState) UnmarshalJSON(data []byte) error { - stateStr := strings.Trim(string(data), "\"") - - state, ok := mgmtpb.PoolServiceState_value[stateStr] - if !ok { - // Try converting the string to an int32, to handle the - // conversion from protobuf message using convert.Types(). - si, err := strconv.ParseInt(stateStr, 0, 32) - if err != nil { - return errors.Errorf("invalid Pool Service state number parse %q", stateStr) - } - - if _, ok = mgmtpb.PoolServiceState_name[int32(si)]; !ok { - return errors.Errorf("invalid Pool Service state name lookup %q", stateStr) - } - state = int32(si) - } - *pss = PoolServiceState(state) - - return nil -} diff --git a/src/control/system/raft/database_test.go b/src/control/system/raft/database_test.go index 0712986840e..9035db01aaf 100644 --- a/src/control/system/raft/database_test.go +++ b/src/control/system/raft/database_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2023 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -272,7 +272,7 @@ func TestSystem_Database_SnapshotRestore(t *testing.T) { ps := &PoolService{ PoolUUID: uuid.New(), PoolLabel: fmt.Sprintf("pool%04d", i), - State: PoolServiceStateReady, + State: system.PoolServiceStateReady, Replicas: <-replicas, Storage: &PoolServiceStorage{ CreationRankStr: fmt.Sprintf("[0-%d]", maxRanks), @@ -735,7 +735,7 @@ func TestSystem_Database_OnEvent(t *testing.T) { { PoolUUID: puuid, PoolLabel: "pool0001", - State: PoolServiceStateReady, + State: system.PoolServiceStateReady, Replicas: []Rank{1, 2, 3, 4, 5}, LastUpdate: time.Now(), }, @@ -746,7 +746,7 @@ func TestSystem_Database_OnEvent(t *testing.T) { { PoolUUID: puuid, PoolLabel: "pool0001", - State: PoolServiceStateReady, + State: system.PoolServiceStateReady, Replicas: []Rank{1, 2, 3, 4, 5}, LastUpdate: time.Now(), }, @@ -757,7 +757,7 @@ func TestSystem_Database_OnEvent(t *testing.T) { { PoolUUID: puuid, PoolLabel: "pool0001", - State: PoolServiceStateReady, + State: system.PoolServiceStateReady, Replicas: []Rank{1, 2, 3, 4, 5}, LastUpdate: time.Now(), }, @@ -768,7 +768,7 @@ func TestSystem_Database_OnEvent(t *testing.T) { { PoolUUID: puuid, PoolLabel: "pool0001", - State: PoolServiceStateReady, + State: system.PoolServiceStateReady, Replicas: []Rank{2, 3, 5, 6, 7}, LastUpdate: time.Now(), }, @@ -824,21 +824,21 @@ func TestSystemDatabase_PoolServiceList(t *testing.T) { ready := &PoolService{ PoolUUID: uuid.New(), PoolLabel: "pool0001", - State: PoolServiceStateReady, + State: system.PoolServiceStateReady, Replicas: []Rank{1, 2, 3, 4, 5}, LastUpdate: time.Now(), } creating := &PoolService{ PoolUUID: uuid.New(), PoolLabel: "pool0002", - State: PoolServiceStateCreating, + State: system.PoolServiceStateCreating, Replicas: []Rank{1, 2, 3, 4, 5}, LastUpdate: time.Now(), } destroying := &PoolService{ PoolUUID: uuid.New(), PoolLabel: "pool0003", - State: PoolServiceStateDestroying, + State: system.PoolServiceStateDestroying, Replicas: []Rank{1, 2, 3, 4, 5}, LastUpdate: time.Now(), } diff --git a/src/control/system/raft/raft_recovery_test.go b/src/control/system/raft/raft_recovery_test.go index 9d818fbfbc0..211291c26ce 100644 --- a/src/control/system/raft/raft_recovery_test.go +++ b/src/control/system/raft/raft_recovery_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2022 Intel Corporation. +// (C) Copyright 2022-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -26,7 +26,7 @@ import ( "github.com/daos-stack/daos/src/control/common/test" . "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/logging" - . "github.com/daos-stack/daos/src/control/system" + "github.com/daos-stack/daos/src/control/system" ) var regenRaftFixtures = flag.Bool("regen-raft-fixtures", false, "regenerate raft test files") @@ -102,12 +102,12 @@ func Test_Raft_RegenerateFixtures(t *testing.T) { t.Log("adding ranks") nextAddr := ctrlAddrGen(ctx, net.IPv4(127, 0, 0, 1), 4) for i := 0; i < maxRanks; i++ { - m := &Member{ + m := &system.Member{ Rank: NilRank, UUID: uuid.New(), Addr: <-nextAddr, - State: MemberStateJoined, - FaultDomain: MustCreateFaultDomainFromString("/my/test/domain"), + State: system.MemberStateJoined, + FaultDomain: system.MustCreateFaultDomainFromString("/my/test/domain"), } if err := db.AddMember(m); err != nil { @@ -120,12 +120,12 @@ func Test_Raft_RegenerateFixtures(t *testing.T) { t.Log("adding pools") replicas := replicaGen(ctx, maxRanks, 3) for i := 0; i < maxPools; i++ { - ps := &PoolService{ + ps := &system.PoolService{ PoolUUID: uuid.New(), PoolLabel: fmt.Sprintf("pool%04d", i), - State: PoolServiceStateReady, + State: system.PoolServiceStateReady, Replicas: <-replicas, - Storage: &PoolServiceStorage{ + Storage: &system.PoolServiceStorage{ CreationRankStr: fmt.Sprintf("[0-%d]", maxRanks), CurrentRankStr: fmt.Sprintf("[0-%d]", maxRanks), PerRankTierStorage: []uint64{1, 2}, diff --git a/src/control/system/raft/testdata/raft_recovery/daos_system.db b/src/control/system/raft/testdata/raft_recovery/daos_system.db index 7f097491608..1202c95bbf1 100644 Binary files a/src/control/system/raft/testdata/raft_recovery/daos_system.db and b/src/control/system/raft/testdata/raft_recovery/daos_system.db differ diff --git a/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1710888709486/meta.json b/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1710888709486/meta.json deleted file mode 100644 index 0deb71db6b4..00000000000 --- a/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1710888709486/meta.json +++ /dev/null @@ -1 +0,0 @@ -{"Version":1,"ID":"2-20-1710888709486","Index":20,"Term":2,"Peers":"ka8xMjcuMC4wLjE6MTAwMDE=","Configuration":{"Servers":[{"Suffrage":0,"ID":"127.0.0.1:10001","Address":"127.0.0.1:10001"}]},"ConfigurationIndex":1,"Size":4469,"CRC":"iy4czkUrmmQ="} diff --git a/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1710888709486/state.bin b/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1710888709486/state.bin deleted file mode 100644 index 0f7c06c2753..00000000000 --- a/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1710888709486/state.bin +++ /dev/null @@ -1 +0,0 @@ -{"Version":8,"NextRank":9,"MapVersion":8,"Members":{"Ranks":{"1":"d50800c9-2104-4ba5-996c-55bcf7a48292","2":"692e11da-0e05-4fd6-be95-84ce57f7a553","3":"85dbe95d-fe54-418c-a547-cddb4af352d8","4":"cc537a2a-5566-42f7-a151-69b414d99425","5":"e267adb5-d205-4c15-b0c8-4e3e75200d48","6":"367ce851-5fc1-4649-a40a-a5d7b097191c","7":"e7c73998-5f99-4dbe-9661-c53139ffe413","8":"16f9d2f3-97e4-4c02-bb63-eac9bc6af509"},"Uuids":{"16f9d2f3-97e4-4c02-bb63-eac9bc6af509":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":8,"incarnation":0,"uuid":"16f9d2f3-97e4-4c02-bb63-eac9bc6af509","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.902982689Z"},"367ce851-5fc1-4649-a40a-a5d7b097191c":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":6,"incarnation":0,"uuid":"367ce851-5fc1-4649-a40a-a5d7b097191c","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.897949913Z"},"692e11da-0e05-4fd6-be95-84ce57f7a553":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":2,"incarnation":0,"uuid":"692e11da-0e05-4fd6-be95-84ce57f7a553","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.886795929Z"},"85dbe95d-fe54-418c-a547-cddb4af352d8":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":3,"incarnation":0,"uuid":"85dbe95d-fe54-418c-a547-cddb4af352d8","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.889541042Z"},"cc537a2a-5566-42f7-a151-69b414d99425":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":4,"incarnation":0,"uuid":"cc537a2a-5566-42f7-a151-69b414d99425","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.891880278Z"},"d50800c9-2104-4ba5-996c-55bcf7a48292":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":1,"incarnation":0,"uuid":"d50800c9-2104-4ba5-996c-55bcf7a48292","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.883262135Z"},"e267adb5-d205-4c15-b0c8-4e3e75200d48":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":5,"incarnation":0,"uuid":"e267adb5-d205-4c15-b0c8-4e3e75200d48","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.895441851Z"},"e7c73998-5f99-4dbe-9661-c53139ffe413":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":7,"incarnation":0,"uuid":"e7c73998-5f99-4dbe-9661-c53139ffe413","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.900455403Z"}},"Addrs":{"127.0.0.1:10001":["d50800c9-2104-4ba5-996c-55bcf7a48292","692e11da-0e05-4fd6-be95-84ce57f7a553","85dbe95d-fe54-418c-a547-cddb4af352d8","cc537a2a-5566-42f7-a151-69b414d99425"],"127.0.0.2:10001":["e267adb5-d205-4c15-b0c8-4e3e75200d48","367ce851-5fc1-4649-a40a-a5d7b097191c","e7c73998-5f99-4dbe-9661-c53139ffe413","16f9d2f3-97e4-4c02-bb63-eac9bc6af509"]},"FaultDomains":{"Domain":{"Domains":null},"ID":1,"Children":[{"Domain":{"Domains":["my"]},"ID":2,"Children":[{"Domain":{"Domains":["my","test"]},"ID":3,"Children":[{"Domain":{"Domains":["my","test","domain"]},"ID":4,"Children":[{"Domain":{"Domains":["my","test","domain","rank1"]},"ID":5,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank2"]},"ID":6,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank3"]},"ID":7,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank4"]},"ID":8,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank5"]},"ID":9,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank6"]},"ID":10,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank7"]},"ID":11,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank8"]},"ID":12,"Children":[]}]}]}]}]}},"Pools":{"Ranks":{},"Uuids":{},"Labels":{}},"Checker":{"Findings":{}},"System":{"Attributes":{}},"SchemaVersion":0} \ No newline at end of file diff --git a/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1713295643181/meta.json b/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1713295643181/meta.json new file mode 100644 index 00000000000..0e6cd99acd1 --- /dev/null +++ b/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1713295643181/meta.json @@ -0,0 +1,19 @@ +{ + "Version": 1, + "ID": "2-20-1713295643181", + "Index": 20, + "Term": 2, + "Peers": "ka8xMjcuMC4wLjE6MTAwMDE=", + "Configuration": { + "Servers": [ + { + "Suffrage": 0, + "ID": "127.0.0.1:10001", + "Address": "127.0.0.1:10001" + } + ] + }, + "ConfigurationIndex": 1, + "Size": 4468, + "CRC": "OHak8IEKGyo=" +} diff --git a/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1713295643181/state.bin b/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1713295643181/state.bin new file mode 100644 index 00000000000..d8a492ac030 --- /dev/null +++ b/src/control/system/raft/testdata/raft_recovery/snapshots/2-20-1713295643181/state.bin @@ -0,0 +1 @@ +{"Version":8,"NextRank":9,"MapVersion":8,"Members":{"Ranks":{"1":"0e08f30e-86a2-47e4-b236-ee1be66086eb","2":"103a4f24-6730-4c07-bc13-1b34c5141b9b","3":"1778f74e-66d0-4496-ba03-7e62ea0d4463","4":"5bc9a6d6-6d7a-49ef-9764-20260f2d9023","5":"2c6bccc4-372b-4b52-9e54-7cfd458e0dae","6":"f5352001-c34f-41c2-a6c0-60dbd6cd4b8f","7":"a283a6e9-c7db-42c6-90ee-8c8636cbeb75","8":"3389e4c9-3724-4093-99e6-5f2c7e813421"},"Uuids":{"0e08f30e-86a2-47e4-b236-ee1be66086eb":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":1,"incarnation":0,"uuid":"0e08f30e-86a2-47e4-b236-ee1be66086eb","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.786131637Z"},"103a4f24-6730-4c07-bc13-1b34c5141b9b":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":2,"incarnation":0,"uuid":"103a4f24-6730-4c07-bc13-1b34c5141b9b","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.790721912Z"},"1778f74e-66d0-4496-ba03-7e62ea0d4463":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":3,"incarnation":0,"uuid":"1778f74e-66d0-4496-ba03-7e62ea0d4463","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.795768326Z"},"2c6bccc4-372b-4b52-9e54-7cfd458e0dae":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":5,"incarnation":0,"uuid":"2c6bccc4-372b-4b52-9e54-7cfd458e0dae","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.800693891Z"},"3389e4c9-3724-4093-99e6-5f2c7e813421":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":8,"incarnation":0,"uuid":"3389e4c9-3724-4093-99e6-5f2c7e813421","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.81085826Z"},"5bc9a6d6-6d7a-49ef-9764-20260f2d9023":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":4,"incarnation":0,"uuid":"5bc9a6d6-6d7a-49ef-9764-20260f2d9023","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.798204514Z"},"a283a6e9-c7db-42c6-90ee-8c8636cbeb75":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":7,"incarnation":0,"uuid":"a283a6e9-c7db-42c6-90ee-8c8636cbeb75","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.805965755Z"},"f5352001-c34f-41c2-a6c0-60dbd6cd4b8f":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":6,"incarnation":0,"uuid":"f5352001-c34f-41c2-a6c0-60dbd6cd4b8f","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.803162578Z"}},"Addrs":{"127.0.0.1:10001":["0e08f30e-86a2-47e4-b236-ee1be66086eb","103a4f24-6730-4c07-bc13-1b34c5141b9b","1778f74e-66d0-4496-ba03-7e62ea0d4463","5bc9a6d6-6d7a-49ef-9764-20260f2d9023"],"127.0.0.2:10001":["2c6bccc4-372b-4b52-9e54-7cfd458e0dae","f5352001-c34f-41c2-a6c0-60dbd6cd4b8f","a283a6e9-c7db-42c6-90ee-8c8636cbeb75","3389e4c9-3724-4093-99e6-5f2c7e813421"]},"FaultDomains":{"Domain":{"Domains":null},"ID":1,"Children":[{"Domain":{"Domains":["my"]},"ID":2,"Children":[{"Domain":{"Domains":["my","test"]},"ID":3,"Children":[{"Domain":{"Domains":["my","test","domain"]},"ID":4,"Children":[{"Domain":{"Domains":["my","test","domain","rank1"]},"ID":5,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank2"]},"ID":6,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank3"]},"ID":7,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank4"]},"ID":8,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank5"]},"ID":9,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank6"]},"ID":10,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank7"]},"ID":11,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank8"]},"ID":12,"Children":[]}]}]}]}]}},"Pools":{"Ranks":{},"Uuids":{},"Labels":{}},"Checker":{"Findings":{}},"System":{"Attributes":{}},"SchemaVersion":0} \ No newline at end of file diff --git a/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1710888711237/meta.json b/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1710888711237/meta.json deleted file mode 100644 index dd094666212..00000000000 --- a/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1710888711237/meta.json +++ /dev/null @@ -1 +0,0 @@ -{"Version":1,"ID":"2-44-1710888711237","Index":44,"Term":2,"Peers":"ka8xMjcuMC4wLjE6MTAwMDE=","Configuration":{"Servers":[{"Suffrage":0,"ID":"127.0.0.1:10001","Address":"127.0.0.1:10001"}]},"ConfigurationIndex":1,"Size":7466,"CRC":"5AdDOZ0IrcY="} diff --git a/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1710888711237/state.bin b/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1710888711237/state.bin deleted file mode 100644 index e4290729cfa..00000000000 --- a/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1710888711237/state.bin +++ /dev/null @@ -1 +0,0 @@ -{"Version":16,"NextRank":9,"MapVersion":8,"Members":{"Ranks":{"1":"d50800c9-2104-4ba5-996c-55bcf7a48292","2":"692e11da-0e05-4fd6-be95-84ce57f7a553","3":"85dbe95d-fe54-418c-a547-cddb4af352d8","4":"cc537a2a-5566-42f7-a151-69b414d99425","5":"e267adb5-d205-4c15-b0c8-4e3e75200d48","6":"367ce851-5fc1-4649-a40a-a5d7b097191c","7":"e7c73998-5f99-4dbe-9661-c53139ffe413","8":"16f9d2f3-97e4-4c02-bb63-eac9bc6af509"},"Uuids":{"16f9d2f3-97e4-4c02-bb63-eac9bc6af509":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":8,"incarnation":0,"uuid":"16f9d2f3-97e4-4c02-bb63-eac9bc6af509","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.902982689Z"},"367ce851-5fc1-4649-a40a-a5d7b097191c":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":6,"incarnation":0,"uuid":"367ce851-5fc1-4649-a40a-a5d7b097191c","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.897949913Z"},"692e11da-0e05-4fd6-be95-84ce57f7a553":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":2,"incarnation":0,"uuid":"692e11da-0e05-4fd6-be95-84ce57f7a553","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.886795929Z"},"85dbe95d-fe54-418c-a547-cddb4af352d8":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":3,"incarnation":0,"uuid":"85dbe95d-fe54-418c-a547-cddb4af352d8","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.889541042Z"},"cc537a2a-5566-42f7-a151-69b414d99425":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":4,"incarnation":0,"uuid":"cc537a2a-5566-42f7-a151-69b414d99425","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.891880278Z"},"d50800c9-2104-4ba5-996c-55bcf7a48292":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":1,"incarnation":0,"uuid":"d50800c9-2104-4ba5-996c-55bcf7a48292","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.883262135Z"},"e267adb5-d205-4c15-b0c8-4e3e75200d48":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":5,"incarnation":0,"uuid":"e267adb5-d205-4c15-b0c8-4e3e75200d48","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.895441851Z"},"e7c73998-5f99-4dbe-9661-c53139ffe413":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":7,"incarnation":0,"uuid":"e7c73998-5f99-4dbe-9661-c53139ffe413","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-03-19T22:51:47.900455403Z"}},"Addrs":{"127.0.0.1:10001":["d50800c9-2104-4ba5-996c-55bcf7a48292","692e11da-0e05-4fd6-be95-84ce57f7a553","85dbe95d-fe54-418c-a547-cddb4af352d8","cc537a2a-5566-42f7-a151-69b414d99425"],"127.0.0.2:10001":["e267adb5-d205-4c15-b0c8-4e3e75200d48","367ce851-5fc1-4649-a40a-a5d7b097191c","e7c73998-5f99-4dbe-9661-c53139ffe413","16f9d2f3-97e4-4c02-bb63-eac9bc6af509"]},"FaultDomains":{"Domain":{"Domains":null},"ID":1,"Children":[{"Domain":{"Domains":["my"]},"ID":2,"Children":[{"Domain":{"Domains":["my","test"]},"ID":3,"Children":[{"Domain":{"Domains":["my","test","domain"]},"ID":4,"Children":[{"Domain":{"Domains":["my","test","domain","rank1"]},"ID":5,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank2"]},"ID":6,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank3"]},"ID":7,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank4"]},"ID":8,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank5"]},"ID":9,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank6"]},"ID":10,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank7"]},"ID":11,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank8"]},"ID":12,"Children":[]}]}]}]}]}},"Pools":{"Ranks":{"0":["3e04eb11-d9da-4913-8ad9-43ee43a252d5","c869c3a6-2a70-4543-89c0-da47dfb92303","7099e240-9d34-42ef-a762-25f525d328ee"],"1":["ab7db85b-57e5-4dc4-b604-4b5458020c6a","7099e240-9d34-42ef-a762-25f525d328ee"],"3":["c869c3a6-2a70-4543-89c0-da47dfb92303"],"4":["687b65a7-19d6-40ef-9cf7-12a45ae17270"],"6":["687b65a7-19d6-40ef-9cf7-12a45ae17270"]},"Uuids":{"15b67f96-f674-4f0b-8ee0-c9ff542de129":{"PoolUUID":"15b67f96-f674-4f0b-8ee0-c9ff542de129","PoolLabel":"pool0007","State":"Ready","Replicas":null,"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-03-19T22:51:49.952013491Z"},"1aaa8dde-b471-48de-b7d2-ca18939ec6aa":{"PoolUUID":"1aaa8dde-b471-48de-b7d2-ca18939ec6aa","PoolLabel":"pool0005","State":"Ready","Replicas":null,"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-03-19T22:51:49.943263063Z"},"3e04eb11-d9da-4913-8ad9-43ee43a252d5":{"PoolUUID":"3e04eb11-d9da-4913-8ad9-43ee43a252d5","PoolLabel":"pool0001","State":"Ready","Replicas":[0],"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-03-19T22:51:49.927044641Z"},"687b65a7-19d6-40ef-9cf7-12a45ae17270":{"PoolUUID":"687b65a7-19d6-40ef-9cf7-12a45ae17270","PoolLabel":"pool0002","State":"Ready","Replicas":[4,6],"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-03-19T22:51:49.931310931Z"},"7099e240-9d34-42ef-a762-25f525d328ee":{"PoolUUID":"7099e240-9d34-42ef-a762-25f525d328ee","PoolLabel":"pool0006","State":"Ready","Replicas":[0,1],"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-03-19T22:51:49.947681515Z"},"ab7db85b-57e5-4dc4-b604-4b5458020c6a":{"PoolUUID":"ab7db85b-57e5-4dc4-b604-4b5458020c6a","PoolLabel":"pool0003","State":"Ready","Replicas":[1],"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-03-19T22:51:49.934622936Z"},"c869c3a6-2a70-4543-89c0-da47dfb92303":{"PoolUUID":"c869c3a6-2a70-4543-89c0-da47dfb92303","PoolLabel":"pool0004","State":"Ready","Replicas":[3,0],"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-03-19T22:51:49.938971457Z"},"df8a3571-8851-414e-b4e3-518c4feed10f":{"PoolUUID":"df8a3571-8851-414e-b4e3-518c4feed10f","PoolLabel":"pool0000","State":"Ready","Replicas":null,"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-03-19T22:51:49.922749044Z"}},"Labels":{"pool0000":"df8a3571-8851-414e-b4e3-518c4feed10f","pool0001":"3e04eb11-d9da-4913-8ad9-43ee43a252d5","pool0002":"687b65a7-19d6-40ef-9cf7-12a45ae17270","pool0003":"ab7db85b-57e5-4dc4-b604-4b5458020c6a","pool0004":"c869c3a6-2a70-4543-89c0-da47dfb92303","pool0005":"1aaa8dde-b471-48de-b7d2-ca18939ec6aa","pool0006":"7099e240-9d34-42ef-a762-25f525d328ee","pool0007":"15b67f96-f674-4f0b-8ee0-c9ff542de129"}},"Checker":{"Findings":{}},"System":{"Attributes":{}},"SchemaVersion":0} \ No newline at end of file diff --git a/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1713295644569/meta.json b/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1713295644569/meta.json new file mode 100644 index 00000000000..8e67ecb5654 --- /dev/null +++ b/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1713295644569/meta.json @@ -0,0 +1,19 @@ +{ + "Version": 1, + "ID": "2-44-1713295644569", + "Index": 44, + "Term": 2, + "Peers": "ka8xMjcuMC4wLjE6MTAwMDE=", + "Configuration": { + "Servers": [ + { + "Suffrage": 0, + "ID": "127.0.0.1:10001", + "Address": "127.0.0.1:10001" + } + ] + }, + "ConfigurationIndex": 1, + "Size": 7328, + "CRC": "4vwX3T5frD4=" +} diff --git a/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1713295644569/state.bin b/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1713295644569/state.bin new file mode 100644 index 00000000000..d21378b8f19 --- /dev/null +++ b/src/control/system/raft/testdata/raft_recovery/snapshots/2-44-1713295644569/state.bin @@ -0,0 +1 @@ +{"Version":16,"NextRank":9,"MapVersion":8,"Members":{"Ranks":{"1":"0e08f30e-86a2-47e4-b236-ee1be66086eb","2":"103a4f24-6730-4c07-bc13-1b34c5141b9b","3":"1778f74e-66d0-4496-ba03-7e62ea0d4463","4":"5bc9a6d6-6d7a-49ef-9764-20260f2d9023","5":"2c6bccc4-372b-4b52-9e54-7cfd458e0dae","6":"f5352001-c34f-41c2-a6c0-60dbd6cd4b8f","7":"a283a6e9-c7db-42c6-90ee-8c8636cbeb75","8":"3389e4c9-3724-4093-99e6-5f2c7e813421"},"Uuids":{"0e08f30e-86a2-47e4-b236-ee1be66086eb":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":1,"incarnation":0,"uuid":"0e08f30e-86a2-47e4-b236-ee1be66086eb","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.786131637Z"},"103a4f24-6730-4c07-bc13-1b34c5141b9b":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":2,"incarnation":0,"uuid":"103a4f24-6730-4c07-bc13-1b34c5141b9b","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.790721912Z"},"1778f74e-66d0-4496-ba03-7e62ea0d4463":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":3,"incarnation":0,"uuid":"1778f74e-66d0-4496-ba03-7e62ea0d4463","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.795768326Z"},"2c6bccc4-372b-4b52-9e54-7cfd458e0dae":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":5,"incarnation":0,"uuid":"2c6bccc4-372b-4b52-9e54-7cfd458e0dae","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.800693891Z"},"3389e4c9-3724-4093-99e6-5f2c7e813421":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":8,"incarnation":0,"uuid":"3389e4c9-3724-4093-99e6-5f2c7e813421","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.81085826Z"},"5bc9a6d6-6d7a-49ef-9764-20260f2d9023":{"addr":"127.0.0.1:10001","state":"joined","fault_domain":"/my/test/domain","rank":4,"incarnation":0,"uuid":"5bc9a6d6-6d7a-49ef-9764-20260f2d9023","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.798204514Z"},"a283a6e9-c7db-42c6-90ee-8c8636cbeb75":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":7,"incarnation":0,"uuid":"a283a6e9-c7db-42c6-90ee-8c8636cbeb75","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.805965755Z"},"f5352001-c34f-41c2-a6c0-60dbd6cd4b8f":{"addr":"127.0.0.2:10001","state":"joined","fault_domain":"/my/test/domain","rank":6,"incarnation":0,"uuid":"f5352001-c34f-41c2-a6c0-60dbd6cd4b8f","fabric_uri":"","secondary_fabric_uris":null,"fabric_contexts":0,"secondary_fabric_contexts":null,"info":"","last_update":"2024-04-16T19:27:21.803162578Z"}},"Addrs":{"127.0.0.1:10001":["0e08f30e-86a2-47e4-b236-ee1be66086eb","103a4f24-6730-4c07-bc13-1b34c5141b9b","1778f74e-66d0-4496-ba03-7e62ea0d4463","5bc9a6d6-6d7a-49ef-9764-20260f2d9023"],"127.0.0.2:10001":["2c6bccc4-372b-4b52-9e54-7cfd458e0dae","f5352001-c34f-41c2-a6c0-60dbd6cd4b8f","a283a6e9-c7db-42c6-90ee-8c8636cbeb75","3389e4c9-3724-4093-99e6-5f2c7e813421"]},"FaultDomains":{"Domain":{"Domains":null},"ID":1,"Children":[{"Domain":{"Domains":["my"]},"ID":2,"Children":[{"Domain":{"Domains":["my","test"]},"ID":3,"Children":[{"Domain":{"Domains":["my","test","domain"]},"ID":4,"Children":[{"Domain":{"Domains":["my","test","domain","rank1"]},"ID":5,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank2"]},"ID":6,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank3"]},"ID":7,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank4"]},"ID":8,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank5"]},"ID":9,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank6"]},"ID":10,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank7"]},"ID":11,"Children":[]},{"Domain":{"Domains":["my","test","domain","rank8"]},"ID":12,"Children":[]}]}]}]}]}},"Pools":{"Ranks":{"1":["7b9f4c04-a32b-47b4-983b-679a9e34c7e3","5e81b4c3-ab13-487c-9718-7f55288e0870"],"2":["7328d39e-3c30-406f-b71c-465f13a3e72c"],"5":["7b9f4c04-a32b-47b4-983b-679a9e34c7e3","00449e23-2ddf-49cb-b677-eedf0a3fe574"],"7":["7328d39e-3c30-406f-b71c-465f13a3e72c"]},"Uuids":{"00449e23-2ddf-49cb-b677-eedf0a3fe574":{"PoolUUID":"00449e23-2ddf-49cb-b677-eedf0a3fe574","PoolLabel":"pool0003","State":1,"Replicas":[5],"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-04-16T19:27:23.84261755Z"},"22a4c7d4-06a1-42bb-ab6d-96226d420580":{"PoolUUID":"22a4c7d4-06a1-42bb-ab6d-96226d420580","PoolLabel":"pool0005","State":1,"Replicas":null,"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-04-16T19:27:23.85192468Z"},"37aadd3b-38d6-4c88-be04-d101d7de5ac7":{"PoolUUID":"37aadd3b-38d6-4c88-be04-d101d7de5ac7","PoolLabel":"pool0006","State":1,"Replicas":null,"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-04-16T19:27:23.855883806Z"},"3c319416-b73a-41ad-bdf2-e6fd7a38a08d":{"PoolUUID":"3c319416-b73a-41ad-bdf2-e6fd7a38a08d","PoolLabel":"pool0001","State":1,"Replicas":null,"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-04-16T19:27:23.83287002Z"},"5e81b4c3-ab13-487c-9718-7f55288e0870":{"PoolUUID":"5e81b4c3-ab13-487c-9718-7f55288e0870","PoolLabel":"pool0007","State":1,"Replicas":[1],"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-04-16T19:27:23.86177197Z"},"7328d39e-3c30-406f-b71c-465f13a3e72c":{"PoolUUID":"7328d39e-3c30-406f-b71c-465f13a3e72c","PoolLabel":"pool0004","State":1,"Replicas":[2,7],"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-04-16T19:27:23.846600145Z"},"7b9f4c04-a32b-47b4-983b-679a9e34c7e3":{"PoolUUID":"7b9f4c04-a32b-47b4-983b-679a9e34c7e3","PoolLabel":"pool0000","State":1,"Replicas":[5,1],"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-04-16T19:27:23.817530746Z"},"b9a029a2-b0b6-430b-8af6-b828035137fd":{"PoolUUID":"b9a029a2-b0b6-430b-8af6-b828035137fd","PoolLabel":"pool0002","State":1,"Replicas":null,"Storage":{"CreationRankStr":"[0-8]","CurrentRankStr":"[0-8]","PerRankTierStorage":[1,2]},"LastUpdate":"2024-04-16T19:27:23.838448404Z"}},"Labels":{"pool0000":"7b9f4c04-a32b-47b4-983b-679a9e34c7e3","pool0001":"3c319416-b73a-41ad-bdf2-e6fd7a38a08d","pool0002":"b9a029a2-b0b6-430b-8af6-b828035137fd","pool0003":"00449e23-2ddf-49cb-b677-eedf0a3fe574","pool0004":"7328d39e-3c30-406f-b71c-465f13a3e72c","pool0005":"22a4c7d4-06a1-42bb-ab6d-96226d420580","pool0006":"37aadd3b-38d6-4c88-be04-d101d7de5ac7","pool0007":"5e81b4c3-ab13-487c-9718-7f55288e0870"}},"Checker":{"Findings":{}},"System":{"Attributes":{}},"SchemaVersion":0} \ No newline at end of file diff --git a/src/dtx/dtx_common.c b/src/dtx/dtx_common.c index 0be9b60ae77..7b879ff2900 100644 --- a/src/dtx/dtx_common.c +++ b/src/dtx/dtx_common.c @@ -1627,6 +1627,7 @@ dtx_end(struct dtx_handle *dth, struct ds_cont_child *cont, int result) static void dtx_flush_on_close(struct dss_module_info *dmi, struct dtx_batched_cont_args *dbca) { + struct dss_xstream *dx = dss_current_xstream(); struct ds_cont_child *cont = dbca->dbca_cont; struct dtx_stat stat = { 0 }; uint64_t total = 0; @@ -1636,7 +1637,8 @@ dtx_flush_on_close(struct dss_module_info *dmi, struct dtx_batched_cont_args *db dtx_stat(cont, &stat); /* dbca->dbca_reg_gen != cont->sc_dtx_batched_gen means someone reopen the container. */ - while (dbca->dbca_reg_gen == cont->sc_dtx_batched_gen && rc >= 0) { + while (!dss_xstream_exiting(dx) && + dbca->dbca_reg_gen == cont->sc_dtx_batched_gen && rc >= 0) { struct dtx_entry **dtes = NULL; struct dtx_cos_key *dcks = NULL; struct dtx_coll_entry *dce = NULL; @@ -1715,6 +1717,10 @@ start_dtx_reindex_ult(struct ds_cont_child *cont) while (cont->sc_dtx_reindex_abort) ABT_thread_yield(); + if (cont->sc_stopping) + return -DER_SHUTDOWN; + + cont->sc_dtx_delay_reset = 0; if (cont->sc_dtx_reindex) return 0; @@ -1732,7 +1738,7 @@ start_dtx_reindex_ult(struct ds_cont_child *cont) } void -stop_dtx_reindex_ult(struct ds_cont_child *cont) +stop_dtx_reindex_ult(struct ds_cont_child *cont, bool force) { /* DTX reindex has been done or not has not been started. */ if (!cont->sc_dtx_reindex) @@ -1742,9 +1748,15 @@ stop_dtx_reindex_ult(struct ds_cont_child *cont) if (dtx_cont_opened(cont)) return; - /* Do not stop DTX reindex if DTX resync is still in-progress. */ - if (cont->sc_dtx_resyncing) + /* + * For non-force case, do not stop DTX re-index if DTX resync + * is in-progress. Related DTX resource will be released after + * DTX resync globally done (via rebuild scanning). + */ + if (unlikely(cont->sc_dtx_resyncing && !force)) { + cont->sc_dtx_delay_reset = 1; return; + } cont->sc_dtx_reindex_abort = 1; @@ -1813,10 +1825,6 @@ dtx_cont_register(struct ds_cont_child *cont) D_GOTO(out, rc = -DER_NOMEM); } - cont->sc_dtx_committable_count = 0; - cont->sc_dtx_committable_coll_count = 0; - D_INIT_LIST_HEAD(&cont->sc_dtx_cos_list); - D_INIT_LIST_HEAD(&cont->sc_dtx_coll_list); ds_cont_child_get(cont); dbca->dbca_refs = 0; dbca->dbca_cont = cont; @@ -1904,7 +1912,7 @@ dtx_cont_open(struct ds_cont_child *cont) } void -dtx_cont_close(struct ds_cont_child *cont) +dtx_cont_close(struct ds_cont_child *cont, bool force) { struct dss_module_info *dmi = dss_get_module_info(); struct dtx_batched_pool_args *dbpa; @@ -1919,7 +1927,7 @@ dtx_cont_close(struct ds_cont_child *cont) d_list_for_each_entry(dbca, &dbpa->dbpa_cont_list, dbca_pool_link) { if (dbca->dbca_cont == cont) { - stop_dtx_reindex_ult(cont); + stop_dtx_reindex_ult(cont, force); d_list_del(&dbca->dbca_sys_link); d_list_add_tail(&dbca->dbca_sys_link, &dmi->dmi_dtx_batched_cont_close_list); @@ -1927,8 +1935,12 @@ dtx_cont_close(struct ds_cont_child *cont) /* If nobody reopen the container during dtx_flush_on_close, * then reset DTX table in VOS to release related resources. + * + * For non-force case, do not reset DTX table if DTX resync + * is in-progress to avoid redoing DTX re-index. We will do + * that after DTX resync done globally. */ - if (!dtx_cont_opened(cont)) + if (likely(!dtx_cont_opened(cont) && cont->sc_dtx_delay_reset == 0)) vos_dtx_cache_reset(cont->sc_hdl, false); return; } @@ -2315,10 +2327,11 @@ int dtx_obj_sync(struct ds_cont_child *cont, daos_unit_oid_t *oid, daos_epoch_t epoch) { - int cnt; - int rc = 0; + struct dss_xstream *dx = dss_current_xstream(); + int cnt; + int rc = 0; - while (dtx_cont_opened(cont)) { + while (!dss_xstream_exiting(dx) && (dtx_cont_opened(cont) || oid == NULL)) { struct dtx_entry **dtes = NULL; struct dtx_cos_key *dcks = NULL; struct dtx_coll_entry *dce = NULL; diff --git a/src/dtx/dtx_internal.h b/src/dtx/dtx_internal.h index 4f174e7314a..e59baede09c 100644 --- a/src/dtx/dtx_internal.h +++ b/src/dtx/dtx_internal.h @@ -207,7 +207,7 @@ struct dtx_coll_prep_args { struct dtx_pool_metrics { struct d_tm_node_t *dpm_batched_degree; struct d_tm_node_t *dpm_batched_total; - struct d_tm_node_t *dpm_total[DTX_PROTO_SRV_RPC_COUNT]; + struct d_tm_node_t *dpm_total[DTX_PROTO_SRV_RPC_COUNT + 1]; }; /* @@ -249,7 +249,6 @@ int dtx_handle_reinit(struct dtx_handle *dth); void dtx_batched_commit(void *arg); void dtx_aggregation_main(void *arg); int start_dtx_reindex_ult(struct ds_cont_child *cont); -void stop_dtx_reindex_ult(struct ds_cont_child *cont); void dtx_merge_check_result(int *tgt, int src); int dtx_leader_get(struct ds_pool *pool, struct dtx_memberships *mbs, daos_unit_oid_t *oid, uint32_t version, struct pool_target **p_tgt); @@ -292,6 +291,7 @@ enum dtx_status_handle_result { enum dtx_rpc_flags { DRF_INITIAL_LEADER = (1 << 0), + DRF_SYNC_COMMIT = (1 << 1), }; enum dtx_cos_flags { diff --git a/src/dtx/dtx_resync.c b/src/dtx/dtx_resync.c index 9e9f47537d2..3698f238442 100644 --- a/src/dtx/dtx_resync.c +++ b/src/dtx/dtx_resync.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -700,9 +700,6 @@ dtx_resync(daos_handle_t po_hdl, uuid_t po_uuid, uuid_t co_uuid, uint32_t ver, b ABT_mutex_unlock(cont->sc_mutex); out: - if (!dtx_cont_opened(cont)) - stop_dtx_reindex_ult(cont); - D_DEBUG(DB_MD, "Exit DTX resync (%s) for "DF_UUID"/"DF_UUID" with ver %u, rc = %d\n", block ? "block" : "non-block", DP_UUID(po_uuid), DP_UUID(co_uuid), ver, rc); @@ -747,8 +744,8 @@ dtx_resync_one(void *data) { struct dtx_scan_args *arg = data; struct ds_pool_child *child; - vos_iter_param_t param = { 0 }; - struct vos_iter_anchors anchor = { 0 }; + vos_iter_param_t *param = NULL; + struct vos_iter_anchors *anchor = NULL; struct dtx_container_scan_arg cb_arg = { 0 }; int rc; @@ -757,17 +754,28 @@ dtx_resync_one(void *data) D_GOTO(out, rc = -DER_NONEXIST); if (unlikely(child->spc_no_storage)) - D_GOTO(put, rc = 0); + D_GOTO(out, rc = 0); + + D_ALLOC_PTR(param); + if (param == NULL) + D_GOTO(out, rc = -DER_NOMEM); + + D_ALLOC_PTR(anchor); + if (anchor == NULL) + D_GOTO(out, rc = -DER_NOMEM); cb_arg.arg = *arg; - param.ip_hdl = child->spc_hdl; - param.ip_flags = VOS_IT_FOR_MIGRATION; - rc = vos_iterate(¶m, VOS_ITER_COUUID, false, &anchor, + param->ip_hdl = child->spc_hdl; + param->ip_flags = VOS_IT_FOR_MIGRATION; + rc = vos_iterate(param, VOS_ITER_COUUID, false, anchor, container_scan_cb, NULL, &cb_arg, NULL); -put: - ds_pool_child_put(child); out: + D_FREE(param); + D_FREE(anchor); + if (child != NULL) + ds_pool_child_put(child); + D_DEBUG(DB_TRACE, DF_UUID" iterate pool done: rc %d\n", DP_UUID(arg->pool_uuid), rc); diff --git a/src/dtx/dtx_rpc.c b/src/dtx/dtx_rpc.c index 12b8d4a003e..2af60538348 100644 --- a/src/dtx/dtx_rpc.c +++ b/src/dtx/dtx_rpc.c @@ -80,10 +80,15 @@ struct dtx_req_rec { uint32_t drr_tag; /* The VOS ID */ int drr_count; /* DTX count */ int drr_result; /* The RPC result */ - uint32_t drr_comp:1; + uint32_t drr_comp:1, + drr_single_dti:1; + uint32_t drr_inline_flags; struct dtx_id *drr_dti; /* The DTX array */ uint32_t *drr_flags; - struct dtx_share_peer **drr_cb_args; /* Used by dtx_req_cb. */ + union { + struct dtx_share_peer **drr_cb_args; /* Used by dtx_req_cb. */ + struct dtx_share_peer *drr_single_cb_arg; + }; }; struct dtx_cf_rec_bundle { @@ -116,15 +121,22 @@ dtx_drr_cleanup(struct dtx_req_rec *drr) { int i; - if (drr->drr_cb_args != NULL) { - for (i = 0; i < drr->drr_count; i++) { - if (drr->drr_cb_args[i] != NULL) - dtx_dsp_free(drr->drr_cb_args[i]); + if (drr->drr_single_dti == 0) { + D_ASSERT(drr->drr_flags != &drr->drr_inline_flags); + + if (drr->drr_cb_args != NULL) { + for (i = 0; i < drr->drr_count; i++) { + if (drr->drr_cb_args[i] != NULL) + dtx_dsp_free(drr->drr_cb_args[i]); + } + D_FREE(drr->drr_cb_args); } - D_FREE(drr->drr_cb_args); + D_FREE(drr->drr_dti); + D_FREE(drr->drr_flags); + } else if (drr->drr_single_cb_arg != NULL) { + dtx_dsp_free(drr->drr_single_cb_arg); } - D_FREE(drr->drr_dti); - D_FREE(drr->drr_flags); + D_FREE(drr); } @@ -164,13 +176,19 @@ dtx_req_cb(const struct crt_cb_info *cb_info) struct dtx_share_peer *dsp; int *ret; - dsp = drr->drr_cb_args[i]; + if (drr->drr_single_dti == 0) { + dsp = drr->drr_cb_args[i]; + drr->drr_cb_args[i] = NULL; + } else { + dsp = drr->drr_single_cb_arg; + drr->drr_single_cb_arg = NULL; + } + if (dsp == NULL) continue; D_ASSERT(d_list_empty(&dsp->dsp_link)); - drr->drr_cb_args[i] = NULL; ret = (int *)dout->do_sub_rets.ca_arrays + i; switch (*ret) { @@ -498,7 +516,8 @@ btr_ops_t dbtree_dtx_cf_ops = { static int dtx_classify_one(struct ds_pool *pool, daos_handle_t tree, d_list_t *head, int *length, - struct dtx_entry *dte, int count, d_rank_t my_rank, uint32_t my_tgtid) + struct dtx_entry *dte, int count, d_rank_t my_rank, uint32_t my_tgtid, + uint32_t opc) { struct dtx_memberships *mbs = dte->dte_mbs; struct pool_target *target; @@ -561,20 +580,28 @@ dtx_classify_one(struct ds_pool *pool, daos_handle_t tree, d_list_t *head, int * } else { struct dtx_req_rec *drr; + D_ASSERT(count == 1); + D_ALLOC_PTR(drr); if (drr == NULL) D_GOTO(out, rc = -DER_NOMEM); - D_ALLOC_PTR(drr->drr_dti); - if (drr->drr_dti == NULL) { - dtx_drr_cleanup(drr); - D_GOTO(out, rc = -DER_NOMEM); + drr->drr_single_dti = 1; + + /* + * Usually, sync commit handles single DTX and batched commit handles + * multiple ones. So we roughly regard single DTX commit case as sync + * commit for metrics purpose. + */ + if (opc == DTX_COMMIT) { + drr->drr_inline_flags = DRF_SYNC_COMMIT; + drr->drr_flags = &drr->drr_inline_flags; } drr->drr_rank = target->ta_comp.co_rank; drr->drr_tag = target->ta_comp.co_index; drr->drr_count = 1; - drr->drr_dti[0] = dte->dte_xid; + drr->drr_dti = &dte->dte_xid; d_list_add_tail(&drr->drr_link, head); (*length)++; } @@ -614,7 +641,7 @@ dtx_rpc_helper(struct dss_chore *chore, bool is_reentrance) for (i = 0; i < dca->dca_count; i++) { rc = dtx_classify_one(pool, dca->dca_tree_hdl, &dca->dca_head, &length, dca->dca_dtes[i], dca->dca_count, - dca->dca_rank, dca->dca_tgtid); + dca->dca_rank, dca->dca_tgtid, dca->dca_dra.dra_opc); if (rc < 0) { ABT_rwlock_unlock(pool->sp_lock); goto done; @@ -1003,6 +1030,8 @@ dtx_refresh_internal(struct ds_cont_child *cont, int *check_count, d_list_t *che d_list_for_each_entry(drr, &head, drr_link) { if (drr->drr_rank == target->ta_comp.co_rank && drr->drr_tag == target->ta_comp.co_index) { + D_ASSERT(drr->drr_single_dti == 0); + drr->drr_dti[drr->drr_count] = dsp->dsp_xid; drr->drr_flags[drr->drr_count] = flags; drr->drr_cb_args[drr->drr_count++] = dsp; @@ -1014,21 +1043,30 @@ dtx_refresh_internal(struct ds_cont_child *cont, int *check_count, d_list_t *che if (drr == NULL) D_GOTO(out, rc = -DER_NOMEM); - D_ALLOC_ARRAY(drr->drr_dti, *check_count); - D_ALLOC_ARRAY(drr->drr_flags, *check_count); - D_ALLOC_ARRAY(drr->drr_cb_args, *check_count); + if (*check_count == 1) { + drr->drr_single_dti = 1; + drr->drr_dti = &dsp->dsp_xid; + drr->drr_inline_flags = flags; + drr->drr_flags = &drr->drr_inline_flags; + drr->drr_single_cb_arg = dsp; + } else { + D_ALLOC_ARRAY(drr->drr_dti, *check_count); + D_ALLOC_ARRAY(drr->drr_flags, *check_count); + D_ALLOC_ARRAY(drr->drr_cb_args, *check_count); + if (drr->drr_dti == NULL || drr->drr_flags == NULL || + drr->drr_cb_args == NULL) { + dtx_drr_cleanup(drr); + D_GOTO(out, rc = -DER_NOMEM); + } - if (drr->drr_dti == NULL || drr->drr_flags == NULL || drr->drr_cb_args == NULL) { - dtx_drr_cleanup(drr); - D_GOTO(out, rc = -DER_NOMEM); + drr->drr_dti[0] = dsp->dsp_xid; + drr->drr_flags[0] = flags; + drr->drr_cb_args[0] = dsp; } drr->drr_rank = target->ta_comp.co_rank; drr->drr_tag = target->ta_comp.co_index; drr->drr_count = 1; - drr->drr_dti[0] = dsp->dsp_xid; - drr->drr_flags[0] = flags; - drr->drr_cb_args[0] = dsp; d_list_add_tail(&drr->drr_link, &head); len++; @@ -1061,12 +1099,19 @@ dtx_refresh_internal(struct ds_cont_child *cont, int *check_count, d_list_t *che if (drr->drr_cb_args == NULL) goto next2; - for (i = 0; i < drr->drr_count; i++) { - if (drr->drr_cb_args[i] == NULL) - continue; - - dsp = drr->drr_cb_args[i]; - drr->drr_cb_args[i] = NULL; + if (drr->drr_single_dti == 0) { + for (i = 0; i < drr->drr_count; i++) { + if (drr->drr_cb_args[i] == NULL) + continue; + + dsp = drr->drr_cb_args[i]; + drr->drr_cb_args[i] = NULL; + dsp->dsp_status = -DER_INPROGRESS; + d_list_add_tail(&dsp->dsp_link, act_list); + } + } else { + dsp = drr->drr_single_cb_arg; + drr->drr_single_cb_arg = NULL; dsp->dsp_status = -DER_INPROGRESS; d_list_add_tail(&dsp->dsp_link, act_list); } diff --git a/src/dtx/dtx_srv.c b/src/dtx/dtx_srv.c index f82be8ef8be..cfba7862bab 100644 --- a/src/dtx/dtx_srv.c +++ b/src/dtx/dtx_srv.c @@ -114,6 +114,13 @@ dtx_metrics_alloc(const char *path, int tgt_id) D_WARN("Failed to create DTX RPC cnt metric for %s: " DF_RC"\n", dtx_opc_to_str(opc), DP_RC(rc)); } + + rc = d_tm_add_metric(&metrics->dpm_total[DTX_PROTO_SRV_RPC_COUNT], D_TM_COUNTER, + "total number of processed sync DTX_COMMIT", "ops", + "%s/ops/sync_dtx_commit/tgt_%u", path, tgt_id); + if (rc != DER_SUCCESS) + D_WARN("Failed to create sync DTX_COMMIT RPC cnt metric: "DF_RC"\n", DP_RC(rc)); + return metrics; } @@ -149,7 +156,7 @@ dtx_handler(crt_rpc_t *rpc) uint32_t vers[DTX_REFRESH_MAX] = { 0 }; uint32_t opc = opc_get(rpc->cr_opc); uint32_t committed = 0; - uint32_t *flags; + uint32_t *flags = NULL; int *ptr; int count = DTX_YIELD_CYCLE; int i = 0; @@ -192,15 +199,23 @@ dtx_handler(crt_rpc_t *rpc) i += count; } - d_tm_inc_counter(dpm->dpm_batched_total, - din->di_dtx_array.ca_count); - rc1 = d_tm_get_counter(NULL, &ent_cnt, dpm->dpm_batched_total); - D_ASSERT(rc1 == DER_SUCCESS); + if (din->di_flags.ca_count > 0) + flags = din->di_flags.ca_arrays; - rc1 = d_tm_get_counter(NULL, &opc_cnt, dpm->dpm_total[opc]); - D_ASSERT(rc1 == DER_SUCCESS); + if (flags != NULL && (*flags & DRF_SYNC_COMMIT)) { + D_ASSERT(din->di_dtx_array.ca_count == 1); + d_tm_inc_counter(dpm->dpm_total[DTX_PROTO_SRV_RPC_COUNT], 1); + } else { + d_tm_inc_counter(dpm->dpm_batched_total, + din->di_dtx_array.ca_count); + rc1 = d_tm_get_counter(NULL, &ent_cnt, dpm->dpm_batched_total); + D_ASSERT(rc1 == DER_SUCCESS); + + rc1 = d_tm_get_counter(NULL, &opc_cnt, dpm->dpm_total[opc]); + D_ASSERT(rc1 == DER_SUCCESS); - d_tm_set_gauge(dpm->dpm_batched_degree, ent_cnt / (opc_cnt + 1)); + d_tm_set_gauge(dpm->dpm_batched_degree, ent_cnt / (opc_cnt + 1)); + } break; } diff --git a/src/engine/ult.c b/src/engine/ult.c index 30d7af7cd74..94a2a0f9390 100644 --- a/src/engine/ult.c +++ b/src/engine/ult.c @@ -374,30 +374,27 @@ static inline uint32_t sched_ult2xs_multisocket(int xs_type, int tgt_id) { static __thread uint32_t offload; - uint32_t socket; + uint32_t socket = tgt_id / dss_tgt_per_numa_nr; uint32_t base; uint32_t target; if (dss_tgt_offload_xs_nr == 0) { if (xs_type == DSS_XS_IOFW && dss_forward_neighbor) { /* Keep the old forwarding behavior, but NUMA aware */ - socket = tgt_id / dss_numa_nr; target = (socket * dss_tgt_per_numa_nr) + (tgt_id + offload) % dss_tgt_per_numa_nr; - offload = target + 17; /* Seed next selection */ - target = DSS_MAIN_XS_ID(target); + target = DSS_MAIN_XS_ID(target); goto check; } return DSS_XS_SELF; } - socket = tgt_id / dss_numa_nr; - base = dss_sys_xs_nr + dss_tgt_nr + (socket * dss_offload_per_numa_nr); - target = base + ((offload + tgt_id) % dss_offload_per_numa_nr); - offload = target + 17; /* Seed next selection */ + base = dss_sys_xs_nr + dss_tgt_nr + (socket * dss_offload_per_numa_nr); + target = base + ((offload + tgt_id) % dss_offload_per_numa_nr); check: D_ASSERT(target < DSS_XS_NR_TOTAL && target >= dss_sys_xs_nr); + offload = target + 17; /* Seed next selection */ return target; } diff --git a/src/include/cart/api.h b/src/include/cart/api.h index 355cec822aa..2c82b8f4303 100644 --- a/src/include/cart/api.h +++ b/src/include/cart/api.h @@ -1935,6 +1935,8 @@ crt_proto_register(struct crt_proto_format *cpf); * \param[in] base_opc the base opcode for the protocol * \param[in] ver array of protocol version * \param[in] count number of elements in ver + * \param[in] timeout Timeout in seconds, ignored if 0 or greater than + * default timeout * \param[in] cb completion callback. crt_proto_query() internally * sends an RPC to \a tgt_ep. \a cb will be called * upon completion of that RPC. The highest protocol @@ -1948,8 +1950,8 @@ crt_proto_register(struct crt_proto_format *cpf); * failure. */ int -crt_proto_query(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, - uint32_t *ver, int count, crt_proto_query_cb_t cb, void *arg); +crt_proto_query(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, uint32_t *ver, int count, + uint32_t timeout, crt_proto_query_cb_t cb, void *arg); /** * query tgt_ep if it has registered base_opc with version using a user provided cart context. @@ -1958,6 +1960,8 @@ crt_proto_query(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, * \param[in] base_opc the base opcode for the protocol * \param[in] ver array of protocol version * \param[in] count number of elements in ver + * \param[in] timeout Timeout in seconds, ignored if 0 or greater than + * default timeout * \param[in] cb completion callback. crt_proto_query() internally * sends an RPC to \a tgt_ep. \a cb will be called * upon completion of that RPC. The highest protocol @@ -1972,7 +1976,7 @@ crt_proto_query(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, */ int crt_proto_query_with_ctx(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc, uint32_t *ver, int count, - crt_proto_query_cb_t cb, void *arg, crt_context_t ctx); + uint32_t timeout, crt_proto_query_cb_t cb, void *arg, crt_context_t ctx); /** * Set self rank. * diff --git a/src/include/daos/event.h b/src/include/daos/event.h index f7d5a0663eb..12c5c4933ec 100644 --- a/src/include/daos/event.h +++ b/src/include/daos/event.h @@ -44,7 +44,7 @@ daos_eq_lib_fini(void); * Initialize event queue library. */ int -daos_eq_lib_init(void); +daos_eq_lib_init(crt_init_options_t *crt_info); /** * reset context after fork diff --git a/src/include/daos/mgmt.h b/src/include/daos/mgmt.h index 13e46ce9af2..980e7a48bf8 100644 --- a/src/include/daos/mgmt.h +++ b/src/include/daos/mgmt.h @@ -58,7 +58,8 @@ void dc_mgmt_sys_detach(struct dc_mgmt_sys *sys); ssize_t dc_mgmt_sys_encode(struct dc_mgmt_sys *sys, void *buf, size_t cap); ssize_t dc_mgmt_sys_decode(void *buf, size_t len, struct dc_mgmt_sys **sysp); -int dc_mgmt_net_cfg(const char *name); +int + dc_mgmt_net_cfg(const char *name, crt_init_options_t *crt_info); int dc_mgmt_net_cfg_check(const char *name); int dc_mgmt_get_pool_svc_ranks(struct dc_mgmt_sys *sys, const uuid_t puuid, d_rank_list_t **svcranksp); diff --git a/src/include/daos_cont.h b/src/include/daos_cont.h index 27d416c8f89..c3bda9837d8 100644 --- a/src/include/daos_cont.h +++ b/src/include/daos_cont.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020-2023 Intel Corporation. + * (C) Copyright 2020-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -441,6 +441,7 @@ daos_cont_delete_acl(daos_handle_t coh, enum daos_acl_principal_type type, * 0 Success * -DER_INVAL Invalid parameter * -DER_NO_PERM Permission denied + * -DER_NONEXIST User or group does not exist * -DER_UNREACH Network is unreachable * -DER_NO_HDL Invalid container handle * -DER_NOMEM Out of memory @@ -449,6 +450,29 @@ int daos_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group, daos_event_t *ev); +/** + * Update a container's owner user and/or owner group, without verifying the user/group exists + * locally on the machine. + * + * \param[in] coh Container handle + * \param[in] user New owner user (NULL if not updating) + * \param[in] group New owner group (NULL if not updating) + * \param[in] ev Completion event, it is optional and can be NULL. + * The function will run in blocking mode if \a ev is NULL. + * + * \return These values will be returned by \a ev::ev_error in + * non-blocking mode: + * 0 Success + * -DER_INVAL Invalid parameter + * -DER_NO_PERM Permission denied + * -DER_UNREACH Network is unreachable + * -DER_NO_HDL Invalid container handle + * -DER_NOMEM Out of memory + */ +int +daos_cont_set_owner_no_check(daos_handle_t coh, d_string_t user, d_string_t group, + daos_event_t *ev); + /** * List the names of all user-defined container attributes. * diff --git a/src/include/daos_fs.h b/src/include/daos_fs.h index d5df87c5b77..ceface7d00e 100644 --- a/src/include/daos_fs.h +++ b/src/include/daos_fs.h @@ -1187,12 +1187,14 @@ dfs_cont_check(daos_handle_t poh, const char *cont, uint64_t flags, const char * * * \param[in] coh Open container handle * \param[in] user New owner user (NULL if not updating) + * \param[in] uid New owner uid, if different from user's on local system (-1 otherwise) * \param[in] group New owner group (NULL if not updating) + * \param[in] gid New owner gid, if different from group's on local system (-1 otherwise) * * \return 0 on success, errno code on failure. */ int -dfs_cont_set_owner(daos_handle_t coh, d_string_t user, d_string_t group); +dfs_cont_set_owner(daos_handle_t coh, d_string_t user, uid_t uid, d_string_t group, gid_t gid); /* * The Pipeline DFS API (everything under this comment) is under heavy development and should not be diff --git a/src/include/daos_pool.h b/src/include/daos_pool.h index 99173ea6638..73f44368913 100644 --- a/src/include/daos_pool.h +++ b/src/include/daos_pool.h @@ -155,15 +155,15 @@ struct daos_rebuild_status { */ enum daos_pool_info_bit { /** true to query pool space usage false to not query space usage. */ - DPI_SPACE = 1ULL << 0, + DPI_SPACE = 1ULL << 0, /** true to query pool rebuild status. false to not query rebuild status. */ - DPI_REBUILD_STATUS = 1ULL << 1, - /** true to return (in \a ranks) engines with all targets enabled (up or draining). - * false to return (in \a ranks) the engines with some or all targets disabled (down). - */ - DPI_ENGINES_ENABLED = 1ULL << 2, + DPI_REBUILD_STATUS = 1ULL << 1, + /** true to include (in \a ranks) engines with all targets enabled (up or draining). */ + DPI_ENGINES_ENABLED = 1ULL << 2, + /** true to include (in \a ranks) engines with some or all targets disabled (down). */ + DPI_ENGINES_DISABLED = 1ULL << 3, /** query all above optional info */ - DPI_ALL = -1, + DPI_ALL = -1, }; /** diff --git a/src/include/daos_srv/container.h b/src/include/daos_srv/container.h index c65c2054f3e..3a4868230b4 100644 --- a/src/include/daos_srv/container.h +++ b/src/include/daos_srv/container.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2015-2023 Intel Corporation. + * (C) Copyright 2015-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -67,6 +67,7 @@ struct ds_cont_child { uint32_t sc_dtx_resyncing:1, sc_dtx_reindex:1, sc_dtx_reindex_abort:1, + sc_dtx_delay_reset:1, sc_dtx_registered:1, sc_props_fetched:1, sc_stopping:1, @@ -185,7 +186,6 @@ int ds_cont_close_by_pool_hdls(uuid_t pool_uuid, uuid_t *pool_hdls, int n_pool_hdls, crt_context_t ctx); int ds_cont_local_close(uuid_t cont_hdl_uuid); -int ds_cont_chk_post(struct ds_pool_child *pool_child); int ds_cont_child_start_all(struct ds_pool_child *pool_child); void ds_cont_child_stop_all(struct ds_pool_child *pool_child); diff --git a/src/include/daos_srv/daos_chk.h b/src/include/daos_srv/daos_chk.h index 756c5ec0cd8..59e34305d28 100644 --- a/src/include/daos_srv/daos_chk.h +++ b/src/include/daos_srv/daos_chk.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -86,4 +86,6 @@ int chk_leader_prop(chk_prop_cb_t prop_cb, void *buf); int chk_leader_act(uint64_t seq, uint32_t act, bool for_all); +int chk_engine_pool_stop(uuid_t pool_uuid, bool destroy); + #endif /* __DAOS_CHK_H__ */ diff --git a/src/include/daos_srv/dtx_srv.h b/src/include/daos_srv/dtx_srv.h index 32df91ac949..e62dbda43eb 100644 --- a/src/include/daos_srv/dtx_srv.h +++ b/src/include/daos_srv/dtx_srv.h @@ -323,12 +323,14 @@ dtx_leader_exec_ops(struct dtx_leader_handle *dlh, dtx_sub_func_t func, int dtx_cont_open(struct ds_cont_child *cont); -void dtx_cont_close(struct ds_cont_child *cont); +void dtx_cont_close(struct ds_cont_child *cont, bool force); int dtx_cont_register(struct ds_cont_child *cont); void dtx_cont_deregister(struct ds_cont_child *cont); +void stop_dtx_reindex_ult(struct ds_cont_child *cont, bool force); + int dtx_obj_sync(struct ds_cont_child *cont, daos_unit_oid_t *oid, daos_epoch_t epoch); diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index 325cd1efc5b..8c63e469beb 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -82,8 +82,11 @@ struct ds_pool { */ uuid_t sp_srv_cont_hdl; uuid_t sp_srv_pool_hdl; - uint32_t sp_stopping : 1, sp_fetch_hdls : 1, sp_disable_rebuild : 1, sp_need_discard : 1; - + uint32_t sp_stopping:1, + sp_cr_checked:1, + sp_fetch_hdls:1, + sp_need_discard:1, + sp_disable_rebuild:1; /* pool_uuid + map version + leader term + rebuild generation define a * rebuild job. */ @@ -252,7 +255,7 @@ void ds_pool_child_put(struct ds_pool_child *child); /* Start ds_pool child */ int ds_pool_child_start(uuid_t pool_uuid, bool recreate); /* Stop ds_pool_child */ -int ds_pool_child_stop(uuid_t pool_uuid); +int ds_pool_child_stop(uuid_t pool_uuid, bool free); /* Query pool child state */ uint32_t ds_pool_child_state(uuid_t pool_uuid, uint32_t tgt_id); @@ -272,15 +275,15 @@ int ds_pool_tgt_finish_rebuild(uuid_t pool_uuid, struct pool_target_id_list *lis int ds_pool_tgt_map_update(struct ds_pool *pool, struct pool_buf *buf, unsigned int map_version); -int ds_pool_chk_post(uuid_t uuid); -int ds_pool_start_with_svc(uuid_t uuid); -int ds_pool_start(uuid_t uuid); +bool ds_pool_skip_for_check(struct ds_pool *pool); +int ds_pool_start_after_check(uuid_t uuid); +int ds_pool_start(uuid_t uuid, bool aft_chk); void ds_pool_stop(uuid_t uuid); -int ds_pool_extend(uuid_t pool_uuid, int ntargets, const d_rank_list_t *rank_list, int ndomains, - const uint32_t *domains, d_rank_list_t *svc_ranks); -int ds_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *ranks, - struct pool_target_addr_list *target_list, - pool_comp_state_t state); +int dsc_pool_svc_extend(uuid_t pool_uuid, d_rank_list_t *svc_ranks, uint64_t deadline, int ntargets, + const d_rank_list_t *rank_list, int ndomains, const uint32_t *domains); +int dsc_pool_svc_update_target_state(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + struct pool_target_addr_list *target_list, + pool_comp_state_t state); int ds_pool_svc_dist_create(const uuid_t pool_uuid, int ntargets, const char *group, @@ -290,25 +293,25 @@ int ds_pool_svc_stop(uuid_t pool_uuid); int ds_pool_svc_rf_to_nreplicas(int svc_rf); int ds_pool_svc_rf_from_nreplicas(int nreplicas); -int ds_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks, - daos_prop_t *prop); -int ds_pool_svc_set_prop(uuid_t pool_uuid, d_rank_list_t *ranks, - daos_prop_t *prop); -int ds_pool_svc_update_acl(uuid_t pool_uuid, d_rank_list_t *ranks, - struct daos_acl *acl); -int ds_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, - enum daos_acl_principal_type principal_type, - const char *principal_name); +int dsc_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + daos_prop_t *prop); +int dsc_pool_svc_set_prop(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + daos_prop_t *prop); +int dsc_pool_svc_update_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + struct daos_acl *acl); +int dsc_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + enum daos_acl_principal_type principal_type, + const char *principal_name); int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, d_rank_list_t **ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver); -int ds_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, d_rank_t rank, - uint32_t tgt_idx, daos_target_info_t *ti); +int dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, + d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti); int ds_pool_prop_fetch(struct ds_pool *pool, unsigned int bit, daos_prop_t **prop_out); -int ds_pool_svc_upgrade(uuid_t pool_uuid, d_rank_list_t *ranks); +int dsc_pool_svc_upgrade(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline); int ds_pool_failed_add(uuid_t uuid, int rc); void ds_pool_failed_remove(uuid_t uuid); int ds_pool_failed_lookup(uuid_t uuid); @@ -367,10 +370,9 @@ int ds_pool_svc_list_cont(uuid_t uuid, d_rank_list_t *ranks, struct daos_pool_cont_info **containers, uint64_t *ncontainers); -int ds_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks, - uuid_t *handles, size_t n_handles, - uint32_t destroy, uint32_t force, - char *machine, uint32_t *count); +int dsc_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + uuid_t *handles, size_t n_handles, uint32_t destroy, uint32_t force, + char *machine, uint32_t *count); int ds_pool_target_status_check(struct ds_pool *pool, uint32_t id, uint8_t matched_status, struct pool_target **p_tgt); diff --git a/src/include/daos_srv/rdb.h b/src/include/daos_srv/rdb.h index 9d1f809ed00..9ee58895332 100644 --- a/src/include/daos_srv/rdb.h +++ b/src/include/daos_srv/rdb.h @@ -188,8 +188,7 @@ int rdb_campaign(struct rdb *db); bool rdb_is_leader(struct rdb *db, uint64_t *term); int rdb_get_leader(struct rdb *db, uint64_t *term, d_rank_t *rank); int rdb_get_ranks(struct rdb *db, d_rank_list_t **ranksp); -int - rdb_get_size(struct rdb *db, size_t *sizep); +int rdb_get_size(struct rdb *db, size_t *sizep); int rdb_add_replicas(struct rdb *db, d_rank_list_t *replicas); int rdb_remove_replicas(struct rdb *db, d_rank_list_t *replicas); int rdb_ping(struct rdb *db, uint64_t caller_term); @@ -264,8 +263,7 @@ struct rdb_tx { /** TX methods */ int rdb_tx_begin(struct rdb *db, uint64_t term, struct rdb_tx *tx); int rdb_tx_begin_local(struct rdb_storage *storage, struct rdb_tx *tx); -void - rdb_tx_discard(struct rdb_tx *tx); +void rdb_tx_discard(struct rdb_tx *tx); int rdb_tx_commit(struct rdb_tx *tx); void rdb_tx_end(struct rdb_tx *tx); diff --git a/src/include/daos_srv/vos_types.h b/src/include/daos_srv/vos_types.h index 4620bdaa436..3d0e8f1e9b2 100644 --- a/src/include/daos_srv/vos_types.h +++ b/src/include/daos_srv/vos_types.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2015-2023 Intel Corporation. + * (C) Copyright 2015-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -353,29 +353,31 @@ D_CASSERT((VOS_USE_TIMESTAMPS & (VOS_GET_MAX | VOS_GET_MIN | VOS_GET_DKEY | enum { /** The absence of any flags means iterate all unsorted extents */ - VOS_IT_RECX_ALL = 0, + VOS_IT_RECX_ALL = 0, /** Include visible extents in sorted iteration */ - VOS_IT_RECX_VISIBLE = (1 << 0), + VOS_IT_RECX_VISIBLE = (1 << 0), /** Include covered extents, implies VOS_IT_RECX_VISIBLE */ - VOS_IT_RECX_COVERED = (1 << 1) | VOS_IT_RECX_VISIBLE, + VOS_IT_RECX_COVERED = (1 << 1) | VOS_IT_RECX_VISIBLE, /** Include hole extents in sorted iteration * Only applicable if VOS_IT_RECX_COVERED is not set */ - VOS_IT_RECX_SKIP_HOLES = (1 << 2), + VOS_IT_RECX_SKIP_HOLES = (1 << 2), /** When sorted iteration is enabled, iterate in reverse */ - VOS_IT_RECX_REVERSE = (1 << 3), + VOS_IT_RECX_REVERSE = (1 << 3), /** The iterator is for purge operation */ - VOS_IT_FOR_PURGE = (1 << 4), + VOS_IT_FOR_PURGE = (1 << 4), /** The iterator is for data migration scan */ - VOS_IT_FOR_MIGRATION = (1 << 5), + VOS_IT_FOR_MIGRATION = (1 << 5), /** Iterate only show punched records in interval */ - VOS_IT_PUNCHED = (1 << 6), + VOS_IT_PUNCHED = (1 << 6), /** Cleanup stale DTX entry. */ - VOS_IT_FOR_DISCARD = (1 << 7), + VOS_IT_FOR_DISCARD = (1 << 7), /** Entry is not committed */ - VOS_IT_UNCOMMITTED = (1 << 8), + VOS_IT_UNCOMMITTED = (1 << 8), + /** The iterator is for an aggregation operation (EC or VOS) */ + VOS_IT_FOR_AGG = (1 << 9), /** Mask for all flags */ - VOS_IT_MASK = (1 << 9) - 1, + VOS_IT_MASK = (1 << 10) - 1, }; typedef struct { diff --git a/src/mgmt/cli_mgmt.c b/src/mgmt/cli_mgmt.c index 290da6140e0..9ad9d760b6d 100644 --- a/src/mgmt/cli_mgmt.c +++ b/src/mgmt/cli_mgmt.c @@ -517,10 +517,10 @@ _split_env(char *env, char **name, char **value) * via the get_attach_info() dRPC. * Configure the client's local environment with these parameters */ -int dc_mgmt_net_cfg(const char *name) +int +dc_mgmt_net_cfg(const char *name, crt_init_options_t *crt_info) { int rc; - char *provider; char *cli_srx_set = NULL; char *crt_timeout = NULL; char buf[SYS_INFO_BUF_SIZE]; @@ -554,11 +554,6 @@ int dc_mgmt_net_cfg(const char *name) /* Save number of server ranks */ g_num_serv_ranks = resp->n_rank_uris; D_INFO("Setting number of server ranks to %d\n", g_num_serv_ranks); - /* These two are always set */ - provider = info->provider; - rc = d_setenv("D_PROVIDER", provider, 1); - if (rc != 0) - D_GOTO(cleanup, rc = d_errno2der(errno)); /* If the server has set this, the client must use the same value. */ if (info->srv_srx_set != -1) { @@ -584,42 +579,39 @@ int dc_mgmt_net_cfg(const char *name) /* Allow client env overrides for these three */ d_agetenv_str(&crt_timeout, "CRT_TIMEOUT"); if (!crt_timeout) { - rc = asprintf(&crt_timeout, "%d", info->crt_timeout); - if (rc < 0) { - crt_timeout = NULL; - D_GOTO(cleanup, rc = -DER_NOMEM); - } - D_INFO("setenv CRT_TIMEOUT=%s\n", crt_timeout); - rc = d_setenv("CRT_TIMEOUT", crt_timeout, 1); - if (rc != 0) - D_GOTO(cleanup, rc = d_errno2der(errno)); + crt_info->cio_crt_timeout = info->crt_timeout; } else { + crt_info->cio_crt_timeout = atoi(crt_timeout); D_DEBUG(DB_MGMT, "Using client provided CRT_TIMEOUT: %s\n", crt_timeout); } - /* client-provided iface/domain were already taken into account by agent */ - /* TODO: These should be set in crt_init_options_t instead of env */ - rc = d_setenv("D_INTERFACE", info->interface, 1); - if (rc != 0) - D_GOTO(cleanup, rc = d_errno2der(errno)); - - rc = d_setenv("D_DOMAIN", info->domain, 1); - if (rc != 0) - D_GOTO(cleanup, rc = d_errno2der(errno)); - sprintf(buf, "%d", info->provider_idx); rc = d_setenv("CRT_SECONDARY_PROVIDER", buf, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); + D_STRNDUP(crt_info->cio_provider, info->provider, DAOS_SYS_INFO_STRING_MAX); + if (NULL == crt_info->cio_provider) + D_GOTO(cleanup, rc = -DER_NOMEM); + D_STRNDUP(crt_info->cio_interface, info->interface, DAOS_SYS_INFO_STRING_MAX); + if (NULL == crt_info->cio_interface) + D_GOTO(cleanup, rc = -DER_NOMEM); + D_STRNDUP(crt_info->cio_domain, info->domain, DAOS_SYS_INFO_STRING_MAX); + if (NULL == crt_info->cio_domain) + D_GOTO(cleanup, rc = -DER_NOMEM); + D_INFO("Network interface: %s, Domain: %s\n", info->interface, info->domain); D_DEBUG(DB_MGMT, "CaRT initialization with:\n" - "\tD_PROVIDER: %s, CRT_TIMEOUT: %s, " - "CRT_SECONDARY_PROVIDER: %s\n", - provider, crt_timeout, buf); + "\tD_PROVIDER: %s, CRT_TIMEOUT: %d, CRT_SECONDARY_PROVIDER: %s\n", + crt_info->cio_provider, crt_info->cio_crt_timeout, buf); cleanup: + if (rc) { + D_FREE(crt_info->cio_provider); + D_FREE(crt_info->cio_interface); + D_FREE(crt_info->cio_domain); + } d_freeenv_str(&crt_timeout); d_freeenv_str(&cli_srx_set); diff --git a/src/mgmt/pool.pb-c.c b/src/mgmt/pool.pb-c.c index 196fcd67eb5..7724d6732f2 100644 --- a/src/mgmt/pool.pb-c.c +++ b/src/mgmt/pool.pb-c.c @@ -1587,7 +1587,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_create_req__field_descriptors[1 offsetof(Mgmt__PoolCreateReq, faultdomains), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -1623,7 +1623,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_create_req__field_descriptors[1 offsetof(Mgmt__PoolCreateReq, tierratio), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -1647,7 +1647,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_create_req__field_descriptors[1 offsetof(Mgmt__PoolCreateReq, ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -1659,7 +1659,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_create_req__field_descriptors[1 offsetof(Mgmt__PoolCreateReq, tierbytes), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -1746,7 +1746,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_create_resp__field_descriptors[ offsetof(Mgmt__PoolCreateResp, svc_reps), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -1758,7 +1758,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_create_resp__field_descriptors[ offsetof(Mgmt__PoolCreateResp, tgt_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -1770,7 +1770,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_create_resp__field_descriptors[ offsetof(Mgmt__PoolCreateResp, tier_bytes), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -1861,7 +1861,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_destroy_req__field_descriptors[ offsetof(Mgmt__PoolDestroyReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -1977,7 +1977,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_evict_req__field_descriptors[7] offsetof(Mgmt__PoolEvictReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2156,7 +2156,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_exclude_req__field_descriptors[ offsetof(Mgmt__PoolExcludeReq, targetidx), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2168,7 +2168,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_exclude_req__field_descriptors[ offsetof(Mgmt__PoolExcludeReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, }; @@ -2284,7 +2284,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_drain_req__field_descriptors[5] offsetof(Mgmt__PoolDrainReq, targetidx), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2296,7 +2296,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_drain_req__field_descriptors[5] offsetof(Mgmt__PoolDrainReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, }; @@ -2400,7 +2400,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_extend_req__field_descriptors[7 offsetof(Mgmt__PoolExtendReq, ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2412,7 +2412,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_extend_req__field_descriptors[7 offsetof(Mgmt__PoolExtendReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2424,7 +2424,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_extend_req__field_descriptors[7 offsetof(Mgmt__PoolExtendReq, tierbytes), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2436,7 +2436,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_extend_req__field_descriptors[7 offsetof(Mgmt__PoolExtendReq, faultdomains), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2504,7 +2504,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_extend_resp__field_descriptors[ offsetof(Mgmt__PoolExtendResp, tier_bytes), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2592,7 +2592,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_reintegrate_req__field_descript offsetof(Mgmt__PoolReintegrateReq, targetidx), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2604,7 +2604,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_reintegrate_req__field_descript offsetof(Mgmt__PoolReintegrateReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2616,7 +2616,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_reintegrate_req__field_descript offsetof(Mgmt__PoolReintegrateReq, tierbytes), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2772,7 +2772,7 @@ static const ProtobufCFieldDescriptor mgmt__list_pools_resp__pool__field_descrip offsetof(Mgmt__ListPoolsResp__Pool, svc_reps), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -2926,7 +2926,7 @@ static const ProtobufCFieldDescriptor mgmt__list_cont_req__field_descriptors[3] offsetof(Mgmt__ListContReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, }; @@ -3044,7 +3044,7 @@ const ProtobufCMessageDescriptor mgmt__list_cont_resp__descriptor = (ProtobufCMessageInit) mgmt__list_cont_resp__init, NULL,NULL,NULL /* reserved[123] */ }; -static const ProtobufCFieldDescriptor mgmt__pool_query_req__field_descriptors[5] = +static const ProtobufCFieldDescriptor mgmt__pool_query_req__field_descriptors[4] = { { "sys", @@ -3079,28 +3079,16 @@ static const ProtobufCFieldDescriptor mgmt__pool_query_req__field_descriptors[5] offsetof(Mgmt__PoolQueryReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ - 0,NULL,NULL /* reserved1,reserved2, etc */ - }, - { - "include_enabled_ranks", - 4, - PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_BOOL, - 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryReq, include_enabled_ranks), - NULL, - NULL, 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { - "include_disabled_ranks", - 5, + "query_mask", + 4, PROTOBUF_C_LABEL_NONE, - PROTOBUF_C_TYPE_BOOL, + PROTOBUF_C_TYPE_UINT64, 0, /* quantifier_offset */ - offsetof(Mgmt__PoolQueryReq, include_disabled_ranks), + offsetof(Mgmt__PoolQueryReq, query_mask), NULL, NULL, 0, /* flags */ @@ -3109,15 +3097,14 @@ static const ProtobufCFieldDescriptor mgmt__pool_query_req__field_descriptors[5] }; static const unsigned mgmt__pool_query_req__field_indices_by_name[] = { 1, /* field[1] = id */ - 4, /* field[4] = include_disabled_ranks */ - 3, /* field[3] = include_enabled_ranks */ + 3, /* field[3] = query_mask */ 2, /* field[2] = svc_ranks */ 0, /* field[0] = sys */ }; static const ProtobufCIntRange mgmt__pool_query_req__number_ranges[1 + 1] = { { 1, 0 }, - { 0, 5 } + { 0, 4 } }; const ProtobufCMessageDescriptor mgmt__pool_query_req__descriptor = { @@ -3127,7 +3114,7 @@ const ProtobufCMessageDescriptor mgmt__pool_query_req__descriptor = "Mgmt__PoolQueryReq", "mgmt", sizeof(Mgmt__PoolQueryReq), - 5, + 4, mgmt__pool_query_req__field_descriptors, mgmt__pool_query_req__field_indices_by_name, 1, mgmt__pool_query_req__number_ranges, @@ -3344,7 +3331,7 @@ const ProtobufCMessageDescriptor mgmt__pool_rebuild_status__descriptor = (ProtobufCMessageInit) mgmt__pool_rebuild_status__init, NULL,NULL,NULL /* reserved[123] */ }; -static const ProtobufCFieldDescriptor mgmt__pool_query_resp__field_descriptors[16] = +static const ProtobufCFieldDescriptor mgmt__pool_query_resp__field_descriptors[19] = { { "status", @@ -3538,6 +3525,42 @@ static const ProtobufCFieldDescriptor mgmt__pool_query_resp__field_descriptors[1 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, + { + "svc_ldr", + 18, + PROTOBUF_C_LABEL_NONE, + PROTOBUF_C_TYPE_UINT32, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, svc_ldr), + NULL, + NULL, + 0, /* flags */ + 0,NULL,NULL /* reserved1,reserved2, etc */ + }, + { + "svc_reps", + 19, + PROTOBUF_C_LABEL_REPEATED, + PROTOBUF_C_TYPE_UINT32, + offsetof(Mgmt__PoolQueryResp, n_svc_reps), + offsetof(Mgmt__PoolQueryResp, svc_reps), + NULL, + NULL, + 0, /* flags */ + 0,NULL,NULL /* reserved1,reserved2, etc */ + }, + { + "query_mask", + 20, + PROTOBUF_C_LABEL_NONE, + PROTOBUF_C_TYPE_UINT64, + 0, /* quantifier_offset */ + offsetof(Mgmt__PoolQueryResp, query_mask), + NULL, + NULL, + 0, /* flags */ + 0,NULL,NULL /* reserved1,reserved2, etc */ + }, }; static const unsigned mgmt__pool_query_resp__field_indices_by_name[] = { 4, /* field[4] = active_targets */ @@ -3547,9 +3570,12 @@ static const unsigned mgmt__pool_query_resp__field_indices_by_name[] = { 2, /* field[2] = label */ 9, /* field[9] = leader */ 13, /* field[13] = pool_layout_ver */ + 18, /* field[18] = query_mask */ 6, /* field[6] = rebuild */ 15, /* field[15] = state */ 0, /* field[0] = status */ + 16, /* field[16] = svc_ldr */ + 17, /* field[17] = svc_reps */ 7, /* field[7] = tier_stats */ 12, /* field[12] = total_engines */ 3, /* field[3] = total_targets */ @@ -3561,7 +3587,7 @@ static const ProtobufCIntRange mgmt__pool_query_resp__number_ranges[2 + 1] = { { 1, 0 }, { 10, 8 }, - { 0, 16 } + { 0, 19 } }; const ProtobufCMessageDescriptor mgmt__pool_query_resp__descriptor = { @@ -3571,7 +3597,7 @@ const ProtobufCMessageDescriptor mgmt__pool_query_resp__descriptor = "Mgmt__PoolQueryResp", "mgmt", sizeof(Mgmt__PoolQueryResp), - 16, + 19, mgmt__pool_query_resp__field_descriptors, mgmt__pool_query_resp__field_indices_by_name, 2, mgmt__pool_query_resp__number_ranges, @@ -3689,7 +3715,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_set_prop_req__field_descriptors offsetof(Mgmt__PoolSetPropReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, }; @@ -3804,7 +3830,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_get_prop_req__field_descriptors offsetof(Mgmt__PoolGetPropReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, }; @@ -3920,7 +3946,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_upgrade_req__field_descriptors[ offsetof(Mgmt__PoolUpgradeReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, }; @@ -4034,7 +4060,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_query_target_req__field_descrip offsetof(Mgmt__PoolQueryTargetReq, targets), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, { @@ -4046,7 +4072,7 @@ static const ProtobufCFieldDescriptor mgmt__pool_query_target_req__field_descrip offsetof(Mgmt__PoolQueryTargetReq, svc_ranks), NULL, NULL, - 0 | PROTOBUF_C_FIELD_FLAG_PACKED, /* flags */ + 0, /* flags */ 0,NULL,NULL /* reserved1,reserved2, etc */ }, }; diff --git a/src/mgmt/pool.pb-c.h b/src/mgmt/pool.pb-c.h index ece6cde9853..08ae342596f 100644 --- a/src/mgmt/pool.pb-c.h +++ b/src/mgmt/pool.pb-c.h @@ -10,7 +10,7 @@ PROTOBUF_C__BEGIN_DECLS #if PROTOBUF_C_VERSION_NUMBER < 1003000 # error This file was generated by a newer version of protoc-c which is incompatible with your libprotobuf-c headers. Please update your headers. -#elif 1003003 < PROTOBUF_C_MIN_COMPILER_VERSION +#elif 1003000 < PROTOBUF_C_MIN_COMPILER_VERSION # error This file was generated by an older version of protoc-c which is incompatible with your libprotobuf-c headers. Please regenerate this file with a newer version of protoc-c. #endif @@ -743,17 +743,13 @@ struct _Mgmt__PoolQueryReq size_t n_svc_ranks; uint32_t *svc_ranks; /* - * True if the list of enabled ranks shall be returned + * Bitmask of pool query options */ - protobuf_c_boolean include_enabled_ranks; - /* - * True if the list of disabled ranks shall be returned - */ - protobuf_c_boolean include_disabled_ranks; + uint64_t query_mask; }; #define MGMT__POOL_QUERY_REQ__INIT \ { PROTOBUF_C_MESSAGE_INIT (&mgmt__pool_query_req__descriptor) \ - , (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0,NULL, 0, 0 } + , (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0,NULL, 0 } /* @@ -837,7 +833,7 @@ struct _Mgmt__PoolQueryResp */ uint32_t version; /* - * current raft leader + * current raft leader (2.4) */ uint32_t leader; /* @@ -864,10 +860,23 @@ struct _Mgmt__PoolQueryResp * pool state */ Mgmt__PoolServiceState state; + /* + * current raft leader (2.6+) + */ + uint32_t svc_ldr; + /* + * service replica ranks + */ + size_t n_svc_reps; + uint32_t *svc_reps; + /* + * Bitmask of pool query options used + */ + uint64_t query_mask; }; #define MGMT__POOL_QUERY_RESP__INIT \ { PROTOBUF_C_MESSAGE_INIT (&mgmt__pool_query_resp__descriptor) \ - , 0, (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0, 0, 0, NULL, 0,NULL, 0, 0, (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0, 0, 0, MGMT__POOL_SERVICE_STATE__Creating } + , 0, (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0, 0, 0, NULL, 0,NULL, 0, 0, (char *)protobuf_c_empty_string, (char *)protobuf_c_empty_string, 0, 0, 0, MGMT__POOL_SERVICE_STATE__Creating, 0, 0,NULL, 0 } typedef enum { diff --git a/src/mgmt/srv_drpc.c b/src/mgmt/srv_drpc.c index 574b6ecb344..6a37c4d85a4 100644 --- a/src/mgmt/srv_drpc.c +++ b/src/mgmt/srv_drpc.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1770,12 +1770,17 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) goto out; } + /* TODO (DAOS-10250) Enabled and disabled engines should be retrieve both if needed */ + if (req->query_mask & DPI_ENGINES_ENABLED && req->query_mask & DPI_ENGINES_DISABLED) { + D_ERROR("cannot query enabled and disabled engines in the same request\n"); + D_GOTO(out, rc = -DER_NOTSUPPORTED); + } + svc_ranks = uint32_array_to_rank_list(req->svc_ranks, req->n_svc_ranks); if (svc_ranks == NULL) D_GOTO(out, rc = -DER_NOMEM); - /* TODO (DAOS-10250) Enabled and disabled engines should be retrieve both if needed */ - pool_info.pi_bits = req->include_enabled_ranks ? DPI_ALL : (DPI_ALL & ~DPI_ENGINES_ENABLED); + pool_info.pi_bits = req->query_mask; rc = ds_mgmt_pool_query(uuid, svc_ranks, &ranks, &pool_info, &resp.pool_layout_ver, &resp.upgrade_layout_ver); if (rc != 0) { @@ -1795,15 +1800,18 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) truncated ? " ...(TRUNCATED)" : ""); /* Populate the response */ + resp.query_mask = pool_info.pi_bits; resp.uuid = req->id; resp.total_targets = pool_info.pi_ntargets; resp.disabled_targets = pool_info.pi_ndisabled; resp.active_targets = pool_info.pi_space.ps_ntargets; resp.total_engines = pool_info.pi_nnodes; - resp.leader = pool_info.pi_leader; + resp.svc_ldr = pool_info.pi_leader; + resp.svc_reps = req->svc_ranks; + resp.n_svc_reps = req->n_svc_ranks; resp.version = pool_info.pi_map_ver; - resp.enabled_ranks = (req->include_enabled_ranks) ? range_list_str : ""; - resp.disabled_ranks = (req->include_disabled_ranks) ? range_list_str : ""; + resp.enabled_ranks = (req->query_mask & DPI_ENGINES_ENABLED) ? range_list_str : ""; + resp.disabled_ranks = (req->query_mask & DPI_ENGINES_DISABLED) ? range_list_str : ""; D_ALLOC_ARRAY(resp.tier_stats, DAOS_MEDIA_MAX); if (resp.tier_stats == NULL) { diff --git a/src/mgmt/srv_internal.h b/src/mgmt/srv_internal.h index ac6217be536..df4e24af53b 100644 --- a/src/mgmt/srv_internal.h +++ b/src/mgmt/srv_internal.h @@ -31,6 +31,18 @@ #include "rpc.h" #include "srv_layout.h" +/* + * Use a fixed timeout that matches what the control plane uses for the + * moment. + * + * TODO: Pass the deadline from dmg (or daos_server). + */ +static inline uint64_t +mgmt_ps_call_deadline(void) +{ + return daos_getmtime_coarse() + 5 * 60 * 1000; +} + /** srv.c */ void ds_mgmt_hdlr_svc_rip(crt_rpc_t *rpc); void ds_mgmt_params_set_hdlr(crt_rpc_t *rpc); diff --git a/src/mgmt/srv_pool.c b/src/mgmt/srv_pool.c index 27d30390b0c..c7f71b4d3ac 100644 --- a/src/mgmt/srv_pool.c +++ b/src/mgmt/srv_pool.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -308,7 +308,8 @@ ds_mgmt_pool_extend(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t *r /* TODO: Need to make pool service aware of new rank UUIDs */ ntargets = unique_add_ranks->rl_nr; - rc = ds_pool_extend(pool_uuid, ntargets, unique_add_ranks, domains_nr, domains, svc_ranks); + rc = dsc_pool_svc_extend(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), ntargets, + unique_add_ranks, domains_nr, domains); out: d_rank_list_free(unique_add_ranks); return rc; @@ -323,8 +324,8 @@ ds_mgmt_evict_pool(uuid_t pool_uuid, d_rank_list_t *svc_ranks, uuid_t *handles, D_DEBUG(DB_MGMT, "evict pool "DF_UUID"\n", DP_UUID(pool_uuid)); /* Evict active pool connections if they exist*/ - rc = ds_pool_svc_check_evict(pool_uuid, svc_ranks, handles, n_handles, - destroy, force_destroy, machine, count); + rc = dsc_pool_svc_check_evict(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), handles, + n_handles, destroy, force_destroy, machine, count); if (rc != 0) { D_ERROR("Failed to evict pool handles" DF_UUID " rc: " DF_RC "\n", DP_UUID(pool_uuid), DP_RC(rc)); @@ -366,7 +367,8 @@ ds_mgmt_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *svc_ranks, } } - rc = ds_pool_target_update_state(pool_uuid, svc_ranks, target_addrs, state); + rc = dsc_pool_svc_update_target_state(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), + target_addrs, state); return rc; } @@ -409,8 +411,6 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **r daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) { - uint64_t deadline; - if (pool_info == NULL) { D_ERROR("pool_info was NULL\n"); return -DER_INVAL; @@ -418,16 +418,8 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **r D_DEBUG(DB_MGMT, "Querying pool "DF_UUID"\n", DP_UUID(pool_uuid)); - /* - * Use a fixed timeout that matches what the control plane uses for the - * moment. - * - * TODO: Pass the deadline from dmg (or daos_server). - */ - deadline = daos_getmtime_coarse() + 5 * 60 * 1000; - - return dsc_pool_svc_query(pool_uuid, svc_ranks, deadline, ranks, pool_info, pool_layout_ver, - upgrade_layout_ver); + return dsc_pool_svc_query(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), ranks, pool_info, + pool_layout_ver, upgrade_layout_ver); } /** @@ -464,10 +456,10 @@ ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t for (i = 0; i < tgts->rl_nr; i++) { D_DEBUG(DB_MGMT, "Querying pool "DF_UUID" rank %u tgt %u\n", DP_UUID(pool_uuid), rank, tgts->rl_ranks[i]); - rc = ds_pool_svc_query_target(pool_uuid, svc_ranks, rank, tgts->rl_ranks[i], - &out_infos[i]); + rc = dsc_pool_svc_query_target(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), rank, + tgts->rl_ranks[i], &out_infos[i]); if (rc != 0) { - D_ERROR(DF_UUID": ds_pool_svc_query_target() failed rank %u tgt %u\n", + D_ERROR(DF_UUID": dsc_pool_svc_query_target() failed rank %u tgt %u\n", DP_UUID(pool_uuid), rank, tgts->rl_ranks[i]); goto out; } @@ -500,7 +492,7 @@ get_access_props(uuid_t pool_uuid, d_rank_list_t *ranks, daos_prop_t **prop) for (i = 0; i < ACCESS_PROPS_LEN; i++) new_prop->dpp_entries[i].dpe_type = ACCESS_PROPS[i]; - rc = ds_pool_svc_get_prop(pool_uuid, ranks, new_prop); + rc = dsc_pool_svc_get_prop(pool_uuid, ranks, mgmt_ps_call_deadline(), new_prop); if (rc != 0) { daos_prop_free(new_prop); return rc; @@ -538,7 +530,7 @@ ds_mgmt_pool_overwrite_acl(uuid_t pool_uuid, d_rank_list_t *svc_ranks, prop->dpp_entries[0].dpe_type = DAOS_PROP_PO_ACL; prop->dpp_entries[0].dpe_val_ptr = daos_acl_dup(acl); - rc = ds_pool_svc_set_prop(pool_uuid, svc_ranks, prop); + rc = dsc_pool_svc_set_prop(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), prop); if (rc != 0) goto out_prop; @@ -561,7 +553,7 @@ ds_mgmt_pool_update_acl(uuid_t pool_uuid, d_rank_list_t *svc_ranks, D_DEBUG(DB_MGMT, "Updating ACL for pool "DF_UUID"\n", DP_UUID(pool_uuid)); - rc = ds_pool_svc_update_acl(pool_uuid, svc_ranks, acl); + rc = dsc_pool_svc_update_acl(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), acl); if (rc != 0) goto out; @@ -588,7 +580,7 @@ ds_mgmt_pool_delete_acl(uuid_t pool_uuid, d_rank_list_t *svc_ranks, if (rc != 0) goto out; - rc = ds_pool_svc_delete_acl(pool_uuid, svc_ranks, type, name); + rc = dsc_pool_svc_delete_acl(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), type, name); if (rc != 0) goto out_name; @@ -617,7 +609,7 @@ ds_mgmt_pool_set_prop(uuid_t pool_uuid, d_rank_list_t *svc_ranks, D_DEBUG(DB_MGMT, "Setting properties for pool "DF_UUID"\n", DP_UUID(pool_uuid)); - rc = ds_pool_svc_set_prop(pool_uuid, svc_ranks, prop); + rc = dsc_pool_svc_set_prop(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), prop); out: return rc; @@ -628,7 +620,7 @@ int ds_mgmt_pool_upgrade(uuid_t pool_uuid, d_rank_list_t *svc_ranks) D_DEBUG(DB_MGMT, "Upgrading pool "DF_UUID"\n", DP_UUID(pool_uuid)); - return ds_pool_svc_upgrade(pool_uuid, svc_ranks); + return dsc_pool_svc_upgrade(pool_uuid, svc_ranks, mgmt_ps_call_deadline()); } int @@ -646,7 +638,7 @@ ds_mgmt_pool_get_prop(uuid_t pool_uuid, d_rank_list_t *svc_ranks, D_DEBUG(DB_MGMT, "Getting properties for pool "DF_UUID"\n", DP_UUID(pool_uuid)); - rc = ds_pool_svc_get_prop(pool_uuid, svc_ranks, prop); + rc = dsc_pool_svc_get_prop(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), prop); out: return rc; @@ -667,7 +659,7 @@ ds_mgmt_tgt_pool_shard_destroy(uuid_t pool_uuid, int shard_idx, d_rank_t rank) tgt_ep.ep_grp = NULL; tgt_ep.ep_rank = rank; - tgt_ep.ep_tag = daos_rpc_tag(DAOS_REQ_MGMT, 0); + tgt_ep.ep_tag = daos_rpc_tag(DAOS_REQ_TGT, shard_idx); opc = DAOS_RPC_OPCODE(MGMT_TGT_SHARD_DESTROY, DAOS_MGMT_MODULE, DAOS_MGMT_VERSION); diff --git a/src/mgmt/srv_target.c b/src/mgmt/srv_target.c index 45ee8cb8d2d..0694d9f9e57 100644 --- a/src/mgmt/srv_target.c +++ b/src/mgmt/srv_target.c @@ -1215,7 +1215,7 @@ ds_mgmt_hdlr_tgt_create(crt_rpc_t *tc_req) tc_out->tc_ranks.ca_arrays = rank; tc_out->tc_ranks.ca_count = 1; - rc = ds_pool_start(tc_in->tc_pool_uuid); + rc = ds_pool_start(tc_in->tc_pool_uuid, false); if (rc) { D_ERROR(DF_UUID": failed to start pool: "DF_RC"\n", DP_UUID(tc_in->tc_pool_uuid), DP_RC(rc)); @@ -1386,6 +1386,15 @@ ds_mgmt_hdlr_tgt_destroy(crt_rpc_t *td_req) D_DEBUG(DB_MGMT, DF_UUID": ready to destroy targets\n", DP_UUID(td_in->td_pool_uuid)); + if (engine_in_check()) { + rc = chk_engine_pool_stop(td_in->td_pool_uuid, true); + if (rc != 0) { + D_ERROR(DF_UUID": failed to stop check engine on pool: "DF_RC"\n", + DP_UUID(td_in->td_pool_uuid), DP_RC(rc)); + goto out; + } + } + /* * If there is a local PS replica, its RDB file will be deleted later * together with the other pool files by the tgt_destroy call below; if @@ -1586,6 +1595,10 @@ ds_mgmt_hdlr_tgt_shard_destroy(crt_rpc_t *req) * stop the pool service. */ + rc = ds_pool_child_stop(tsdi->tsdi_pool_uuid, true); + if (rc != 0 && rc != -DER_NONEXIST) + goto out; + rc = ds_mgmt_tgt_file(tsdi->tsdi_pool_uuid, VOS_FILE, &tsdi->tsdi_shard_idx, &path); if (rc == 0) { rc = unlink(path); @@ -1599,6 +1612,7 @@ ds_mgmt_hdlr_tgt_shard_destroy(crt_rpc_t *req) D_FREE(path); } +out: D_DEBUG(DB_MGMT, "Processed rpc %p to destroy pool "DF_UUIDF" shard %u: "DF_RC"\n", req, DP_UUID(tsdi->tsdi_pool_uuid), tsdi->tsdi_shard_idx, DP_RC(rc)); diff --git a/src/mgmt/tests/srv_drpc_tests.c b/src/mgmt/tests/srv_drpc_tests.c index 4d577880201..4d93b5e6728 100644 --- a/src/mgmt/tests/srv_drpc_tests.c +++ b/src/mgmt/tests/srv_drpc_tests.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -35,6 +35,7 @@ #ifndef UUID_STR_LEN #define UUID_STR_LEN 37 #endif +#define DEFAULT_QUERY_BITS (DPI_ALL ^ (DPI_ENGINES_DISABLED | DPI_ENGINES_ENABLED)) static uint32_t TEST_IDXS[] = {0, 1, 2}; static uint32_t TEST_RANKS[] = {0, 1, 2}; @@ -1243,12 +1244,12 @@ pack_pool_query_req(Drpc__Call *call, Mgmt__PoolQueryReq *req) } static void -setup_pool_query_drpc_call(Drpc__Call *call, char *uuid) +setup_pool_query_drpc_call(Drpc__Call *call, char *uuid, uint64_t qflags) { Mgmt__PoolQueryReq req = MGMT__POOL_QUERY_REQ__INIT; req.id = uuid; - req.include_enabled_ranks = true; + req.query_mask = DEFAULT_QUERY_BITS | qflags; pack_pool_query_req(call, &req); } @@ -1268,13 +1269,29 @@ expect_drpc_pool_query_resp_with_error(Drpc__Response *resp, int expected_err) mgmt__pool_query_resp__free_unpacked(pq_resp, NULL); } +static void +test_drpc_pool_query_incompat_ranks_flags(void **state) +{ + Drpc__Call call = DRPC__CALL__INIT; + Drpc__Response resp = DRPC__RESPONSE__INIT; + + setup_pool_query_drpc_call(&call, TEST_UUID, DPI_ENGINES_DISABLED | DPI_ENGINES_ENABLED); + + ds_mgmt_drpc_pool_query(&call, &resp); + + expect_drpc_pool_query_resp_with_error(&resp, -DER_NOTSUPPORTED); + + D_FREE(call.body.data); + D_FREE(resp.body.data); +} + static void test_drpc_pool_query_bad_uuid(void **state) { Drpc__Call call = DRPC__CALL__INIT; Drpc__Response resp = DRPC__RESPONSE__INIT; - setup_pool_query_drpc_call(&call, "BAD"); + setup_pool_query_drpc_call(&call, "BAD", 0); ds_mgmt_drpc_pool_query(&call, &resp); @@ -1290,7 +1307,7 @@ test_drpc_pool_query_mgmt_svc_fails(void **state) Drpc__Call call = DRPC__CALL__INIT; Drpc__Response resp = DRPC__RESPONSE__INIT; - setup_pool_query_drpc_call(&call, TEST_UUID); + setup_pool_query_drpc_call(&call, TEST_UUID, 0); ds_mgmt_pool_query_return = -DER_MISC; ds_mgmt_drpc_pool_query(&call, &resp); @@ -1309,7 +1326,7 @@ init_test_pool_info(daos_pool_info_t *pool_info) if (uuid_parse(TEST_UUID, pool_info->pi_uuid)) return; - pool_info->pi_bits = DPI_ALL; + pool_info->pi_bits = DEFAULT_QUERY_BITS; /* Values are arbitrary, just want to see that they are copied over */ pool_info->pi_ntargets = 100; @@ -1412,7 +1429,7 @@ test_drpc_pool_query_success(void **state) init_test_rebuild_status(&exp_info.pi_rebuild_st); ds_mgmt_pool_query_info_out = exp_info; - setup_pool_query_drpc_call(&call, TEST_UUID); + setup_pool_query_drpc_call(&call, TEST_UUID, DPI_ENGINES_ENABLED); ds_mgmt_drpc_pool_query(&call, &resp); @@ -1422,7 +1439,8 @@ test_drpc_pool_query_success(void **state) assert_int_equal(uuid_compare(exp_uuid, ds_mgmt_pool_query_uuid), 0); assert_non_null(ds_mgmt_pool_query_info_ptr); assert_non_null(ds_mgmt_pool_query_ranks_out); - assert_int_equal(ds_mgmt_pool_query_info_in.pi_bits, DPI_ALL); + assert_int_equal(ds_mgmt_pool_query_info_in.pi_bits, + DEFAULT_QUERY_BITS | DPI_ENGINES_ENABLED); expect_query_resp_with_info(&exp_info, MGMT__POOL_REBUILD_STATUS__STATE__IDLE, @@ -1444,7 +1462,7 @@ test_drpc_pool_query_success_rebuild_busy(void **state) exp_info.pi_rebuild_st.rs_version = 1; ds_mgmt_pool_query_info_out = exp_info; - setup_pool_query_drpc_call(&call, TEST_UUID); + setup_pool_query_drpc_call(&call, TEST_UUID, 0); ds_mgmt_drpc_pool_query(&call, &resp); @@ -1469,7 +1487,7 @@ test_drpc_pool_query_success_rebuild_done(void **state) exp_info.pi_rebuild_st.rs_state = DRS_COMPLETED; ds_mgmt_pool_query_info_out = exp_info; - setup_pool_query_drpc_call(&call, TEST_UUID); + setup_pool_query_drpc_call(&call, TEST_UUID, 0); ds_mgmt_drpc_pool_query(&call, &resp); @@ -1500,7 +1518,7 @@ test_drpc_pool_query_success_rebuild_err(void **state) ds_mgmt_pool_query_info_out.pi_rebuild_st.rs_obj_nr = 42; ds_mgmt_pool_query_info_out.pi_rebuild_st.rs_rec_nr = 999; - setup_pool_query_drpc_call(&call, TEST_UUID); + setup_pool_query_drpc_call(&call, TEST_UUID, 0); ds_mgmt_drpc_pool_query(&call, &resp); @@ -3055,82 +3073,83 @@ int main(void) { const struct CMUnitTest tests[] = { - cmocka_unit_test(test_mgmt_drpc_handlers_bad_call_payload), - ACL_TEST(test_drpc_pool_get_acl_bad_uuid), - ACL_TEST(test_drpc_pool_get_acl_mgmt_svc_fails), - ACL_TEST(test_drpc_pool_get_acl_cant_translate_acl), - ACL_TEST(test_drpc_pool_get_acl_success), - ACL_TEST(test_drpc_pool_overwrite_acl_bad_uuid), - ACL_TEST(test_drpc_pool_overwrite_acl_bad_acl), - ACL_TEST(test_drpc_pool_overwrite_acl_mgmt_svc_fails), - ACL_TEST(test_drpc_pool_overwrite_acl_success), - ACL_TEST(test_drpc_pool_update_acl_bad_uuid), - ACL_TEST(test_drpc_pool_update_acl_bad_acl), - ACL_TEST(test_drpc_pool_update_acl_mgmt_svc_fails), - ACL_TEST(test_drpc_pool_update_acl_success), - ACL_TEST(test_drpc_pool_delete_acl_bad_uuid), - ACL_TEST(test_drpc_pool_delete_acl_mgmt_svc_fails), - ACL_TEST(test_drpc_pool_delete_acl_success), - LIST_CONT_TEST(test_drpc_pool_list_cont_bad_uuid), - LIST_CONT_TEST(test_drpc_pool_list_cont_mgmt_svc_fails), - LIST_CONT_TEST(test_drpc_pool_list_cont_no_containers), - LIST_CONT_TEST(test_drpc_pool_list_cont_with_containers), - POOL_SET_PROP_TEST(test_drpc_pool_set_prop_invalid_value_type), - POOL_SET_PROP_TEST(test_drpc_pool_set_prop_bad_uuid), - POOL_SET_PROP_TEST(test_drpc_pool_set_prop_success), - POOL_GET_PROP_TEST(test_drpc_pool_get_prop_bad_uuid), - POOL_GET_PROP_TEST(test_drpc_pool_get_prop_num_success), - POOL_GET_PROP_TEST(test_drpc_pool_get_prop_str_success), - POOL_GET_PROP_TEST(test_drpc_pool_get_prop_svcl_success), - POOL_GET_PROP_TEST(test_drpc_pool_get_prop_null_svcl), - EXCLUDE_TEST(test_drpc_exclude_bad_uuid), - EXCLUDE_TEST(test_drpc_exclude_mgmt_svc_fails), - EXCLUDE_TEST(test_drpc_exclude_success), - DRAIN_TEST(test_drpc_drain_bad_uuid), - DRAIN_TEST(test_drpc_drain_mgmt_svc_fails), - DRAIN_TEST(test_drpc_drain_success), - POOL_EXTEND_TEST(test_drpc_extend_bad_uuid), - POOL_EXTEND_TEST(test_drpc_extend_mgmt_svc_fails), - POOL_EXTEND_TEST(test_drpc_extend_success), - REINTEGRATE_TEST(test_drpc_reintegrate_bad_uuid), - QUERY_TEST(test_drpc_pool_query_bad_uuid), - QUERY_TEST(test_drpc_pool_query_mgmt_svc_fails), - QUERY_TEST(test_drpc_pool_query_success), - QUERY_TEST(test_drpc_pool_query_success_rebuild_busy), - QUERY_TEST(test_drpc_pool_query_success_rebuild_done), - QUERY_TEST(test_drpc_pool_query_success_rebuild_err), - QUERY_TARGETS_TEST(test_drpc_pool_query_targets_bad_uuid), - QUERY_TARGETS_TEST(test_drpc_pool_query_targets_mgmt_svc_fails), - QUERY_TARGETS_TEST(test_drpc_pool_query_targets_with_targets), - POOL_CREATE_TEST(test_drpc_pool_create_invalid_acl), - POOL_EVICT_TEST(test_drpc_pool_evict_bad_uuid), - POOL_EVICT_TEST(test_drpc_pool_evict_mgmt_svc_fails), - POOL_EVICT_TEST(test_drpc_pool_evict_success), - PING_RANK_TEST(test_drpc_ping_rank_success), - PREP_SHUTDOWN_TEST(test_drpc_prep_shutdown_success), - SET_LOG_MASKS_TEST(test_drpc_set_log_masks_success), - CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_bad_cont_uuid), - CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_bad_pool_uuid), - CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_failed), - CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_success), - POOL_UPGRADE_TEST(test_drpc_pool_upgrade_bad_uuid), - POOL_UPGRADE_TEST(test_drpc_pool_upgrade_mgmt_svc_fails), - POOL_UPGRADE_TEST(test_drpc_pool_upgrade_success), - LED_MANAGE_TEST(test_drpc_dev_manage_led_bad_tr_addr), - LED_MANAGE_TEST(test_drpc_dev_manage_led_fails), - LED_MANAGE_TEST(test_drpc_dev_manage_led_success), - DEV_REPLACE_TEST(test_drpc_dev_replace_bad_old_uuid), - DEV_REPLACE_TEST(test_drpc_dev_replace_bad_new_uuid), - DEV_REPLACE_TEST(test_drpc_dev_replace_fails), - DEV_REPLACE_TEST(test_drpc_dev_replace_success), - SET_FAULTY_TEST(test_drpc_dev_set_faulty_bad_uuid), - SET_FAULTY_TEST(test_drpc_dev_set_faulty_fails), - SET_FAULTY_TEST(test_drpc_dev_set_faulty_success), - CHECK_START_TEST(test_drpc_check_start_success), - CHECK_STOP_TEST(test_drpc_check_stop_success), - CHECK_QUERY_TEST(test_drpc_check_query_success), - CHECK_PROP_TEST(test_drpc_check_prop_success), - CHECK_ACT_TEST(test_drpc_check_act_success), + cmocka_unit_test(test_mgmt_drpc_handlers_bad_call_payload), + ACL_TEST(test_drpc_pool_get_acl_bad_uuid), + ACL_TEST(test_drpc_pool_get_acl_mgmt_svc_fails), + ACL_TEST(test_drpc_pool_get_acl_cant_translate_acl), + ACL_TEST(test_drpc_pool_get_acl_success), + ACL_TEST(test_drpc_pool_overwrite_acl_bad_uuid), + ACL_TEST(test_drpc_pool_overwrite_acl_bad_acl), + ACL_TEST(test_drpc_pool_overwrite_acl_mgmt_svc_fails), + ACL_TEST(test_drpc_pool_overwrite_acl_success), + ACL_TEST(test_drpc_pool_update_acl_bad_uuid), + ACL_TEST(test_drpc_pool_update_acl_bad_acl), + ACL_TEST(test_drpc_pool_update_acl_mgmt_svc_fails), + ACL_TEST(test_drpc_pool_update_acl_success), + ACL_TEST(test_drpc_pool_delete_acl_bad_uuid), + ACL_TEST(test_drpc_pool_delete_acl_mgmt_svc_fails), + ACL_TEST(test_drpc_pool_delete_acl_success), + LIST_CONT_TEST(test_drpc_pool_list_cont_bad_uuid), + LIST_CONT_TEST(test_drpc_pool_list_cont_mgmt_svc_fails), + LIST_CONT_TEST(test_drpc_pool_list_cont_no_containers), + LIST_CONT_TEST(test_drpc_pool_list_cont_with_containers), + POOL_SET_PROP_TEST(test_drpc_pool_set_prop_invalid_value_type), + POOL_SET_PROP_TEST(test_drpc_pool_set_prop_bad_uuid), + POOL_SET_PROP_TEST(test_drpc_pool_set_prop_success), + POOL_GET_PROP_TEST(test_drpc_pool_get_prop_bad_uuid), + POOL_GET_PROP_TEST(test_drpc_pool_get_prop_num_success), + POOL_GET_PROP_TEST(test_drpc_pool_get_prop_str_success), + POOL_GET_PROP_TEST(test_drpc_pool_get_prop_svcl_success), + POOL_GET_PROP_TEST(test_drpc_pool_get_prop_null_svcl), + EXCLUDE_TEST(test_drpc_exclude_bad_uuid), + EXCLUDE_TEST(test_drpc_exclude_mgmt_svc_fails), + EXCLUDE_TEST(test_drpc_exclude_success), + DRAIN_TEST(test_drpc_drain_bad_uuid), + DRAIN_TEST(test_drpc_drain_mgmt_svc_fails), + DRAIN_TEST(test_drpc_drain_success), + POOL_EXTEND_TEST(test_drpc_extend_bad_uuid), + POOL_EXTEND_TEST(test_drpc_extend_mgmt_svc_fails), + POOL_EXTEND_TEST(test_drpc_extend_success), + REINTEGRATE_TEST(test_drpc_reintegrate_bad_uuid), + QUERY_TEST(test_drpc_pool_query_incompat_ranks_flags), + QUERY_TEST(test_drpc_pool_query_bad_uuid), + QUERY_TEST(test_drpc_pool_query_mgmt_svc_fails), + QUERY_TEST(test_drpc_pool_query_success), + QUERY_TEST(test_drpc_pool_query_success_rebuild_busy), + QUERY_TEST(test_drpc_pool_query_success_rebuild_done), + QUERY_TEST(test_drpc_pool_query_success_rebuild_err), + QUERY_TARGETS_TEST(test_drpc_pool_query_targets_bad_uuid), + QUERY_TARGETS_TEST(test_drpc_pool_query_targets_mgmt_svc_fails), + QUERY_TARGETS_TEST(test_drpc_pool_query_targets_with_targets), + POOL_CREATE_TEST(test_drpc_pool_create_invalid_acl), + POOL_EVICT_TEST(test_drpc_pool_evict_bad_uuid), + POOL_EVICT_TEST(test_drpc_pool_evict_mgmt_svc_fails), + POOL_EVICT_TEST(test_drpc_pool_evict_success), + PING_RANK_TEST(test_drpc_ping_rank_success), + PREP_SHUTDOWN_TEST(test_drpc_prep_shutdown_success), + SET_LOG_MASKS_TEST(test_drpc_set_log_masks_success), + CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_bad_cont_uuid), + CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_bad_pool_uuid), + CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_failed), + CONT_SET_OWNER_TEST(test_drpc_cont_set_owner_success), + POOL_UPGRADE_TEST(test_drpc_pool_upgrade_bad_uuid), + POOL_UPGRADE_TEST(test_drpc_pool_upgrade_mgmt_svc_fails), + POOL_UPGRADE_TEST(test_drpc_pool_upgrade_success), + LED_MANAGE_TEST(test_drpc_dev_manage_led_bad_tr_addr), + LED_MANAGE_TEST(test_drpc_dev_manage_led_fails), + LED_MANAGE_TEST(test_drpc_dev_manage_led_success), + DEV_REPLACE_TEST(test_drpc_dev_replace_bad_old_uuid), + DEV_REPLACE_TEST(test_drpc_dev_replace_bad_new_uuid), + DEV_REPLACE_TEST(test_drpc_dev_replace_fails), + DEV_REPLACE_TEST(test_drpc_dev_replace_success), + SET_FAULTY_TEST(test_drpc_dev_set_faulty_bad_uuid), + SET_FAULTY_TEST(test_drpc_dev_set_faulty_fails), + SET_FAULTY_TEST(test_drpc_dev_set_faulty_success), + CHECK_START_TEST(test_drpc_check_start_success), + CHECK_STOP_TEST(test_drpc_check_stop_success), + CHECK_QUERY_TEST(test_drpc_check_query_success), + CHECK_PROP_TEST(test_drpc_check_prop_success), + CHECK_ACT_TEST(test_drpc_check_act_success), }; return cmocka_run_group_tests_name("mgmt_srv_drpc", tests, NULL, NULL); diff --git a/src/object/cli_obj.c b/src/object/cli_obj.c index c209bfdeeed..2c0c0a952ea 100644 --- a/src/object/cli_obj.c +++ b/src/object/cli_obj.c @@ -2216,6 +2216,22 @@ obj_iod_sgl_valid(daos_obj_id_t oid, unsigned int nr, daos_iod_t *iods, return -DER_INVAL; } } + if (sgls != NULL && sgls[i].sg_nr > 0) { + d_sg_list_t *sg = &sgls[i]; + d_iov_t *iov; + + for (j = 0; j < sg->sg_nr; j++) { + iov = sg->sg_iovs + j; + if (iov == NULL || (iov->iov_buf_len > 0 && iov->iov_buf == NULL)) { + if (iov == NULL) + D_ERROR("Bad iov - j %d, NULL iov\n", j); + else + D_ERROR("Bad iov - j %d, NULL iov_buf, " + "bul_len %zu\n", j, iov->iov_buf_len); + return -DER_INVAL; + } + } + } switch (iods[i].iod_type) { default: diff --git a/src/object/obj_internal.h b/src/object/obj_internal.h index 23717eb4af6..78c4f53712e 100644 --- a/src/object/obj_internal.h +++ b/src/object/obj_internal.h @@ -665,6 +665,8 @@ struct obj_pool_metrics { struct d_tm_node_t *opm_update_ec_full; /** Total number of EC partial update operations (type = counter) */ struct d_tm_node_t *opm_update_ec_partial; + /** Total number of EC agg conflicts with VOS aggregation or discard */ + struct d_tm_node_t *opm_ec_agg_blocked; }; void diff --git a/src/object/obj_utils.c b/src/object/obj_utils.c index 631bcea696f..7bf0ef4aaf9 100644 --- a/src/object/obj_utils.c +++ b/src/object/obj_utils.c @@ -231,6 +231,15 @@ obj_metrics_alloc_internal(const char *path, int tgt_id, bool server) if (rc) D_WARN("Failed to create EC partial update counter: " DF_RC "\n", DP_RC(rc)); + /** Total number of times EC aggregation conflicts with discard or VOS + * aggregation + */ + rc = d_tm_add_metric(&metrics->opm_ec_agg_blocked, D_TM_COUNTER, + "total number of EC agg pauses due to VOS discard or agg", NULL, + "%s/EC_agg/blocked%s", path, tgt_path); + if (rc) + D_WARN("Failed to create EC agg blocked counter: " DF_RC "\n", DP_RC(rc)); + return metrics; } diff --git a/src/object/srv_ec_aggregate.c b/src/object/srv_ec_aggregate.c index e196eb86bb4..83c52db1e1b 100644 --- a/src/object/srv_ec_aggregate.c +++ b/src/object/srv_ec_aggregate.c @@ -2608,10 +2608,12 @@ static int cont_ec_aggregate_cb(struct ds_cont_child *cont, daos_epoch_range_t *epr, uint32_t flags, struct agg_param *agg_param) { + struct obj_pool_metrics *opm; struct ec_agg_param *ec_agg_param = agg_param->ap_data; vos_iter_param_t iter_param = { 0 }; struct vos_iter_anchors anchors = { 0 }; int rc = 0; + int blocks = 0; /* * Avoid calling into vos_aggregate() when aborting aggregation @@ -2645,15 +2647,11 @@ cont_ec_aggregate_cb(struct ds_cont_child *cont, daos_epoch_range_t *epr, goto update_hae; } - rc = vos_aggregate_enter(cont->sc_hdl, epr); - if (rc) - goto update_hae; - iter_param.ip_hdl = cont->sc_hdl; iter_param.ip_epr.epr_lo = epr->epr_lo; iter_param.ip_epr.epr_hi = epr->epr_hi; iter_param.ip_epc_expr = VOS_IT_EPC_RR; - iter_param.ip_flags = VOS_IT_RECX_VISIBLE; + iter_param.ip_flags = VOS_IT_RECX_VISIBLE | VOS_IT_FOR_AGG; iter_param.ip_recx.rx_idx = 0ULL; iter_param.ip_recx.rx_nr = ~PARITY_INDICATOR; iter_param.ip_filter_cb = agg_filter; @@ -2661,8 +2659,9 @@ cont_ec_aggregate_cb(struct ds_cont_child *cont, daos_epoch_range_t *epr, agg_reset_entry(&ec_agg_param->ap_agg_entry, NULL, NULL); - rc = vos_iterate(&iter_param, VOS_ITER_OBJ, true, &anchors, - agg_iterate_pre_cb, agg_iterate_post_cb, ec_agg_param, NULL); +retry: + rc = vos_iterate(&iter_param, VOS_ITER_OBJ, true, &anchors, agg_iterate_pre_cb, + agg_iterate_post_cb, ec_agg_param, NULL); /* Post_cb may not being executed in some cases */ agg_clear_extents(&ec_agg_param->ap_agg_entry); @@ -2681,7 +2680,20 @@ cont_ec_aggregate_cb(struct ds_cont_child *cont, daos_epoch_range_t *epr, sched_req_sleep(cont->sc_ec_agg_req, 5 * 1000); } - vos_aggregate_exit(cont->sc_hdl); + if (rc == -DER_BUSY) { + /** Hit an object conflict VOS aggregation or discard. Rather than exiting, let's + * yield and try again. + */ + opm = cont->sc_pool->spc_metrics[DAOS_OBJ_MODULE]; + d_tm_inc_counter(opm->opm_ec_agg_blocked, 1); + blocks++; + /** Warn once if it goes over 20 times */ + D_CDEBUG(blocks == 20, DLOG_WARN, DB_EPC, + "EC agg hit conflict with VOS agg or discard (nr=%d), retrying...\n", + blocks); + ec_aggregate_yield(ec_agg_param); + goto retry; + } update_hae: if (rc == 0) { diff --git a/src/pool/srv_cli.c b/src/pool/srv_cli.c index 6dd4f684e76..1decb57b3ec 100644 --- a/src/pool/srv_cli.c +++ b/src/pool/srv_cli.c @@ -151,7 +151,7 @@ struct dsc_pool_svc_call_cbs { /* * Initialize the request of \a rpc and potentially certain \a arg - * fields. See pool_query_init for an example. + * fields. See pool_query_init for an example. This can be NULL. */ int (*pscc_init)(uuid_t uuid, crt_rpc_t *rpc, void *arg); @@ -166,7 +166,7 @@ struct dsc_pool_svc_call_cbs { /* * Finalize the request of \a rpc and potentially certain \a arg - * fields. See pool_query_fini for an example. + * fields. See pool_query_fini for an example. This can be NULL. */ void (*pscc_fini)(uuid_t uuid, crt_rpc_t *rpc, void *arg); }; @@ -176,7 +176,7 @@ struct dsc_pool_svc_call_cbs { * to this template for calling PS operations. * * The PS is designated by uuid and ranks, the operation by cbs and arg, and - * the deadline of the whole call by deadline. + * the deadline in milliseconds of the whole call by deadline. * * The implementation is simple, if not overly so. A few ideas for future * consideration: @@ -199,6 +199,7 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb struct rsvc_client client; struct d_backoff_seq backoff_seq; + uint64_t req_time = 0; uuid_t no_uuid; struct dss_module_info *info = dss_get_module_info(); int rc; @@ -227,7 +228,6 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb uint32_t rpc_timeout; uint64_t t; struct pool_op_out *out; - uint64_t req_time = 0; uint32_t backoff = d_backoff_seq_next(&backoff_seq); ep.ep_grp = NULL; @@ -245,17 +245,20 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb break; } - rc = cbs->pscc_init(uuid, rpc, arg); - if (rc != 0) { - D_ERROR(DF_PRE": initialize RPC: "DF_RC"\n", DP_PRE(uuid, cbs), DP_RC(rc)); - crt_req_decref(rpc); - break; + if (cbs->pscc_init != NULL) { + rc = cbs->pscc_init(uuid, rpc, arg); + if (rc != 0) { + DL_ERROR(rc, DF_PRE ": initialize RPC", DP_PRE(uuid, cbs)); + crt_req_decref(rpc); + break; + } } /* Cap the RPC timeout according to the deadline. */ t = daos_getmtime_coarse(); if (t >= deadline) { - cbs->pscc_fini(uuid, rpc, arg); + if (cbs->pscc_fini != NULL) + cbs->pscc_fini(uuid, rpc, arg); crt_req_decref(rpc); goto time_out; } @@ -270,7 +273,8 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb * out the call. */ if (rpc_timeout < 1) { - cbs->pscc_fini(uuid, rpc, arg); + if (cbs->pscc_fini != NULL) + cbs->pscc_fini(uuid, rpc, arg); crt_req_decref(rpc); goto time_out; } @@ -290,13 +294,15 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb if (rc == DSC_POOL_SVC_CALL_AGAIN_NOW) { backoff = 0; } else if (rc != DSC_POOL_SVC_CALL_AGAIN) { - cbs->pscc_fini(uuid, rpc, arg); + if (cbs->pscc_fini != NULL) + cbs->pscc_fini(uuid, rpc, arg); crt_req_decref(rpc); break; } } - cbs->pscc_fini(uuid, rpc, arg); + if (cbs->pscc_fini != NULL) + cbs->pscc_fini(uuid, rpc, arg); crt_req_decref(rpc); t = daos_getmtime_coarse(); @@ -491,3 +497,615 @@ dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, return dsc_pool_svc_call(pool_uuid, ps_ranks, &pool_query_cbs, &arg, deadline); } + +struct pool_query_target_arg { + d_rank_t pqta_rank; + uint32_t pqta_tgt_idx; + daos_target_info_t *pqta_info; +}; + +static int +pool_query_target_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_query_target_arg *arg = varg; + + pool_query_info_in_set_data(rpc, arg->pqta_rank, arg->pqta_tgt_idx); + return 0; +} + +static int +pool_query_target_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_query_target_arg *arg = varg; + struct pool_query_info_out *out = crt_reply_get(rpc); + int i; + int rc = out->pqio_op.po_rc; + + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to query pool rank %u target %u", DP_UUID(pool_uuid), + arg->pqta_rank, arg->pqta_tgt_idx); + return rc; + } + + D_DEBUG(DB_MGMT, DF_UUID ": Successfully queried pool rank %u target %u\n", + DP_UUID(pool_uuid), arg->pqta_rank, arg->pqta_tgt_idx); + + arg->pqta_info->ta_type = DAOS_TP_UNKNOWN; + arg->pqta_info->ta_state = out->pqio_state; + for (i = 0; i < DAOS_MEDIA_MAX; i++) { + arg->pqta_info->ta_space.s_total[i] = out->pqio_space.s_total[i]; + arg->pqta_info->ta_space.s_free[i] = out->pqio_space.s_free[i]; + } + + return 0; +} + +static struct dsc_pool_svc_call_cbs pool_query_target_cbs = { + .pscc_op = POOL_QUERY_INFO, + .pscc_init = pool_query_target_init, + .pscc_consume = pool_query_target_consume, + .pscc_fini = NULL +}; + +/** + * Query pool target information without holding a pool handle. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ps_ranks Ranks of pool svc replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in] rank Pool storage engine rank + * \param[in] tgt_idx Target index within the pool storage engine + * \param[out] ti Target information (state, storage capacity and usage) + * + * \return 0 Success + * -DER_INVAL Invalid input + * Negative value Other error + */ +int +dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, + d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti) +{ + struct pool_query_target_arg arg = { + .pqta_rank = rank, + .pqta_tgt_idx = tgt_idx, + .pqta_info = ti + }; + + if (ti == NULL) + return -DER_INVAL; + D_DEBUG(DB_MGMT, DF_UUID ": Querying pool target %u\n", DP_UUID(pool_uuid), tgt_idx); + return dsc_pool_svc_call(pool_uuid, ps_ranks, &pool_query_target_cbs, &arg, deadline); +} + +struct pool_evict_arg { + uuid_t *pea_handles; + size_t pea_n_handles; + char *pea_machine; + uint32_t pea_destroy; + uint32_t pea_force; + uint32_t *pea_count; +}; + +static int +pool_evict_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_evict_arg *arg = varg; + struct pool_evict_in *in = crt_req_get(rpc); + + in->pvi_hdls.ca_arrays = arg->pea_handles; + in->pvi_hdls.ca_count = arg->pea_n_handles; + in->pvi_machine = arg->pea_machine; + /* Pool destroy (force=false): assert no open handles / do not evict. + * Pool destroy (force=true): evict any/all open handles on the pool. + */ + in->pvi_pool_destroy = arg->pea_destroy; + in->pvi_pool_destroy_force = arg->pea_force; + return 0; +} + +static int +pool_evict_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_evict_arg *arg = varg; + struct pool_evict_out *out = crt_reply_get(rpc); + int rc = out->pvo_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": POOL_EVICT failed: destroy=%u force=%u", DP_UUID(pool_uuid), + arg->pea_destroy, arg->pea_force); + if (arg->pea_count != NULL) + *arg->pea_count = out->pvo_n_hdls_evicted; + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_evict_cbs = { + .pscc_op = POOL_EVICT, + .pscc_init = pool_evict_init, + .pscc_consume = pool_evict_consume, + .pscc_fini = NULL +}; + +/** + * Test and (if applicable based on destroy and force option) evict all open + * handles on a pool. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ranks Pool service replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in] handles List of handles to selectively evict + * \param[in] n_handles Number of items in handles + * \param[in] destroy If true the evict request is a destroy request + * \param[in] force If true and destroy is true request all handles + * be forcibly evicted + * \param[in] machine Hostname to use as filter for evicting handles + * \param[out] count Number of handles evicted + * + * \return 0 Success + * -DER_BUSY Open pool handles exist and no force requested + */ +int +dsc_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, uuid_t *handles, + size_t n_handles, uint32_t destroy, uint32_t force, char *machine, + uint32_t *count) +{ + struct pool_evict_arg arg = { + .pea_handles = handles, + .pea_n_handles = n_handles, + .pea_machine = machine, + .pea_destroy = destroy, + .pea_force = force, + .pea_count = count + }; + + D_DEBUG(DB_MGMT, DF_UUID ": destroy=%u force=%u\n", DP_UUID(pool_uuid), destroy, force); + return dsc_pool_svc_call(pool_uuid, ranks, &pool_evict_cbs, &arg, deadline); +} + +struct pool_get_prop_arg { + daos_prop_t *pgpa_prop; +}; + +static int +pool_get_prop_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_get_prop_arg *arg = varg; + + pool_prop_get_in_set_data(rpc, pool_query_bits(NULL, arg->pgpa_prop)); + return 0; +} + +static int +pool_get_prop_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_get_prop_arg *arg = varg; + struct pool_prop_get_out *out = crt_reply_get(rpc); + int rc = out->pgo_op.po_rc; + + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to get prop for pool", DP_UUID(pool_uuid)); + return rc; + } + + return daos_prop_copy(arg->pgpa_prop, out->pgo_prop); +} + +static struct dsc_pool_svc_call_cbs pool_get_prop_cbs = { + .pscc_op = POOL_PROP_GET, + .pscc_init = pool_get_prop_init, + .pscc_consume = pool_get_prop_consume, + .pscc_fini = NULL +}; + +/** + * Get the ACL pool property. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ranks Pool service replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in][out] prop Prop with requested properties, to be + * filled out and returned. + * + * \return 0 Success + * + */ +int +dsc_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, daos_prop_t *prop) +{ + struct pool_get_prop_arg arg = { + .pgpa_prop = prop + }; + + D_DEBUG(DB_MGMT, DF_UUID ": Getting prop\n", DP_UUID(pool_uuid)); + return dsc_pool_svc_call(pool_uuid, ranks, &pool_get_prop_cbs, &arg, deadline); +} + +struct pool_set_prop_arg { + daos_prop_t *pspa_prop; +}; + +static int +pool_set_prop_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_set_prop_arg *arg = varg; + + pool_prop_set_in_set_data(rpc, arg->pspa_prop); + return 0; +} + +static int +pool_set_prop_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_prop_set_out *out = crt_reply_get(rpc); + int rc = out->pso_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": failed to set prop for pool", DP_UUID(pool_uuid)); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_set_prop_cbs = { + .pscc_op = POOL_PROP_SET, + .pscc_init = pool_set_prop_init, + .pscc_consume = pool_set_prop_consume, + .pscc_fini = NULL +}; + +/** + * Set the requested pool properties. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ranks Pool service replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in] prop Pool prop + * + * \return 0 Success + */ +int +dsc_pool_svc_set_prop(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, daos_prop_t *prop) +{ + struct pool_set_prop_arg arg ={ + .pspa_prop = prop + }; + + D_DEBUG(DB_MGMT, DF_UUID ": Setting pool prop\n", DP_UUID(pool_uuid)); + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_PERF_DOMAIN)) { + D_ERROR("Can't set perf_domain on existing pool.\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_REDUN_FAC)) { + D_ERROR("Can't set set redundancy factor on existing pool.\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_EC_PDA)) { + D_ERROR("Can't set EC performance domain affinity on existing pool\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_RP_PDA)) { + D_ERROR("Can't set RP performance domain affinity on existing pool\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_GLOBAL_VERSION)) { + D_ERROR("Can't set pool global version if pool is created.\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_UPGRADE_STATUS)) { + D_ERROR("Can't set pool upgrade status if pool is created.\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_OPS_ENABLED)) { + D_ERROR("Can't set pool svc_ops_enabled on existing pool.\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_OPS_ENTRY_AGE)) { + D_ERROR("Can't set pool svc_ops_entry_age on existing pool.\n"); + return -DER_NO_PERM; + } + + /* Disallow to begin with; will support in the future. */ + if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_REDUN_FAC)) { + D_ERROR(DF_UUID ": cannot set pool service redundancy factor on existing pool\n", + DP_UUID(pool_uuid)); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_OBJ_VERSION)) { + D_ERROR("Can't set pool obj version if pool is created.\n"); + return -DER_NO_PERM; + } + + return dsc_pool_svc_call(pool_uuid, ranks, &pool_set_prop_cbs, &arg, deadline); +} + +struct pool_extend_arg { + int pea_ntargets; + const d_rank_list_t *pea_rank_list; + int pea_ndomains; + const uint32_t *pea_domains; +}; + +static int +pool_extend_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_extend_arg *arg = varg; + struct pool_extend_in *in = crt_req_get(rpc); + + in->pei_ntgts = arg->pea_ntargets; + in->pei_ndomains = arg->pea_ndomains; + in->pei_tgt_ranks = (d_rank_list_t *)arg->pea_rank_list; + in->pei_domains.ca_count = arg->pea_ndomains; + in->pei_domains.ca_arrays = (uint32_t *)arg->pea_domains; + return 0; +} + +static int +pool_extend_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_extend_out *out = crt_reply_get(rpc); + int rc = out->peo_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": Failed to set targets to UP state for reintegration", + DP_UUID(pool_uuid)); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_extend_cbs = { + .pscc_op = POOL_EXTEND, + .pscc_init = pool_extend_init, + .pscc_consume = pool_extend_consume, + .pscc_fini = NULL +}; + +int +dsc_pool_svc_extend(uuid_t pool_uuid, d_rank_list_t *svc_ranks, uint64_t deadline, int ntargets, + const d_rank_list_t *rank_list, int ndomains, const uint32_t *domains) +{ + struct pool_extend_arg arg = { + .pea_ntargets = ntargets, + .pea_rank_list = rank_list, + .pea_ndomains = ndomains, + .pea_domains = domains + }; + + return dsc_pool_svc_call(pool_uuid, svc_ranks, &pool_extend_cbs, &arg, deadline); +} + +struct pool_update_target_state_arg { + struct pool_target_addr_list *puta_target_addrs; + pool_comp_state_t puta_state; +}; + +static int +pool_update_target_state_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_update_target_state_arg *arg = varg; + + pool_tgt_update_in_set_data(rpc, arg->puta_target_addrs->pta_addrs, + (size_t)arg->puta_target_addrs->pta_number); + return 0; +} + +static int +pool_update_target_state_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_update_target_state_arg *arg = varg; + struct pool_tgt_update_out *out = crt_reply_get(rpc); + int rc = out->pto_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": Failed to set targets to %s state", DP_UUID(pool_uuid), + arg->puta_state == PO_COMP_ST_DOWN ? "DOWN" + : arg->puta_state == PO_COMP_ST_UP ? "UP" + : "UNKNOWN"); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_exclude_cbs = { + .pscc_op = POOL_EXCLUDE, + .pscc_init = pool_update_target_state_init, + .pscc_consume = pool_update_target_state_consume, + .pscc_fini = NULL +}; + +static struct dsc_pool_svc_call_cbs pool_reint_cbs = { + .pscc_op = POOL_REINT, + .pscc_init = pool_update_target_state_init, + .pscc_consume = pool_update_target_state_consume, + .pscc_fini = NULL +}; + +static struct dsc_pool_svc_call_cbs pool_drain_cbs = { + .pscc_op = POOL_DRAIN, + .pscc_init = pool_update_target_state_init, + .pscc_consume = pool_update_target_state_consume, + .pscc_fini = NULL +}; + +int +dsc_pool_svc_update_target_state(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + struct pool_target_addr_list *target_addrs, + pool_comp_state_t state) +{ + struct pool_update_target_state_arg arg = { + .puta_target_addrs = target_addrs, + .puta_state = state + }; + struct dsc_pool_svc_call_cbs *cbs; + + switch (state) { + case PO_COMP_ST_DOWN: + cbs = &pool_exclude_cbs; + break; + case PO_COMP_ST_UP: + cbs = &pool_reint_cbs; + break; + case PO_COMP_ST_DRAIN: + cbs = &pool_drain_cbs; + break; + default: + return -DER_INVAL; + } + + return dsc_pool_svc_call(pool_uuid, ranks, cbs, &arg, deadline); +} + +struct pool_update_acl_arg { + struct daos_acl *puaa_acl; +}; + +static int +pool_update_acl_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_update_acl_arg *arg = varg; + + pool_acl_update_in_set_data(rpc, arg->puaa_acl); + return 0; +} + +static int +pool_update_acl_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_acl_update_out *out = crt_reply_get(rpc); + int rc = out->puo_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": failed to update ACL for pool", DP_UUID(pool_uuid)); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_update_acl_cbs = { + .pscc_op = POOL_ACL_UPDATE, + .pscc_init = pool_update_acl_init, + .pscc_consume = pool_update_acl_consume, + .pscc_fini = NULL +}; + +/** + * Update the pool ACL by adding and updating entries. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ranks Pool service replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in] acl ACL to merge with the current pool ACL + * + * \return 0 Success + */ +int +dsc_pool_svc_update_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + struct daos_acl *acl) +{ + struct pool_update_acl_arg arg = { + .puaa_acl = acl + }; + + D_DEBUG(DB_MGMT, DF_UUID ": Updating pool ACL\n", DP_UUID(pool_uuid)); + return dsc_pool_svc_call(pool_uuid, ranks, &pool_update_acl_cbs, &arg, deadline); +} + +struct pool_delete_acl_arg { + enum daos_acl_principal_type pdaa_principal_type; + char *pdaa_name_buf; +}; + +static int +pool_delete_acl_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_delete_acl_arg *arg = varg; + struct pool_acl_delete_in *in; + + in = crt_req_get(rpc); + in->pdi_type = (uint8_t)arg->pdaa_principal_type; + in->pdi_principal = arg->pdaa_name_buf; + return 0; +} + +static int +pool_delete_acl_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_acl_delete_out *out = crt_reply_get(rpc); + int rc = out->pdo_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": failed to delete ACL entry for pool", DP_UUID(pool_uuid)); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_delete_acl_cbs = { + .pscc_op = POOL_ACL_DELETE, + .pscc_init = pool_delete_acl_init, + .pscc_consume = pool_delete_acl_consume, + .pscc_fini = NULL +}; + +/** + * Remove an entry by principal from the pool's ACL. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ranks Pool service replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in] principal_type Type of the principal to be removed + * \param[in] principal_name Name of the principal to be removed + * + * \return 0 Success + */ +int +dsc_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + enum daos_acl_principal_type principal_type, const char *principal_name) +{ + struct pool_delete_acl_arg arg = { + .pdaa_principal_type = principal_type, + .pdaa_name_buf = NULL + }; + size_t name_buf_len; + int rc; + + D_DEBUG(DB_MGMT, DF_UUID ": Deleting entry from pool ACL\n", DP_UUID(pool_uuid)); + + if (principal_name != NULL) { + /* Need to sanitize the incoming string */ + name_buf_len = DAOS_ACL_MAX_PRINCIPAL_BUF_LEN; + D_ALLOC_ARRAY(arg.pdaa_name_buf, name_buf_len); + if (arg.pdaa_name_buf == NULL) + return -DER_NOMEM; + /* force null terminator in copy */ + strncpy(arg.pdaa_name_buf, principal_name, name_buf_len - 1); + } + + rc = dsc_pool_svc_call(pool_uuid, ranks, &pool_delete_acl_cbs, &arg, deadline); + + D_FREE(arg.pdaa_name_buf); + return rc; +} + +static int +pool_upgrade_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_upgrade_out *out = crt_reply_get(rpc); + int rc = out->poo_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": failed to upgrade pool", DP_UUID(pool_uuid)); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_upgrade_cbs = { + .pscc_op = POOL_UPGRADE, + .pscc_init = NULL, + .pscc_consume = pool_upgrade_consume, + .pscc_fini = NULL +}; + +int +dsc_pool_svc_upgrade(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline) +{ + D_DEBUG(DB_MGMT, DF_UUID ": Upgrading pool prop\n", DP_UUID(pool_uuid)); + return dsc_pool_svc_call(pool_uuid, ranks, &pool_upgrade_cbs, NULL /* arg */, deadline); +} diff --git a/src/pool/srv_internal.h b/src/pool/srv_internal.h index 6b317ef7dbd..3cbf91f9d6c 100644 --- a/src/pool/srv_internal.h +++ b/src/pool/srv_internal.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -25,6 +25,16 @@ struct pool_metrics { struct d_tm_node_t *query_total; struct d_tm_node_t *query_space_total; struct d_tm_node_t *evict_total; + + /* service metrics */ + struct d_tm_node_t *service_leader; + struct d_tm_node_t *map_version; + struct d_tm_node_t *open_handles; + struct d_tm_node_t *total_targets; + struct d_tm_node_t *disabled_targets; + struct d_tm_node_t *draining_targets; + struct d_tm_node_t *total_ranks; + struct d_tm_node_t *degraded_ranks; }; /* Pool thread-local storage */ diff --git a/src/pool/srv_metrics.c b/src/pool/srv_metrics.c index 615af9deba1..4a2b583758f 100644 --- a/src/pool/srv_metrics.c +++ b/src/pool/srv_metrics.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2021-2022 Intel Corporation. + * (C) Copyright 2021-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -65,6 +65,48 @@ ds_pool_metrics_alloc(const char *path, int tgt_id) if (rc != 0) D_WARN("Failed to create pool query space counter: "DF_RC"\n", DP_RC(rc)); + rc = d_tm_add_metric(&metrics->service_leader, D_TM_GAUGE, "Pool service leader rank", NULL, + "%s/svc/leader", path); + if (rc != 0) + DL_WARN(rc, "Failed to create pool service leader metric"); + + rc = d_tm_add_metric(&metrics->map_version, D_TM_COUNTER, "Pool map version", NULL, + "%s/svc/map_version", path); + if (rc != 0) + DL_WARN(rc, "Failed to create pool map version metric"); + + rc = d_tm_add_metric(&metrics->open_handles, D_TM_GAUGE, "Pool handles held by clients", + NULL, "%s/svc/open_pool_handles", path); + if (rc != 0) + DL_WARN(rc, "Failed to create pool handle metric"); + + rc = d_tm_add_metric(&metrics->total_ranks, D_TM_GAUGE, "Pool storage ranks (total)", NULL, + "%s/svc/total_ranks", path); + if (rc != 0) + DL_WARN(rc, "Failed to create pool total_ranks metric"); + + rc = d_tm_add_metric(&metrics->degraded_ranks, D_TM_GAUGE, "Pool storage ranks (degraded)", + NULL, "%s/svc/degraded_ranks", path); + if (rc != 0) + DL_WARN(rc, "Failed to create pool degraded_ranks metric"); + + rc = d_tm_add_metric(&metrics->total_targets, D_TM_GAUGE, "Pool storage targets (total)", + NULL, "%s/svc/total_targets", path); + if (rc != 0) + DL_WARN(rc, "Failed to create pool total_targets metric"); + + rc = d_tm_add_metric(&metrics->draining_targets, D_TM_GAUGE, + "Pool storage targets (draining)", NULL, "%s/svc/draining_targets", + path); + if (rc != 0) + DL_WARN(rc, "Failed to create pool draining_targets metric"); + + rc = d_tm_add_metric(&metrics->disabled_targets, D_TM_GAUGE, + "Pool storage targets (disabled)", NULL, "%s/svc/disabled_targets", + path); + if (rc != 0) + DL_WARN(rc, "Failed to create pool disabled_targets metric"); + return metrics; } diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 4f80d248d2a..13002572160 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -385,6 +385,9 @@ pool_prop_default_copy(daos_prop_t *prop_def, daos_prop_t *prop) case DAOS_PROP_PO_SVC_OPS_ENABLED: case DAOS_PROP_PO_SVC_OPS_ENTRY_AGE: case DAOS_PROP_PO_DATA_THRESH: + case DAOS_PROP_PO_CHECKPOINT_MODE: + case DAOS_PROP_PO_CHECKPOINT_THRESH: + case DAOS_PROP_PO_CHECKPOINT_FREQ: entry_def->dpe_val = entry->dpe_val; break; case DAOS_PROP_PO_ACL: @@ -1417,7 +1420,7 @@ init_events(struct pool_svc *svc) D_ASSERT(events->pse_handler == ABT_THREAD_NULL); D_ASSERT(events->pse_stop == false); - if (!engine_in_check()) { + if (!ds_pool_skip_for_check(svc->ps_pool)) { rc = crt_register_event_cb(ds_pool_crt_event_cb, svc); if (rc != 0) { D_ERROR(DF_UUID": failed to register event callback: "DF_RC"\n", @@ -1448,7 +1451,7 @@ init_events(struct pool_svc *svc) return 0; err_cb: - if (!engine_in_check()) + if (!ds_pool_skip_for_check(svc->ps_pool)) crt_unregister_event_cb(ds_pool_crt_event_cb, svc); discard_events(&events->pse_queue); err: @@ -1463,7 +1466,7 @@ fini_events(struct pool_svc *svc) D_ASSERT(events->pse_handler != ABT_THREAD_NULL); - if (!engine_in_check()) + if (!ds_pool_skip_for_check(svc->ps_pool)) crt_unregister_event_cb(ds_pool_crt_event_cb, svc); ABT_mutex_lock(events->pse_mutex); @@ -1831,10 +1834,140 @@ pool_svc_check_node_status(struct pool_svc *svc) D_PRINT(fmt, ## __VA_ARGS__); \ } while (0) +static int +pool_svc_update_map_metrics(uuid_t uuid, struct pool_map *map, struct pool_metrics *metrics) +{ + unsigned int num_total = 0; + unsigned int num_enabled = 0; + unsigned int num_draining = 0; + unsigned int num_disabled = 0; + d_rank_list_t *ranks; + int rc; + + if (map == NULL || metrics == NULL) + return -DER_INVAL; + + rc = pool_map_find_failed_tgts(map, NULL, &num_disabled); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to get failed targets", DP_UUID(uuid)); + D_GOTO(out, rc); + } + d_tm_set_gauge(metrics->disabled_targets, num_disabled); + + rc = pool_map_find_tgts_by_state(map, PO_COMP_ST_DRAIN, NULL, &num_draining); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to get draining targets", DP_UUID(uuid)); + D_GOTO(out, rc); + } + d_tm_set_gauge(metrics->draining_targets, num_draining); + + rc = pool_map_find_tgts_by_state(map, -1, NULL, &num_total); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to get total targets", DP_UUID(uuid)); + D_GOTO(out, rc); + } + d_tm_set_gauge(metrics->total_targets, num_total); + + rc = pool_map_get_ranks(uuid, map, false, &ranks); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to get degraded ranks", DP_UUID(uuid)); + D_GOTO(out, rc); + } + num_disabled = ranks->rl_nr; + d_tm_set_gauge(metrics->degraded_ranks, num_disabled); + + d_rank_list_free(ranks); + rc = pool_map_get_ranks(uuid, map, true, &ranks); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to get enabled ranks", DP_UUID(uuid)); + D_GOTO(out, rc); + } + num_enabled = ranks->rl_nr; + d_tm_set_gauge(metrics->total_ranks, num_enabled + num_disabled); + + d_rank_list_free(ranks); +out: + return rc; +} + +static int +count_iter_cb(daos_handle_t ih, d_iov_t *key, d_iov_t *val, void *varg) +{ + uint64_t *counter = varg; + + if (counter == NULL) + return -DER_INVAL; + *counter = *counter + 1; + + return 0; +} + +static int +pool_svc_step_up_metrics(struct pool_svc *svc, d_rank_t leader, uint32_t map_version, + struct pool_buf *map_buf) +{ + struct pool_map *map; + struct pool_metrics *metrics = svc->ps_pool->sp_metrics[DAOS_POOL_MODULE]; + struct rdb_tx tx; + uint64_t handle_count = 0; + int rc; + + rc = pool_map_create(map_buf, map_version, &map); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to create pool map", DP_UUID(svc->ps_uuid)); + D_GOTO(out, rc); + } + + d_tm_set_gauge(metrics->service_leader, leader); + d_tm_set_counter(metrics->map_version, map_version); + + rc = pool_svc_update_map_metrics(svc->ps_uuid, map, metrics); + if (rc != 0) { + DL_WARN(rc, DF_UUID ": failed to update pool metrics", DP_UUID(svc->ps_uuid)); + rc = 0; /* not fatal */ + } + + rc = rdb_tx_begin(svc->ps_rsvc.s_db, svc->ps_rsvc.s_term, &tx); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to get rdb transaction", DP_UUID(svc->ps_uuid)); + D_GOTO(out_map, rc); + } + + rc = rdb_tx_iterate(&tx, &svc->ps_handles, false, count_iter_cb, &handle_count); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to count open pool handles", DP_UUID(svc->ps_uuid)); + D_GOTO(out_tx, rc); + } + d_tm_set_gauge(metrics->open_handles, handle_count); + +out_tx: + rdb_tx_end(&tx); +out_map: + pool_map_decref(map); +out: + return rc; +} + +static void +pool_svc_step_down_metrics(struct pool_svc *svc) +{ + struct pool_metrics *metrics = svc->ps_pool->sp_metrics[DAOS_POOL_MODULE]; + + /* NB: zero these out to indicate that this rank is not leader */ + d_tm_set_gauge(metrics->service_leader, 0); + d_tm_set_counter(metrics->map_version, 0); + d_tm_set_gauge(metrics->open_handles, 0); + d_tm_set_gauge(metrics->draining_targets, 0); + d_tm_set_gauge(metrics->disabled_targets, 0); + d_tm_set_gauge(metrics->total_targets, 0); + d_tm_set_gauge(metrics->degraded_ranks, 0); + d_tm_set_gauge(metrics->total_ranks, 0); +} + static int pool_svc_schedule(struct pool_svc *svc, struct pool_svc_sched *sched, - void (*func)(void *), void *arg, bool for_chk); + void (*func)(void *), void *arg); static int pool_svc_schedule_reconf(struct pool_svc *svc, struct pool_map *map, - uint32_t map_version_for, bool sync_remove, bool for_chk); + uint32_t map_version_for, bool sync_remove); static void pool_svc_rfcheck_ult(void *arg); static int @@ -1895,20 +2028,22 @@ pool_svc_step_up_cb(struct ds_rsvc *rsvc) * reconfigurations or the last MS notification. */ svc->ps_force_notify = true; - rc = pool_svc_schedule_reconf(svc, NULL /* map */, map_version, false /* sync_remove */, - false /* for_chk */); + rc = pool_svc_schedule_reconf(svc, NULL /* map */, map_version, false /* sync_remove */); if (rc == -DER_OP_CANCELED) { DL_INFO(rc, DF_UUID": not scheduling pool service reconfiguration", DP_UUID(svc->ps_uuid)); + rc = 0; } else if (rc != 0) { DL_ERROR(rc, DF_UUID": failed to schedule pool service reconfiguration", DP_UUID(svc->ps_uuid)); goto out; } - rc = pool_svc_schedule(svc, &svc->ps_rfcheck_sched, pool_svc_rfcheck_ult, NULL /* arg */, - false /* for_chk */); - if (rc != 0) { + rc = pool_svc_schedule(svc, &svc->ps_rfcheck_sched, pool_svc_rfcheck_ult, NULL); + if (rc == -DER_OP_CANCELED) { + DL_INFO(rc, DF_UUID ": not scheduling RF check", DP_UUID(svc->ps_uuid)); + rc = 0; + } else if (rc != 0) { DL_ERROR(rc, DF_UUID": failed to schedule RF check", DP_UUID(svc->ps_uuid)); goto out; } @@ -1948,6 +2083,13 @@ pool_svc_step_up_cb(struct ds_rsvc *rsvc) if (rc != 0) goto out; + rc = pool_svc_step_up_metrics(svc, rank, map_version, map_buf); + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to initialize pool service metrics", + DP_UUID(svc->ps_uuid)); + D_GOTO(out, rc); + } + DS_POOL_NOTE_PRINT(DF_UUID": rank %u became pool service leader "DF_U64": srv_pool_hdl=" DF_UUID" srv_cont_hdl="DF_UUID"\n", DP_UUID(svc->ps_uuid), rank, svc->ps_rsvc.s_term, DP_UUID(pool_hdl_uuid), DP_UUID(cont_hdl_uuid)); @@ -1979,6 +2121,7 @@ pool_svc_step_down_cb(struct ds_rsvc *rsvc) struct pool_svc *svc = pool_svc_obj(rsvc); d_rank_t rank = dss_self_rank(); + pool_svc_step_down_metrics(svc); fini_events(svc); sched_cancel_and_wait(&svc->ps_reconf_sched); sched_cancel_and_wait(&svc->ps_rfcheck_sched); @@ -1997,7 +2140,8 @@ pool_svc_drain_cb(struct ds_rsvc *rsvc) static int pool_svc_map_dist_cb(struct ds_rsvc *rsvc, uint32_t *version) { - struct pool_svc *svc = pool_svc_obj(rsvc); + struct pool_svc *svc = pool_svc_obj(rsvc); + struct pool_metrics *metrics; struct rdb_tx tx; struct pool_buf *map_buf = NULL; uint32_t map_version; @@ -2024,6 +2168,9 @@ pool_svc_map_dist_cb(struct ds_rsvc *rsvc, uint32_t *version) } *version = map_version; + + metrics = svc->ps_pool->sp_metrics[DAOS_POOL_MODULE]; + d_tm_set_counter(metrics->map_version, map_version); out: if (map_buf != NULL) D_FREE(map_buf); @@ -2211,7 +2358,7 @@ start_one(uuid_t uuid, void *varg) D_DEBUG(DB_MD, DF_UUID": starting pool\n", DP_UUID(uuid)); - rc = ds_pool_start(uuid); + rc = ds_pool_start(uuid, varg != NULL ? true : false); if (rc != 0) { D_ERROR(DF_UUID": failed to start pool: %d\n", DP_UUID(uuid), rc); @@ -2257,9 +2404,11 @@ pool_start_all(void *arg) } int -ds_pool_start_with_svc(uuid_t uuid) +ds_pool_start_after_check(uuid_t uuid) { - return start_one(uuid, NULL); + bool aft_chk = true; + + return start_one(uuid, &aft_chk); } /* Note that this function is currently called from the main xstream. */ @@ -3442,6 +3591,16 @@ ds_pool_connect_handler(crt_rpc_t *rpc, int handler_version) D_GOTO(out_lock, rc = -DER_BUSY); } + /* + * NOTE: Under check mode, there is a small race window between ds_pool_mark_connectable() + * the PS restart for full service. If some client tries to connect the pool during + * such internal, it will get -DER_BUSY temporarily. + */ + if (unlikely(ds_pool_skip_for_check(svc->ps_pool))) { + D_ERROR(DF_UUID" is not ready for full pool service\n", DP_UUID(in->pci_op.pi_uuid)); + D_GOTO(out_lock, rc = -DER_BUSY); + } + /* Check existing pool handles. */ d_iov_set(&key, in->pci_op.pi_hdl, sizeof(uuid_t)); d_iov_set(&value, NULL, 0); @@ -3663,6 +3822,7 @@ ds_pool_connect_handler(crt_rpc_t *rpc, int handler_version) /** update metric */ metrics = svc->ps_pool->sp_metrics[DAOS_POOL_MODULE]; d_tm_inc_counter(metrics->connect_total, 1); + d_tm_inc_gauge(metrics->open_handles, 1); } if ((rc == 0) && (query_bits & DAOS_PO_QUERY_SPACE)) @@ -3778,6 +3938,7 @@ pool_disconnect_hdls(struct rdb_tx *tx, struct pool_svc *svc, uuid_t *hdl_uuids, { d_iov_t value; uint32_t nhandles; + struct pool_metrics *metrics; int i; int rc; @@ -3821,6 +3982,8 @@ pool_disconnect_hdls(struct rdb_tx *tx, struct pool_svc *svc, uuid_t *hdl_uuids, if (rc != 0) D_GOTO(out, rc); + metrics = svc->ps_pool->sp_metrics[DAOS_POOL_MODULE]; + d_tm_dec_gauge(metrics->open_handles, n_hdl_uuids); out: if (rc == 0) D_INFO(DF_UUID": success\n", DP_UUID(svc->ps_uuid)); @@ -4803,98 +4966,6 @@ ds_pool_query_info_handler_v5(crt_rpc_t *rpc) ds_pool_query_info_handler(rpc, 5); } -/** - * Query pool target information without holding a pool handle. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ps_ranks Ranks of pool svc replicas - * \param[in] rank Pool storage engine rank - * \param[in] tgt_idx Target index within the pool storage engine - * \param[out] ti Target information (state, storage capacity and usage) - * - * \return 0 Success - * -DER_INVAL Invalid input - * Negative value Other error - */ -int -ds_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, d_rank_t rank, - uint32_t tgt_idx, daos_target_info_t *ti) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - int i; - struct pool_query_info_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - uuid_clear(no_uuid); - - if (ti == NULL) - D_GOTO(out, rc = -DER_INVAL); - - D_DEBUG(DB_MGMT, DF_UUID": Querying pool target %u\n", DP_UUID(pool_uuid), tgt_idx); - - rc = rsvc_client_init(&client, ps_ranks); - if (rc != 0) - goto out; - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_QUERY_INFO, pool_uuid, no_uuid, &req_time, - &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool query target rpc", - DP_UUID(pool_uuid)); - goto out_client; - } - pool_query_info_in_set_data(rpc, rank, tgt_idx); - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pqio_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - goto rechoose; - } - - rc = out->pqio_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": failed to query pool rank %u target %u "DF_RC"\n", - DP_UUID(pool_uuid), rank, tgt_idx, DP_RC(rc)); - goto out_rpc; - } - - D_DEBUG(DB_MGMT, DF_UUID": Successfully queried pool rank %u target %u\n", - DP_UUID(pool_uuid), rank, tgt_idx); - - ti->ta_type = DAOS_TP_UNKNOWN; - ti->ta_state = out->pqio_state; - for (i = 0; i < DAOS_MEDIA_MAX; i++) { - ti->ta_space.s_total[i] = out->pqio_space.s_total[i]; - ti->ta_space.s_free[i] = out->pqio_space.s_free[i]; - } - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - /** * Query a pool's properties without having a handle for the pool */ @@ -4946,294 +5017,69 @@ ds_pool_prop_get_handler(crt_rpc_t *rpc) } /** - * Send a CaRT message to the pool svc to get the ACL pool property. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ranks Pool service replicas - * \param[in][out] prop Prop with requested properties, to be - * filled out and returned. - * - * \return 0 Success - * + * Set a pool's properties without having a handle for the pool */ -int -ds_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks, - daos_prop_t *prop) +void +ds_pool_prop_set_handler(crt_rpc_t *rpc) { + struct pool_prop_set_in *in = crt_req_get(rpc); + struct pool_prop_set_out *out = crt_reply_get(rpc); + struct pool_svc *svc; + struct rdb_tx tx; + daos_prop_t *prop_in = NULL; + daos_prop_t *prop = NULL; + bool dup_op = false; + struct ds_pool_svc_op_val op_val; + bool fi_pass_noreply = DAOS_FAIL_CHECK(DAOS_MD_OP_PASS_NOREPLY); + bool fi_fail_noreply = DAOS_FAIL_CHECK(DAOS_MD_OP_FAIL_NOREPLY); int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_prop_get_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - D_DEBUG(DB_MGMT, DF_UUID": Getting prop\n", DP_UUID(pool_uuid)); - uuid_clear(no_uuid); + D_DEBUG(DB_MD, DF_UUID": processing rpc %p\n", + DP_UUID(in->psi_op.pi_uuid), rpc); - rc = rsvc_client_init(&client, ranks); + pool_prop_set_in_get_data(rpc, &prop_in); + + rc = pool_svc_lookup_leader(in->psi_op.pi_uuid, &svc, &out->pso_op.po_hint); if (rc != 0) D_GOTO(out, rc); -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = - pool_req_create(info->dmi_ctx, &ep, POOL_PROP_GET, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool get prop rpc", DP_UUID(pool_uuid)); - goto out_client; + if (!daos_prop_valid(prop_in, true /* pool */, true /* input */)) { + D_ERROR(DF_UUID": invalid properties input\n", + DP_UUID(in->psi_op.pi_uuid)); + D_GOTO(out_svc, rc = -DER_INVAL); } - pool_prop_get_in_set_data(rpc, pool_query_bits(NULL, prop)); + rc = rdb_tx_begin(svc->ps_rsvc.s_db, svc->ps_rsvc.s_term, &tx); + if (rc != 0) + D_GOTO(out_svc, rc); - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); + ABT_rwlock_wrlock(svc->ps_lock); - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pgo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } + rc = pool_op_lookup(&tx, svc, rpc, DAOS_POOL_VERSION, &dup_op, &op_val); + if (rc != 0) + goto out_lock; + else if (dup_op || fi_fail_noreply) + goto out_commit; - rc = out->pgo_op.po_rc; + rc = pool_prop_write(&tx, &svc->ps_root, prop_in); if (rc != 0) { - D_ERROR(DF_UUID": failed to get prop for pool: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - D_GOTO(out_rpc, rc); + D_ERROR(DF_UUID": failed to write prop for pool: %d\n", + DP_UUID(in->psi_op.pi_uuid), rc); + D_GOTO(out_commit, rc); } - rc = daos_prop_copy(prop, out->pgo_prop); - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - -int -ds_pool_extend(uuid_t pool_uuid, int ntargets, const d_rank_list_t *rank_list, int ndomains, - const uint32_t *domains, d_rank_list_t *svc_ranks) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_extend_in *in; - struct pool_extend_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - uuid_clear(no_uuid); - - rc = rsvc_client_init(&client, svc_ranks); +out_commit: + if ((rc == 0) && !dup_op && fi_fail_noreply) + rc = -DER_MISC; + rc = pool_op_save(&tx, svc, rpc, DAOS_POOL_VERSION, dup_op, rc, &op_val); if (rc != 0) - return rc; + goto out_lock; -rechoose: - - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_EXTEND, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool extend rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - in = crt_req_get(rpc); - in->pei_ntgts = ntargets; - in->pei_ndomains = ndomains; - in->pei_tgt_ranks = (d_rank_list_t *)rank_list; - in->pei_domains.ca_count = ndomains; - in->pei_domains.ca_arrays = (uint32_t *)domains; - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->peo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->peo_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": Failed to set targets to UP state for " - "reintegration: "DF_RC"\n", DP_UUID(pool_uuid), - DP_RC(rc)); - D_GOTO(out_rpc, rc); - } - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); - return rc; -} - -int -ds_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *ranks, - struct pool_target_addr_list *target_addrs, - pool_comp_state_t state) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_tgt_update_out *out; - crt_opcode_t opcode; - uuid_t no_uuid; - uint64_t req_time = 0; - - uuid_clear(no_uuid); - rc = rsvc_client_init(&client, ranks); - if (rc != 0) - return rc; - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - switch (state) { - case PO_COMP_ST_DOWN: - opcode = POOL_EXCLUDE; - break; - case PO_COMP_ST_UP: - opcode = POOL_REINT; - break; - case PO_COMP_ST_DRAIN: - opcode = POOL_DRAIN; - break; - default: - D_GOTO(out_client, rc = -DER_INVAL); - } - - rc = pool_req_create(info->dmi_ctx, &ep, opcode, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool req", DP_UUID(pool_uuid)); - goto out_client; - } - - pool_tgt_update_in_set_data(rpc, target_addrs->pta_addrs, (size_t)target_addrs->pta_number); - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pto_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->pto_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": Failed to set targets to %s state: "DF_RC"\n", - DP_UUID(pool_uuid), - state == PO_COMP_ST_DOWN ? "DOWN" : - state == PO_COMP_ST_UP ? "UP" : "UNKNOWN", - DP_RC(rc)); - D_GOTO(out_rpc, rc); - } - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); - return rc; -} - -/** - * Set a pool's properties without having a handle for the pool - */ -void -ds_pool_prop_set_handler(crt_rpc_t *rpc) -{ - struct pool_prop_set_in *in = crt_req_get(rpc); - struct pool_prop_set_out *out = crt_reply_get(rpc); - struct pool_svc *svc; - struct rdb_tx tx; - daos_prop_t *prop_in = NULL; - daos_prop_t *prop = NULL; - bool dup_op = false; - struct ds_pool_svc_op_val op_val; - bool fi_pass_noreply = DAOS_FAIL_CHECK(DAOS_MD_OP_PASS_NOREPLY); - bool fi_fail_noreply = DAOS_FAIL_CHECK(DAOS_MD_OP_FAIL_NOREPLY); - int rc; - - D_DEBUG(DB_MD, DF_UUID": processing rpc %p\n", - DP_UUID(in->psi_op.pi_uuid), rpc); - - pool_prop_set_in_get_data(rpc, &prop_in); - - rc = pool_svc_lookup_leader(in->psi_op.pi_uuid, &svc, &out->pso_op.po_hint); - if (rc != 0) - D_GOTO(out, rc); - - if (!daos_prop_valid(prop_in, true /* pool */, true /* input */)) { - D_ERROR(DF_UUID": invalid properties input\n", - DP_UUID(in->psi_op.pi_uuid)); - D_GOTO(out_svc, rc = -DER_INVAL); - } - - rc = rdb_tx_begin(svc->ps_rsvc.s_db, svc->ps_rsvc.s_term, &tx); - if (rc != 0) - D_GOTO(out_svc, rc); - - ABT_rwlock_wrlock(svc->ps_lock); - - rc = pool_op_lookup(&tx, svc, rpc, DAOS_POOL_VERSION, &dup_op, &op_val); - if (rc != 0) - goto out_lock; - else if (dup_op || fi_fail_noreply) - goto out_commit; - - rc = pool_prop_write(&tx, &svc->ps_root, prop_in); - if (rc != 0) { - D_ERROR(DF_UUID": failed to write prop for pool: %d\n", - DP_UUID(in->psi_op.pi_uuid), rc); - D_GOTO(out_commit, rc); - } - -out_commit: - if ((rc == 0) && !dup_op && fi_fail_noreply) - rc = -DER_MISC; - rc = pool_op_save(&tx, svc, rpc, DAOS_POOL_VERSION, dup_op, rc, &op_val); - if (rc != 0) - goto out_lock; - - rc = rdb_tx_commit(&tx); - if (rc != 0) - goto out_lock; - if (op_val.ov_rc != 0) - D_GOTO(out_lock, rc = op_val.ov_rc); + rc = rdb_tx_commit(&tx); + if (rc != 0) + goto out_lock; + if (op_val.ov_rc != 0) + D_GOTO(out_lock, rc = op_val.ov_rc); /* Read all props & update prop IV */ rc = pool_prop_read(&tx, svc, DAOS_PO_QUERY_PROP_ALL, &prop); @@ -5998,135 +5844,6 @@ ds_pool_upgrade_handler(crt_rpc_t *rpc) crt_reply_send(rpc); } -/** - * Send a CaRT message to the pool svc to set the requested pool properties. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ranks Pool service replicas - * \param[in] prop Pool prop - * - * \return 0 Success - * - */ -int -ds_pool_svc_set_prop(uuid_t pool_uuid, d_rank_list_t *ranks, daos_prop_t *prop) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_prop_set_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, DF_UUID": Setting pool prop\n", DP_UUID(pool_uuid)); - uuid_clear(no_uuid); - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_PERF_DOMAIN)) { - D_ERROR("Can't set perf_domain on existing pool.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_REDUN_FAC)) { - D_ERROR("Can't set set redundancy factor on existing pool.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_EC_PDA)) { - D_ERROR("Can't set EC performance domain affinity on existing pool\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_RP_PDA)) { - D_ERROR("Can't set RP performance domain affinity on existing pool\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_GLOBAL_VERSION)) { - D_ERROR("Can't set pool global version if pool is created.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_UPGRADE_STATUS)) { - D_ERROR("Can't set pool upgrade status if pool is created.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_OPS_ENABLED)) { - D_ERROR("Can't set pool svc_ops_enabled on existing pool.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_OPS_ENTRY_AGE)) { - D_ERROR("Can't set pool svc_ops_entry_age on existing pool.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - /* Disallow to begin with; will support in the future. */ - if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_REDUN_FAC)) { - D_ERROR(DF_UUID ": cannot set pool service redundancy factor on existing pool\n", - DP_UUID(pool_uuid)); - rc = -DER_NO_PERM; - goto out; - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_OBJ_VERSION)) { - D_ERROR("Can't set pool obj version if pool is created.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) { - D_ERROR(DF_UUID": failed to init rsvc client: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - D_GOTO(out, rc); - } - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = - pool_req_create(info->dmi_ctx, &ep, POOL_PROP_SET, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool set prop rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - pool_prop_set_in_set_data(rpc, prop); - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pso_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->pso_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": failed to set prop for pool: %d\n", - DP_UUID(pool_uuid), rc); - D_GOTO(out_rpc, rc); - } - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - /* * Adds the contents of new_acl to the original ACL. If an entry is added for * a principal already in the ACL, the old entry will be replaced. @@ -6255,78 +5972,6 @@ ds_pool_acl_update_handler(crt_rpc_t *rpc) crt_reply_send(rpc); } -/** - * Send a CaRT message to the pool svc to update the pool ACL by adding and - * updating entries. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ranks Pool service replicas - * \param[in] acl ACL to merge with the current pool ACL - * - * \return 0 Success - * - */ -int -ds_pool_svc_update_acl(uuid_t pool_uuid, d_rank_list_t *ranks, - struct daos_acl *acl) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_acl_update_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, DF_UUID": Updating pool ACL\n", DP_UUID(pool_uuid)); - uuid_clear(no_uuid); - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) - D_GOTO(out, rc); - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_ACL_UPDATE, pool_uuid, no_uuid, &req_time, - &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool update ACL rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - pool_acl_update_in_set_data(rpc, acl); - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->puo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->puo_op.po_rc; - if (rc != 0) - D_ERROR(DF_UUID": failed to update ACL for pool: %d\n", - DP_UUID(pool_uuid), rc); - - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - /** * Delete entries in a pool's ACL without having a handle for the pool */ @@ -6431,97 +6076,6 @@ ds_pool_acl_delete_handler(crt_rpc_t *rpc) crt_reply_send(rpc); } -/** - * Send a CaRT message to the pool svc to remove an entry by principal from the - * pool's ACL. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ranks Pool service replicas - * \param[in] principal_type Type of the principal to be removed - * \param[in] principal_name Name of the principal to be removed - * - * \return 0 Success - * - */ -int -ds_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, - enum daos_acl_principal_type principal_type, - const char *principal_name) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_acl_delete_in *in; - struct pool_acl_delete_out *out; - char *name_buf = NULL; - size_t name_buf_len; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, DF_UUID": Deleting entry from pool ACL\n", - DP_UUID(pool_uuid)); - uuid_clear(no_uuid); - - if (principal_name != NULL) { - /* Need to sanitize the incoming string */ - name_buf_len = DAOS_ACL_MAX_PRINCIPAL_BUF_LEN; - D_ALLOC_ARRAY(name_buf, name_buf_len); - if (name_buf == NULL) - D_GOTO(out, rc = -DER_NOMEM); - /* force null terminator in copy */ - strncpy(name_buf, principal_name, name_buf_len - 1); - } - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) - D_GOTO(out, rc); - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_ACL_DELETE, pool_uuid, no_uuid, &req_time, - &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool delete ACL rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - in = crt_req_get(rpc); - in->pdi_type = (uint8_t)principal_type; - in->pdi_principal = name_buf; - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pdo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->pdo_op.po_rc; - if (rc != 0) - D_ERROR(DF_UUID": failed to delete ACL entry for pool: %d\n", - DP_UUID(pool_uuid), rc); - - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - D_FREE(name_buf); - return rc; -} - struct pool_svc_reconf_arg { struct pool_map *sca_map; uint32_t sca_map_version_for; @@ -6692,18 +6246,19 @@ pool_svc_reconf_ult(void *varg) D_DEBUG(DB_MD, DF_UUID": end: "DF_RC"\n", DP_UUID(svc->ps_uuid), DP_RC(rc)); } +/* If returning 0, this function must have scheduled func(arg). */ static int pool_svc_schedule(struct pool_svc *svc, struct pool_svc_sched *sched, void (*func)(void *), - void *arg, bool for_chk) + void *arg) { enum ds_rsvc_state state; int rc; D_DEBUG(DB_MD, DF_UUID": begin\n", DP_UUID(svc->ps_uuid)); - if (!for_chk && engine_in_check()) { + if (ds_pool_skip_for_check(svc->ps_pool)) { D_DEBUG(DB_MD, DF_UUID": end: skip in check mode\n", DP_UUID(svc->ps_uuid)); - return 0; + return -DER_OP_CANCELED; } /* @@ -6754,8 +6309,9 @@ ds_pool_svc_schedule_reconf(struct ds_pool_svc *svc) * Pass 1 as map_version_for, since there shall be no other * reconfiguration in progress. */ + s->ps_pool->sp_cr_checked = 1; rc = pool_svc_schedule_reconf(s, NULL /* map */, 1 /* map_version_for */, - false /* sync_remove */, true /* for_chk */); + true /* sync_remove */); if (rc != 0) DL_ERROR(rc, DF_UUID": failed to schedule pool service reconfiguration", DP_UUID(s->ps_uuid)); @@ -6820,7 +6376,7 @@ pool_svc_rfcheck_ult(void *arg) */ static int pool_svc_schedule_reconf(struct pool_svc *svc, struct pool_map *map, uint32_t map_version_for, - bool sync_remove, bool for_chk) + bool sync_remove) { struct pool_svc_reconf_arg *reconf_arg; uint32_t v; @@ -6858,7 +6414,7 @@ pool_svc_schedule_reconf(struct pool_svc *svc, struct pool_map *map, uint32_t ma * If successful, this call passes the ownership of reconf_arg to * pool_svc_reconf_ult. */ - rc = pool_svc_schedule(svc, &svc->ps_reconf_sched, pool_svc_reconf_ult, reconf_arg, false); + rc = pool_svc_schedule(svc, &svc->ps_reconf_sched, pool_svc_reconf_ult, reconf_arg); if (rc != 0) { D_FREE(reconf_arg); return rc; @@ -7042,8 +6598,7 @@ pool_svc_update_map_internal(struct pool_svc *svc, unsigned int opc, * Remove all undesired PS replicas (if any) before committing map, so * that the set of PS replicas remains a subset of the pool groups. */ - rc = pool_svc_schedule_reconf(svc, map, 0 /* map_version_for */, true /* sync_remove */, - false /* for_chk */); + rc = pool_svc_schedule_reconf(svc, map, 0 /* map_version_for */, true /* sync_remove */); if (rc != 0) { DL_ERROR(rc, DF_UUID": failed to remove undesired pool service replicas", DP_UUID(svc->ps_uuid)); @@ -7075,19 +6630,26 @@ pool_svc_update_map_internal(struct pool_svc *svc, unsigned int opc, ds_rsvc_request_map_dist(&svc->ps_rsvc); - rc = pool_svc_schedule_reconf(svc, NULL /* map */, map_version, false /* sync_remove */, - false /* for_chk */); - if (rc != 0) + rc = pool_svc_schedule_reconf(svc, NULL /* map */, map_version, false /* sync_remove */); + if (rc != 0) { DL_INFO(rc, DF_UUID": failed to schedule pool service reconfiguration", DP_UUID(svc->ps_uuid)); + rc = 0; + } if (opc == MAP_EXCLUDE) { - rc = pool_svc_schedule(svc, &svc->ps_rfcheck_sched, pool_svc_rfcheck_ult, - NULL /* arg */, false /* for_chk */); + rc = pool_svc_schedule(svc, &svc->ps_rfcheck_sched, pool_svc_rfcheck_ult, NULL); if (rc != 0) DL_INFO(rc, DF_UUID": failed to schedule RF check", DP_UUID(svc->ps_uuid)); } + rc = pool_svc_update_map_metrics(svc->ps_uuid, map, + svc->ps_pool->sp_metrics[DAOS_POOL_MODULE]); + if (rc != 0) { + DL_WARN(rc, DF_UUID ": failed to update pool metrics", DP_UUID(svc->ps_uuid)); + rc = 0; /* not fatal */ + } + out_map_buf: pool_buf_free(map_buf); out_map: @@ -7805,100 +7367,6 @@ ds_pool_evict_handler(crt_rpc_t *rpc) crt_reply_send(rpc); } -/** - * Send a CaRT message to the pool svc to test and - * (if applicable based on destroy and force option) evict all open handles - * on a pool. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ranks Pool service replicas - * \param[in] handles List of handles to selectively evict - * \param[in] n_handles Number of items in handles - * \param[in] destroy If true the evict request is a destroy request - * \param[in] force If true and destroy is true request all handles - * be forcibly evicted - * \param[in] machine Hostname to use as filter for evicting handles - * \param[out] count Number of handles evicted - * - * \return 0 Success - * -DER_BUSY Open pool handles exist and no force requested - * - */ -int -ds_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks, - uuid_t *handles, size_t n_handles, - uint32_t destroy, uint32_t force, - char *machine, uint32_t *count) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_evict_in *in; - struct pool_evict_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, - DF_UUID": Destroy pool (force: %d), inspect/evict handles\n", - DP_UUID(pool_uuid), force); - uuid_clear(no_uuid); - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) - D_GOTO(out, rc); - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_EVICT, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool evict rpc", DP_UUID(pool_uuid)); - D_GOTO(out_client, rc); - } - - in = crt_req_get(rpc); - in->pvi_hdls.ca_arrays = handles; - in->pvi_hdls.ca_count = n_handles; - in->pvi_machine = machine; - - /* Pool destroy (force=false): assert no open handles / do not evict. - * Pool destroy (force=true): evict any/all open handles on the pool. - */ - in->pvi_pool_destroy = destroy; - in->pvi_pool_destroy_force = force; - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pvo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->pvo_op.po_rc; - if (rc != 0) - DL_ERROR(rc, DF_UUID ": pool destroy failed to evict handles", DP_UUID(pool_uuid)); - if (count) - *count = out->pvo_n_hdls_evicted; - - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - /* * Transfer list of pool ranks to "remote_bulk". If the remote bulk buffer * is too small, then return -DER_TRUNC. RPC response will contain the number @@ -8596,68 +8064,6 @@ is_pool_from_srv(uuid_t pool_uuid, uuid_t poh_uuid) return rc ? true : false; } -int ds_pool_svc_upgrade(uuid_t pool_uuid, d_rank_list_t *ranks) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_upgrade_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, DF_UUID": Upgrading pool prop\n", DP_UUID(pool_uuid)); - uuid_clear(no_uuid); - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) { - D_ERROR(DF_UUID": failed to init rsvc client: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - D_GOTO(out, rc); - } - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_UPGRADE, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool upgrade rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->poo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->poo_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": failed to upgrade pool: %d\n", - DP_UUID(pool_uuid), rc); - D_GOTO(out_rpc, rc); - } - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - /* Check if the target(by id) matched the status */ int ds_pool_target_status_check(struct ds_pool *pool, uint32_t id, uint8_t matched_status, @@ -8989,3 +8395,9 @@ ds_pool_svc_upgrade_vos_pool(struct ds_pool *pool) ds_rsvc_put(rsvc); return rc; } + +bool +ds_pool_skip_for_check(struct ds_pool *pool) +{ + return engine_in_check() && !pool->sp_cr_checked; +} diff --git a/src/pool/srv_pool_scrub_ult.c b/src/pool/srv_pool_scrub_ult.c index 8dac29347da..744742067d5 100644 --- a/src/pool/srv_pool_scrub_ult.c +++ b/src/pool/srv_pool_scrub_ult.c @@ -244,7 +244,9 @@ drain_pool_target(uuid_t pool_uuid, d_rank_t rank, uint32_t target) addr.pta_target = target; target_list.pta_addrs = &addr; - rc = ds_pool_target_update_state(pool_uuid, &out_ranks, &target_list, PO_COMP_ST_DRAIN); + rc = dsc_pool_svc_update_target_state(pool_uuid, &out_ranks, + daos_getmtime_coarse() + 5 * 60 * 1000, &target_list, + PO_COMP_ST_DRAIN); if (rc != DER_SUCCESS) D_ERROR("pool target update status failed: "DF_RC"\n", DP_RC(rc)); map_ranks_fini(&out_ranks); diff --git a/src/pool/srv_target.c b/src/pool/srv_target.c index 47c9c484671..96a848d9de8 100644 --- a/src/pool/srv_target.c +++ b/src/pool/srv_target.c @@ -491,7 +491,7 @@ pool_child_start(struct ds_pool_child *child, bool recreate) goto done; } - if (!engine_in_check()) { + if (!ds_pool_skip_for_check(child->spc_pool)) { rc = start_gc_ult(child); if (rc != 0) goto out_close; @@ -628,9 +628,10 @@ pool_child_stop(struct ds_pool_child *child) } int -ds_pool_child_stop(uuid_t pool_uuid) +ds_pool_child_stop(uuid_t pool_uuid, bool free) { struct ds_pool_child *child; + int rc; child = pool_child_lookup_noref(pool_uuid); if (child == NULL) { @@ -638,7 +639,11 @@ ds_pool_child_stop(uuid_t pool_uuid) return -DER_NONEXIST; } - return pool_child_stop(child); + rc = pool_child_stop(child); + if (rc == 0 && free) + pool_child_free(child); + + return rc; } struct pool_child_lookup_arg { @@ -1121,111 +1126,12 @@ pool_fetch_hdls_ult_abort(struct ds_pool *pool) D_INFO(DF_UUID": fetch hdls ULT aborted\n", DP_UUID(pool->sp_uuid)); } -static int -ds_pool_chk_post_one(void *varg) -{ - struct pool_child_lookup_arg *arg = varg; - struct ds_pool_child *child = NULL; - int rc = 0; - - /* The pool shard must has been opened. */ - child = ds_pool_child_lookup(arg->pla_uuid); - if (child == NULL) - D_GOTO(out, rc = -DER_NONEXIST); - - D_ASSERT(*child->spc_state == POOL_CHILD_STARTED); - - if (unlikely(child->spc_no_storage)) - D_GOTO(out, rc = 0); - - rc = start_gc_ult(child); - if (rc != 0) - goto out; - - rc = start_flush_ult(child); - if (rc != 0) - goto out; - - rc = ds_start_scrubbing_ult(child); - if (rc != 0) - goto out; - - rc = ds_cont_chk_post(child); - -out: - if (child != NULL) { - if (rc != 0) { - ds_stop_scrubbing_ult(child); - stop_flush_ult(child); - stop_gc_ult(child); - } - - ds_pool_child_put(child); - } - - return rc; -} - -int -ds_pool_chk_post(uuid_t uuid) -{ - struct ds_pool *pool = NULL; - struct daos_llink *llink = NULL; - struct pool_child_lookup_arg collective_arg = { 0 }; - int rc = 0; - - D_ASSERT(engine_in_check()); - D_ASSERT(dss_get_module_info()->dmi_xs_id == 0); - - D_DEBUG(DB_MGMT, "Post handle pool starting for "DF_UUIDF" after DAOS check: "DF_RC"\n", - DP_UUID(uuid), DP_RC(rc)); - - /* The pool must has been opened. */ - rc = daos_lru_ref_hold(pool_cache, (void *)uuid, sizeof(uuid_t), - NULL /* create_args */, &llink); - if (rc != 0) - goto out; - - pool = pool_obj(llink); - if (pool->sp_stopping) - D_GOTO(out, rc = -DER_SHUTDOWN); - - pool->sp_fetch_hdls = 1; - pool_fetch_hdls_ult(pool); - - rc = ds_pool_start_ec_eph_query_ult(pool); - if (rc != 0) { - D_ERROR(DF_UUID": failed to start ec eph query ult: "DF_RC"\n", - DP_UUID(uuid), DP_RC(rc)); - goto out; - } - - collective_arg.pla_uuid = uuid; - rc = dss_thread_collective(ds_pool_chk_post_one, &collective_arg, 0); - -out: - if (pool != NULL) { - if (rc != 0) { - ds_pool_tgt_ec_eph_query_abort(pool); - pool_fetch_hdls_ult_abort(pool); - } - - daos_lru_ref_release(pool_cache, &pool->sp_entry); - } - - D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, - "Post handle pool started for "DF_UUIDF" after DAOS check: "DF_RC"\n", - DP_UUID(uuid), DP_RC(rc)); - - return rc; -} - /* * Start a pool. Must be called on the system xstream. Hold the ds_pool object * till ds_pool_stop. Only for mgmt and pool modules. */ int -ds_pool_start(uuid_t uuid) +ds_pool_start(uuid_t uuid, bool aft_chk) { struct ds_pool *pool; struct daos_llink *llink; @@ -1246,6 +1152,14 @@ ds_pool_start(uuid_t uuid) D_ERROR(DF_UUID": stopping isn't done yet\n", DP_UUID(uuid)); rc = -DER_BUSY; + } else if (unlikely(aft_chk)) { + /* + * Someone still references the pool after CR check + * that blocks pool (re)start for full pool service. + */ + D_ERROR(DF_UUID": someone still references the pool after CR check\n", + DP_UUID(uuid)); + rc = -DER_BUSY; } /* Already started; drop our reference. */ daos_lru_ref_release(pool_cache, &pool->sp_entry); @@ -1269,7 +1183,12 @@ ds_pool_start(uuid_t uuid) pool = pool_obj(llink); - if (!engine_in_check()) { + if (aft_chk) + pool->sp_cr_checked = 1; + else + pool->sp_cr_checked = 0; + + if (!ds_pool_skip_for_check(pool)) { rc = dss_ult_create(pool_fetch_hdls_ult, pool, DSS_XS_SYS, 0, DSS_DEEP_STACK_SZ, NULL); if (rc != 0) { @@ -1936,19 +1855,20 @@ ds_pool_tgt_map_update(struct ds_pool *pool, struct pool_buf *buf, struct dtx_scan_args *arg; int ret; - /* Since the map has been updated successfully, so let's - * ignore the dtx resync failure for now. - */ + if (ds_pool_skip_for_check(pool)) + D_GOTO(out, rc = 0); + D_ALLOC_PTR(arg); if (arg == NULL) - D_GOTO(out, rc); + D_GOTO(out, rc = -DER_NOMEM); uuid_copy(arg->pool_uuid, pool->sp_uuid); arg->version = pool->sp_map_version; ret = dss_ult_create(dtx_resync_ult, arg, DSS_XS_SYS, 0, 0, NULL); if (ret) { - D_ERROR("dtx_resync_ult failure %d\n", ret); + /* Ignore DTX resync failure that is not fatal. */ + D_WARN("dtx_resync_ult failure %d\n", ret); D_FREE(arg); } } else { diff --git a/src/pool/srv_util.c b/src/pool/srv_util.c index 6873498e780..e39072568e1 100644 --- a/src/pool/srv_util.c +++ b/src/pool/srv_util.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1702,7 +1702,7 @@ manage_target(bool start) if (start) rc = ds_pool_child_start(pool_info->spi_id, true); else - rc = ds_pool_child_stop(pool_info->spi_id); + rc = ds_pool_child_stop(pool_info->spi_id, false); if (rc < 0) { DL_ERROR(rc, DF_UUID": Failed to %s pool child.", diff --git a/src/proto/mgmt/pool.proto b/src/proto/mgmt/pool.proto index 30540e2904e..712779d5f34 100644 --- a/src/proto/mgmt/pool.proto +++ b/src/proto/mgmt/pool.proto @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -179,8 +179,7 @@ message PoolQueryReq { string sys = 1; // DAOS system identifier string id = 2; repeated uint32 svc_ranks = 3; // List of pool service ranks - bool include_enabled_ranks = 4; // True if the list of enabled ranks shall be returned - bool include_disabled_ranks = 5; // True if the list of disabled ranks shall be returned + uint64 query_mask = 4; // Bitmask of pool query options } enum StorageMediaType { @@ -232,13 +231,16 @@ message PoolQueryResp { PoolRebuildStatus rebuild = 7; // pool rebuild status repeated StorageUsageStats tier_stats = 8; // storage tiers usage stats uint32 version = 10; // latest pool map version - uint32 leader = 11; // current raft leader + uint32 leader = 11; // current raft leader (2.4) string enabled_ranks = 12; // optional set of ranks enabled string disabled_ranks = 13; // optional set of ranks disabled uint32 total_engines = 14; // total engines in pool uint32 pool_layout_ver = 15; // current pool global version uint32 upgrade_layout_ver = 16; // latest pool global version to upgrade PoolServiceState state = 17; // pool state + uint32 svc_ldr = 18; // current raft leader (2.6+) + repeated uint32 svc_reps = 19; // service replica ranks + uint64 query_mask = 20; // Bitmask of pool query options used } message PoolProperty { diff --git a/src/rdb/rdb_internal.h b/src/rdb/rdb_internal.h index 9195314e5c7..58e8d1fe7e8 100644 --- a/src/rdb/rdb_internal.h +++ b/src/rdb/rdb_internal.h @@ -493,6 +493,11 @@ int rdb_scm_left(struct rdb *db, daos_size_t *scm_left_outp); /* rdb_tx.c *******************************************************************/ int rdb_tx_count_vops(struct rdb *db, const void *buf, size_t len); + +enum rdb_tx_apply_err { + RDB_TX_APPLY_ERR_DETERMINISTIC = 1 +}; + int rdb_tx_apply(struct rdb *db, uint64_t index, const void *buf, size_t len, void *result, bool *critp, rdb_vos_tx_t vtx); diff --git a/src/rdb/rdb_raft.c b/src/rdb/rdb_raft.c index 8056c54da48..2bb84546eea 100644 --- a/src/rdb/rdb_raft.c +++ b/src/rdb/rdb_raft.c @@ -1119,54 +1119,138 @@ rdb_raft_update_node(struct rdb *db, uint64_t index, raft_entry_t *entry, rdb_vo /* See rdb_raft_log_offer_single. */ #define RDB_RAFT_ENTRY_NVOPS 2 +static int +rdb_raft_entry_count_vops(struct rdb *db, raft_entry_t *entry) +{ + int count = 0; + + /* Count those that will be invoked when applying the entry. */ + if (entry->type == RAFT_LOGTYPE_NORMAL) { + int rc; + + rc = rdb_tx_count_vops(db, entry->data.buf, entry->data.len); + if (rc < 0) + return rc; + count += rc; + } else if (raft_entry_is_cfg_change(entry)) { + count += RDB_RAFT_UPDATE_NODE_NVOPS; + } else { + D_ERROR(DF_DB ": unknown entry type %d\n", DP_DB(db), entry->type); + return -DER_IO; + } + + /* Count those that will be invoked when storing the entry. */ + count += RDB_RAFT_ENTRY_NVOPS; + + return count; +} + /* * Must invoke no more than RDB_RAFT_ENTRY_NVOPS VOS TX operations directly * (i.e., not including those invoked by rdb_tx_apply and - * rdb_raft_update_node). + * rdb_raft_update_node) per VOS TX. */ static int -rdb_raft_log_offer_single(struct rdb *db, raft_entry_t *entry, uint64_t index, rdb_vos_tx_t vtx) -{ - d_iov_t keys[2]; - d_iov_t values[2]; - struct rdb_entry header; - int n = 0; - bool crit; - int rc; - - D_ASSERTF(index == db->d_lc_record.dlr_tail, DF_U64" == "DF_U64"\n", - index, db->d_lc_record.dlr_tail); +rdb_raft_log_offer_single(struct rdb *db, raft_entry_t *entry, uint64_t index) +{ + rdb_vos_tx_t vtx; + bool skip_tx_apply = false; + d_iov_t keys[2]; + d_iov_t values[2]; + struct rdb_entry header; + int n; + bool crit = true; + bool dirtied_tail; + bool dirtied_kvss; + int rc; + +retry: + /* Initialize or reset per TX variables. */ + dirtied_tail = false; + dirtied_kvss = false; + + /* Begin a VOS TX. */ + if (skip_tx_apply) { + D_ASSERTF(entry->type == RAFT_LOGTYPE_NORMAL, "%d == %d\n", entry->type, + RAFT_LOGTYPE_NORMAL); + rc = RDB_RAFT_ENTRY_NVOPS; + } else { + rc = rdb_raft_entry_count_vops(db, entry); + if (rc < 0) { + DL_ERROR(rc, DF_DB ": failed to count VOS operations for entry %ld", + DP_DB(db), index); + return rc; + } + } + rc = rdb_vos_tx_begin(db, rc /* nvops */, &vtx); + if (rc != 0) { + DL_ERROR(rc, DF_DB ": failed to begin VOS TX for entry %ld", DP_DB(db), index); + return rc; + } /* - * If this is an rdb_tx entry, apply it. Note that the updates involved - * won't become visible to queries until entry index is committed. - * (Implicit queries resulted from rdb_kvs cache lookups won't happen - * until the TX releases the locks for the updates after the - * rdb_tx_commit() call returns.) + * Update the log tail. To get the same minor epoch for all MC updates + * across different VOS TXs, we update the MC first. */ + D_ASSERTF(index == db->d_lc_record.dlr_tail, DF_U64 " == " DF_U64 "\n", index, + db->d_lc_record.dlr_tail); + db->d_lc_record.dlr_tail = index + 1; + dirtied_tail = true; + d_iov_set(&values[0], &db->d_lc_record, sizeof(db->d_lc_record)); + rc = rdb_mc_update(db->d_mc, RDB_MC_ATTRS, 1 /* n */, &rdb_mc_lc, &values[0], vtx); + if (rc != 0) { + DL_ERROR(rc, DF_DB ": failed to update log tail " DF_U64, DP_DB(db), + db->d_lc_record.dlr_tail); + goto out_vtx; + } + if (entry->type == RAFT_LOGTYPE_NORMAL) { - rc = rdb_tx_apply(db, index, entry->data.buf, entry->data.len, - rdb_raft_lookup_result(db, index), &crit, vtx); - if (rc != 0) { - DL_ERROR(rc, DF_DB ": failed to apply entry " DF_U64, DP_DB(db), index); - return rc; + if (!skip_tx_apply) { + /* + * If this is an rdb_tx entry, apply it. Note that the updates involved + * won't become visible to queries until entry index is committed. + * (Implicit queries resulted from rdb_kvs cache lookups won't happen + * until the TX releases the locks for the updates after the + * rdb_tx_commit() call returns.) + */ + rc = rdb_tx_apply(db, index, entry->data.buf, entry->data.len, + rdb_raft_lookup_result(db, index), &crit, vtx); + if (rc == RDB_TX_APPLY_ERR_DETERMINISTIC) { + /* + * We must abort VOS TX to discard any partial application of the + * entry, and begin a new VOS TX to store the entry without applying + * it. The new VOS TX will reuse crit. + */ + D_DEBUG(DB_TRACE, DF_DB ": deterministic error for entry %ld\n", + DP_DB(db), index); + rc = -DER_AGAIN; + skip_tx_apply = true; + goto out_vtx; + } else if (rc != 0) { + DL_ERROR(rc, DF_DB ": failed to apply entry " DF_U64, DP_DB(db), + index); + goto out_vtx; + } + dirtied_kvss = true; } } else if (raft_entry_is_cfg_change(entry)) { - crit = true; rc = rdb_raft_update_node(db, index, entry, vtx); if (rc != 0) { DL_ERROR(rc, DF_DB ": failed to update replicas " DF_U64, DP_DB(db), index); - return rc; + goto out_vtx; } } else { - D_ERROR(DF_DB": unknown entry "DF_U64" type: %d\n", DP_DB(db), index, entry->type); - return -DER_IO; + D_ERROR(DF_DB ": unknown entry " DF_U64 " type: %d\n", DP_DB(db), index, + entry->type); + rc = -DER_IO; + goto out_vtx; } /* * Persist the header and the data (if nonempty). Discard the unused * entry->id. */ + n = 0; header.dre_term = entry->term; header.dre_type = entry->type; header.dre_size = entry->data.len; @@ -1181,7 +1265,7 @@ rdb_raft_log_offer_single(struct rdb *db, raft_entry_t *entry, uint64_t index, r rc = rdb_lc_update(db->d_lc, index, RDB_LC_ATTRS, crit, n, keys, values, vtx); if (rc != 0) { DL_ERROR(rc, DF_DB ": failed to persist entry " DF_U64, DP_DB(db), index); - return rc; + goto out_vtx; } /* Replace entry->data.buf with the data's persistent memory address. */ @@ -1191,63 +1275,45 @@ rdb_raft_log_offer_single(struct rdb *db, raft_entry_t *entry, uint64_t index, r if (rc != 0) { DL_ERROR(rc, DF_DB ": failed to look up entry " DF_U64 " data", DP_DB(db), index); - return rc; + goto out_vtx; } entry->data.buf = values[0].iov_buf; } else { entry->data.buf = NULL; } - /* Update the log tail. See the log tail assertion above. */ - db->d_lc_record.dlr_tail++; - d_iov_set(&values[0], &db->d_lc_record, sizeof(db->d_lc_record)); - rc = rdb_mc_update(db->d_mc, RDB_MC_ATTRS, 1 /* n */, &rdb_mc_lc, &values[0], vtx); +out_vtx: + /* End the VOS TX. If there's an error, revert all cache changes. */ + rc = rdb_vos_tx_end(db, vtx, rc); if (rc != 0) { - DL_ERROR(rc, DF_DB ": failed to update log tail " DF_U64, DP_DB(db), - db->d_lc_record.dlr_tail); - db->d_lc_record.dlr_tail--; - return rc; - } - - D_DEBUG(DB_TRACE, DF_DB": appended entry "DF_U64": term=%ld type=%s buf=%p len=%u\n", - DP_DB(db), index, entry->term, rdb_raft_entry_type_str(entry->type), - entry->data.buf, entry->data.len); - return 0; -} - -static int -rdb_raft_entry_count_vops(struct rdb *db, raft_entry_t *entry) -{ - int count = 0; - - /* Count those that will be invoked when applying the entry. */ - if (entry->type == RAFT_LOGTYPE_NORMAL) { - int rc; - - rc = rdb_tx_count_vops(db, entry->data.buf, entry->data.len); - if (rc < 0) + if (dirtied_kvss) + rdb_kvs_cache_evict(db->d_kvss); + if (dirtied_tail) + db->d_lc_record.dlr_tail = index; + if (rc == -DER_AGAIN && skip_tx_apply) { + D_DEBUG(DB_TRACE, DF_DB ": aborted before retrying entry %ld\n", DP_DB(db), + index); + goto retry; + } else { + DL_ERROR(rc, DF_DB ": failed to end VOS TX for entry %ld", DP_DB(db), + index); return rc; - count += rc; - } else if (raft_entry_is_cfg_change(entry)) { - count += RDB_RAFT_UPDATE_NODE_NVOPS; - } else { - D_ERROR(DF_DB ": unknown entry type %d\n", DP_DB(db), entry->type); - return -DER_IO; + } } - /* Count those that will be invoked when storing the entry. */ - count += RDB_RAFT_ENTRY_NVOPS; - - return count; + D_DEBUG(DB_TRACE, DF_DB ": appended entry %ld: term=%ld type=%s buf=%p len=%u\n", DP_DB(db), + index, entry->term, rdb_raft_entry_type_str(entry->type), entry->data.buf, + entry->data.len); + return 0; } static int -rdb_raft_cb_log_offer(raft_server_t *raft, void *arg, raft_entry_t *entries, - raft_index_t index, int *n_entries) +rdb_raft_cb_log_offer(raft_server_t *raft, void *arg, raft_entry_t *entries, raft_index_t index, + int *n_entries) { - struct rdb *db = arg; - int i; - int rc = 0; + struct rdb *db = arg; + int i; + int rc = 0; if (!db->d_raft_loaded) return 0; @@ -1259,29 +1325,9 @@ rdb_raft_cb_log_offer(raft_server_t *raft, void *arg, raft_entry_t *entries, * batching TXs, we can optimize this process further. */ for (i = 0; i < *n_entries; i++) { - rdb_vos_tx_t vtx; - - rc = rdb_raft_entry_count_vops(db, &entries[i]); - if (rc < 0) { - DL_ERROR(rc, DF_DB ": failed to count VOS operations", DP_DB(db)); - break; - } - - rc = rdb_vos_tx_begin(db, rc, &vtx); - if (rc != 0) { - DL_ERROR(rc, DF_DB ": failed to begin VOS TX for entry %ld", DP_DB(db), - index); - break; - } - - rc = rdb_raft_log_offer_single(db, &entries[i], index + i, vtx); - - rc = rdb_vos_tx_end(db, vtx, rc); - if (rc != 0) { - DL_ERROR(rc, DF_DB ": failed to end VOS TX for entry %ld", DP_DB(db), - index); + rc = rdb_raft_log_offer_single(db, &entries[i], index + i); + if (rc != 0) break; - } } *n_entries = i; @@ -2399,10 +2445,9 @@ rdb_raft_load_entry(struct rdb *db, uint64_t index) return rdb_raft_rc(rc); } - D_DEBUG(DB_TRACE, - DF_DB": loaded entry "DF_U64": term=%ld type=%d buf=%p " - "len=%u\n", DP_DB(db), index, entry.term, entry.type, - entry.data.buf, entry.data.len); + D_DEBUG(DB_TRACE, DF_DB ": loaded entry " DF_U64 ": term=%ld type=%s buf=%p len=%u\n", + DP_DB(db), index, entry.term, rdb_raft_entry_type_str(entry.type), entry.data.buf, + entry.data.len); return 0; } @@ -2849,6 +2894,14 @@ rdb_raft_load(struct rdb *db) if (rc != 0) goto out; + D_DEBUG(DB_MD, + DF_DB ": term=" DF_U64 " vote=%d lc.uuid=" DF_UUID " lc.base=" DF_U64 + " lc.base_term=" DF_U64 " lc.tail=" DF_U64 " lc.aggregated=" DF_U64 + " lc.term=" DF_U64 " lc.seq=" DF_U64 "\n", + DP_DB(db), term, vote, DP_UUID(db->d_lc_record.dlr_uuid), db->d_lc_record.dlr_base, + db->d_lc_record.dlr_base_term, db->d_lc_record.dlr_tail, + db->d_lc_record.dlr_aggregated, db->d_lc_record.dlr_term, db->d_lc_record.dlr_seq); + db->d_raft_loaded = true; out: D_DEBUG(DB_MD, DF_DB": load persistent state: end: "DF_RC"\n", DP_DB(db), DP_RC(rc)); diff --git a/src/rdb/rdb_tx.c b/src/rdb/rdb_tx.c index 3536181fda4..3e0f0617224 100644 --- a/src/rdb/rdb_tx.c +++ b/src/rdb/rdb_tx.c @@ -995,9 +995,10 @@ rdb_tx_deterministic_error(int error) } /* - * Apply an entry and return the error only if a nondeterministic error - * happens. This function tries to discard index if an error occurs. - * Interpret header to know if ops in the TX are deemed "critical". + * Apply an entry and return 0, RDB_TX_APPLY_ERR_DETERMINISTIC, or a + * nondeterministic error. Interpret header to know if ops in the TX are deemed + * "critical", and output to critp when returning 0 or + * RDB_TX_APPLY_ERR_DETERMINISTIC. */ int rdb_tx_apply(struct rdb *db, uint64_t index, const void *buf, size_t len, void *result, bool *critp, @@ -1044,18 +1045,22 @@ rdb_tx_apply(struct rdb *db, uint64_t index, const void *buf, size_t len, void * hdr.critical, scm_remaining); while (p < buf + len) { - struct rdb_tx_op op; + struct rdb_tx_op op; n = rdb_tx_op_decode(p, buf + len - p, &op); if (n < 0) { - D_ERROR(DF_DB": invalid entry format: buf=%p len="DF_U64 - " p=%p\n", DP_DB(db), buf, len, p); + D_ERROR(DF_DB ": invalid entry format: buf=%p len=" DF_U64 " p=%p\n", + DP_DB(db), buf, len, p); rc = n; break; } rc = rdb_tx_apply_op(db, index, &op, hdr.critical, vtx); if (rc != 0) { - if (!rdb_tx_deterministic_error(rc)) + if (rdb_tx_deterministic_error(rc)) + D_DEBUG(DB_TRACE, + DF_DB ": entry " DF_U64 " op %u <%td, %zd>: " DF_RC "\n", + DP_DB(db), index, op.dto_opc, p - buf, n, DP_RC(rc)); + else D_ERROR(DF_DB ": failed to apply entry " DF_U64 " op %u <%td, %zd>: " DF_RC "\n", DP_DB(db), index, op.dto_opc, p - buf, n, DP_RC(rc)); @@ -1087,7 +1092,7 @@ rdb_tx_apply(struct rdb *db, uint64_t index, const void *buf, size_t len, void * *(int *)result = rc; *critp = hdr.critical; - return 0; + return rc == 0 ? 0 : RDB_TX_APPLY_ERR_DETERMINISTIC; } /* Called at the beginning of every query. */ diff --git a/src/rdb/rdb_util.c b/src/rdb/rdb_util.c index 38e35e6f90f..14a89a8eb8b 100644 --- a/src/rdb/rdb_util.c +++ b/src/rdb/rdb_util.c @@ -259,7 +259,7 @@ rdb_vos_tx_end(struct rdb *db, rdb_vos_tx_t vtx, int err) int rc; rc = dtx_end(dth, NULL /* cont */, err); - if (rc != 0) + if (rc != err) DL_ERROR(rc, DF_DB ": failed to %s VOS TX", DP_DB(db), err == 0 ? "commit" : "abort"); diff --git a/src/rdb/tests/rdb_test.c b/src/rdb/tests/rdb_test.c index fad06e2da43..dcae690fc75 100644 --- a/src/rdb/tests/rdb_test.c +++ b/src/rdb/tests/rdb_test.c @@ -455,6 +455,8 @@ rdbt_test_tx(bool update, enum rdbt_membership_op memb_op, uint64_t user_key, char value_written[] = "value"; char buf[32]; uint64_t keys[] = {11, 22, 33, user_key}; + uint64_t to_be_abort_key = 111; /* must not in keys[] */ + char nonexistent_kvs[] = "nonexistent_kvs"; struct rdb_tx tx; struct iterate_cb_arg arg; uint64_t k = 0; @@ -497,6 +499,17 @@ rdbt_test_tx(bool update, enum rdbt_membership_op memb_op, uint64_t user_key, MUST(rdb_tx_commit(&tx)); rdb_tx_end(&tx); + D_WARN("commit deterministic-error tx\n"); + MUST(rdb_tx_begin(svc->rt_rsvc.s_db, RDB_NIL_TERM, &tx)); + d_iov_set(&key, &to_be_abort_key, sizeof(to_be_abort_key)); + d_iov_set(&value, value_written, strlen(value_written) + 1); + MUST(rdb_tx_update(&tx, &svc->rt_kvs1_path, &key, &value)); + d_iov_set(&key, nonexistent_kvs, sizeof(nonexistent_kvs)); + MUST(rdb_tx_destroy_kvs(&tx, &svc->rt_root_kvs_path, &key)); + rc = rdb_tx_commit(&tx); + D_ASSERTF(rc == -DER_NONEXIST, "%d == %d\n", rc, -DER_NONEXIST); + rdb_tx_end(&tx); + D_WARN("update: user record: (K=0x%"PRIx64", V="DF_U64")\n", user_key, user_val_in); MUST(rdb_tx_begin(svc->rt_rsvc.s_db, RDB_NIL_TERM, &tx)); @@ -551,6 +564,14 @@ rdbt_test_tx(bool update, enum rdbt_membership_op memb_op, uint64_t user_key, MUST(rdb_tx_begin_local(storage, &tx)); else MUST(rdb_tx_begin(svc->rt_rsvc.s_db, RDB_NIL_TERM, &tx)); + + /* Look up to_be_abort_key. */ + d_iov_set(&key, &to_be_abort_key, sizeof(to_be_abort_key)); + d_iov_set(&value, buf, sizeof(buf)); + value.iov_len = 0; /* no size check */ + rc = rdb_tx_lookup(&tx, &svc->rt_kvs1_path, &key, &value); + D_ASSERTF(rc == -DER_NONEXIST, "%d == %d\n", rc, -DER_NONEXIST); + /* Look up keys[0]. */ d_iov_set(&key, &keys[0], sizeof(keys[0])); d_iov_set(&value, buf, sizeof(buf)); diff --git a/src/rebuild/scan.c b/src/rebuild/scan.c index bd32d1d359b..d8479fe1220 100644 --- a/src/rebuild/scan.c +++ b/src/rebuild/scan.c @@ -839,6 +839,17 @@ rebuild_container_scan_cb(daos_handle_t ih, vos_iter_entry_t *entry, DP_UUID(entry->ie_couuid), DP_RC(rc)); D_GOTO(close, rc); } + + /* + * The container may has been closed by the application, but some resource (DRAM) occupied + * by DTX may be not released because DTX resync was in-progress at that time. When arrive + * here, DTX resync must has completed globally. Let's release related resource. + */ + if (unlikely(cont_child->sc_dtx_delay_reset == 1)) { + stop_dtx_reindex_ult(cont_child, true); + vos_dtx_cache_reset(cont_child->sc_hdl, false); + } + cont_child->sc_rebuilding = 1; rc = ds_cont_fetch_snaps(rpt->rt_pool->sp_iv_ns, entry->ie_couuid, NULL, diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index 624b90a6f4b..f40ea2d2fe3 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -1941,6 +1941,12 @@ ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver, return 0; } + if (ds_pool_skip_for_check(pool)) { + D_DEBUG(DB_REBUILD, DF_UUID" skip rebuild under check mode\n", + DP_UUID(pool->sp_uuid)); + return 0; + } + if (tgts != NULL && tgts->pti_number > 0 && rebuild_op != RB_OP_RECLAIM && rebuild_op != RB_OP_FAIL_RECLAIM) { /* Check if the pool already in the queue list */ diff --git a/src/tests/ftest/aggregation/dfuse_space_check.py b/src/tests/ftest/aggregation/dfuse_space_check.py index f95025328f3..1d119f807a7 100644 --- a/src/tests/ftest/aggregation/dfuse_space_check.py +++ b/src/tests/ftest/aggregation/dfuse_space_check.py @@ -7,6 +7,7 @@ import os import time +from dfuse_utils import get_dfuse, start_dfuse from ior_test_base import IorTestBase @@ -57,16 +58,19 @@ def wait_for_aggregation(self, retries=4, interval=60): self.log.info("Free space when test terminated: %s", current_space) self.fail("Aggregation did not complete within {} seconds".format(retries * interval)) - def write_multiple_files(self): + def write_multiple_files(self, dfuse): """Write multiple files. + Args: + dfuse (Dfuse): the dfuse object + Returns: int: Total number of files created before going out of space. """ file_count = 0 while self.get_nvme_free_space(False) >= self.block_size: - file_path = os.path.join(self.dfuse.mount_dir.value, "file{}.txt".format(file_count)) + file_path = os.path.join(dfuse.mount_dir.value, "file{}.txt".format(file_count)) write_dd_cmd = "dd if=/dev/zero of={} bs={} count=1".format(file_path, self.block_size) if 0 in self.execute_cmd(write_dd_cmd, fail_on_err=True, display_output=False): file_count += 1 @@ -106,13 +110,14 @@ def test_dfusespacecheck(self): # Create a pool, container, and start dfuse self.create_pool() self.create_cont() - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, self.pool, self.container) # get nvme space before write self.initial_space = self.get_nvme_free_space() # Create a file as large as we can - large_file = os.path.join(self.dfuse.mount_dir.value, 'largefile.txt') + large_file = os.path.join(dfuse.mount_dir.value, 'largefile.txt') self.execute_cmd('touch {}'.format(large_file)) dd_count = (self.initial_space // self.block_size) + 1 write_dd_cmd = "dd if=/dev/zero of={} bs={} count={}".format( @@ -130,14 +135,14 @@ def test_dfusespacecheck(self): self.pool.set_property("reclaim", "disabled") # Write small files until we run out of space - file_count1 = self.write_multiple_files() + file_count1 = self.write_multiple_files(dfuse) # Enable aggregation self.log.info("Enabling aggregation") self.pool.set_property("reclaim", "time") # remove all the small files created above. - self.execute_cmd("rm -rf {}".format(os.path.join(self.dfuse.mount_dir.value, '*'))) + self.execute_cmd("rm -rf {}".format(os.path.join(dfuse.mount_dir.value, '*'))) # Wait for aggregation to complete after file removal self.wait_for_aggregation() @@ -147,7 +152,7 @@ def test_dfusespacecheck(self): self.pool.set_property("reclaim", "disabled") # Write small files again until we run out of space and verify we wrote the same amount - file_count2 = self.write_multiple_files() + file_count2 = self.write_multiple_files(dfuse) self.log.info('file_count1 = %s', file_count1) self.log.info('file_count2 = %s', file_count2) diff --git a/src/tests/ftest/cart/test_proto_client.c b/src/tests/ftest/cart/test_proto_client.c index 6c14b735b22..ffcec7461e0 100644 --- a/src/tests/ftest/cart/test_proto_client.c +++ b/src/tests/ftest/cart/test_proto_client.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2018-2022 Intel Corporation. + * (C) Copyright 2018-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -65,6 +65,7 @@ test_run() uint32_t s_high_ver = 0xFFFFFFFF; uint32_t c_high_ver = test.tg_num_proto - 1; int rc; + uint32_t timeout; fprintf(stderr, "local group: %s remote group: %s\n", test.tg_local_group_name, test.tg_remote_group_name); @@ -119,8 +120,11 @@ test_run() server_ep.ep_rank = 0; DBG_PRINT("proto query\n"); - rc = crt_proto_query(&server_ep, OPC_MY_PROTO, my_ver_array, 7, - query_cb, &s_high_ver); + timeout = 1; + do { + rc = crt_proto_query(&server_ep, OPC_MY_PROTO, my_ver_array, 7, timeout++, query_cb, + &s_high_ver); + } while (rc == -DER_TIMEDOUT); D_ASSERT(rc == 0); while (s_high_ver == 0xFFFFFFFF) diff --git a/src/tests/ftest/control/dmg_pool_query_test.py b/src/tests/ftest/control/dmg_pool_query_test.py index a3cf6797a85..3c20281c227 100644 --- a/src/tests/ftest/control/dmg_pool_query_test.py +++ b/src/tests/ftest/control/dmg_pool_query_test.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2022 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -35,7 +35,7 @@ def test_pool_query_basic(self): :avocado: tags=all,daily_regression :avocado: tags=hw,medium - :avocado: tags=dmg,pool_query,basic,control + :avocado: tags=dmg,pool_query,basic,control,pool :avocado: tags=DmgPoolQueryTest,test_pool_query_basic """ self.log.info("==> Verify dmg output against expected output:") @@ -45,28 +45,37 @@ def test_pool_query_basic(self): # fluctuate across test runs. In addition, they're related to object # placement and testing them wouldn't be straightforward, so we'll need # some separate test cases. - del self.pool.query_data["response"]["tier_stats"][0]["free"] - del self.pool.query_data["response"]["tier_stats"][0]["min"] - del self.pool.query_data["response"]["tier_stats"][0]["max"] - del self.pool.query_data["response"]["tier_stats"][0]["mean"] - del self.pool.query_data["response"]["tier_stats"][1]["free"] - del self.pool.query_data["response"]["tier_stats"][1]["min"] - del self.pool.query_data["response"]["tier_stats"][1]["max"] - del self.pool.query_data["response"]["tier_stats"][1]["mean"] + for tier in self.pool.query_data["response"]["tier_stats"]: + del tier["free"] + del tier["min"] + del tier["max"] + del tier["mean"] + for usage in self.pool.query_data["response"]["usage"]: + del usage["free"] + del usage["imbalance"] # Get the expected pool query values from the test yaml. This should be as simple as: # exp_info = self.params.get("exp_vals", path="/run/*", default={}) # but this yields an empty dictionary (the default), so it needs to be defined manually: exp_info = { + "enabled_ranks": None, + "disabled_ranks": None, "status": self.params.get("pool_status", path="/run/exp_vals/*"), - 'state': self.params.get("pool_state", path="/run/exp_vals/*"), + "state": self.params.get("pool_state", path="/run/exp_vals/*"), "uuid": self.pool.uuid.lower(), "total_targets": self.params.get("total_targets", path="/run/exp_vals/*"), "active_targets": self.params.get("active_targets", path="/run/exp_vals/*"), "total_engines": self.params.get("total_engines", path="/run/exp_vals/*"), "disabled_targets": self.params.get("disabled_targets", path="/run/exp_vals/*"), "version": self.params.get("version", path="/run/exp_vals/*"), - "leader": self.params.get("leader", path="/run/exp_vals/*"), + "svc_ldr": self.params.get("leader", path="/run/exp_vals/*"), + "svc_reps": self.params.get("replicas", path="/run/exp_vals/*"), + "rebuild": { + "status": self.params.get("rebuild_status", path="/run/exp_vals/rebuild/*"), + "state": self.params.get("state", path="/run/exp_vals/rebuild/*"), + "objects": self.params.get("objects", path="/run/exp_vals/rebuild/*"), + "records": self.params.get("records", path="/run/exp_vals/rebuild/*") + }, "tier_stats": [ { "media_type": "scm", @@ -78,15 +87,18 @@ def test_pool_query_basic(self): } ], "pool_layout_ver": 3, + "query_mask": self.params.get("query_mask", path="/run/exp_vals/*"), "upgrade_layout_ver": 3, - "rebuild": { - "status": self.params.get("rebuild_status", path="/run/exp_vals/rebuild/*"), - "state": self.params.get("state", path="/run/exp_vals/rebuild/*"), - "objects": self.params.get("objects", path="/run/exp_vals/rebuild/*"), - "records": self.params.get("records", path="/run/exp_vals/rebuild/*") - }, - "enabled_ranks": None, - "disabled_ranks": None + "usage": [ + { + "tier_name": "SCM", + "size": self.params.get("total", path="/run/exp_vals/scm/*") + }, + { + "tier_name": "NVME", + "size": self.params.get("total", path="/run/exp_vals/nvme/*") + } + ] } self.assertDictEqual( @@ -105,7 +117,7 @@ def test_pool_query_inputs(self): :avocado: tags=all,daily_regression :avocado: tags=hw,medium - :avocado: tags=dmg,pool_query,basic,control + :avocado: tags=dmg,pool_query,basic,control,pool :avocado: tags=DmgPoolQueryTest,test_pool_query_inputs """ # Get test UUIDs @@ -152,7 +164,7 @@ def test_pool_query_ior(self): :avocado: tags=all,daily_regression :avocado: tags=hw,medium - :avocado: tags=dmg,pool_query,basic,control + :avocado: tags=dmg,pool_query,basic,control,pool :avocado: tags=DmgPoolQueryTest,test_pool_query_ior """ # Store original pool info diff --git a/src/tests/ftest/control/dmg_pool_query_test.yaml b/src/tests/ftest/control/dmg_pool_query_test.yaml index 52b4d2a47fa..d5aeacc49f1 100644 --- a/src/tests/ftest/control/dmg_pool_query_test.yaml +++ b/src/tests/ftest/control/dmg_pool_query_test.yaml @@ -33,6 +33,8 @@ exp_vals: disabled_targets: 0 version: 1 leader: 0 + replicas: [0] + query_mask: "rebuild,space" scm: total: 16000008192 nvme: diff --git a/src/tests/ftest/control/dmg_telemetry_basic.py b/src/tests/ftest/control/dmg_telemetry_basic.py index 7882ff0fe71..39eb520aef2 100644 --- a/src/tests/ftest/control/dmg_telemetry_basic.py +++ b/src/tests/ftest/control/dmg_telemetry_basic.py @@ -98,7 +98,7 @@ def test_container_telemetry(self): open_close_qty = self.params.get("open_close_qty", "/run/test/*", 2) self.add_pool(connect=False) self.pool.set_query_data() - pool_leader_rank = self.pool.query_data["response"]["leader"] + pool_leader_rank = self.pool.query_data["response"]["svc_ldr"] self.pool_leader_host = self.server_managers[0].get_host(pool_leader_rank) self.log.info( "Pool leader host: %s (rank: %s)", diff --git a/src/tests/ftest/daos_test/dfuse.py b/src/tests/ftest/daos_test/dfuse.py index 5c048115564..205d1cd11a6 100644 --- a/src/tests/ftest/daos_test/dfuse.py +++ b/src/tests/ftest/daos_test/dfuse.py @@ -7,13 +7,14 @@ import os from collections import OrderedDict +from apricot import TestWithServers from cmocka_utils import CmockaUtils -from dfuse_test_base import DfuseTestBase +from dfuse_utils import get_dfuse, start_dfuse from general_utils import create_directory, get_log_file from job_manager_utils import get_job_manager -class DaosCoreTestDfuse(DfuseTestBase): +class DaosCoreTestDfuse(TestWithServers): """Runs DAOS DFuse tests. :avocado: recursive @@ -29,11 +30,11 @@ def run_test(self, il_lib=None): if il_lib is None: self.fail('il_lib is not defined.') - self.daos_test = os.path.join(self.bin, 'dfuse_test') + daos_test = os.path.join(self.bin, 'dfuse_test') # Create a pool, container and start dfuse. - self.add_pool(connect=False) - self.add_container(self.pool) + pool = self.get_pool(connect=False) + container = self.get_container(pool) cont_attrs = OrderedDict() @@ -67,14 +68,15 @@ def run_test(self, il_lib=None): elif cache_mode == 'native': use_dfuse = False else: - self.fail('Invalid cache_mode: {}'.format(cache_mode)) + self.fail(f'Invalid cache_mode: {cache_mode}') if use_dfuse: - self.container.set_attr(attrs=cont_attrs) + container.set_attr(attrs=cont_attrs) - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool, container) - mount_dir = self.dfuse.mount_dir.value + mount_dir = dfuse.mount_dir.value else: # Bypass, simply create a remote directory and use that. mount_dir = '/tmp/dfuse-test' @@ -94,14 +96,14 @@ def run_test(self, il_lib=None): if il_lib == 'libpil4dfs.so': daos_test_env['D_IL_MOUNT_POINT'] = mount_dir - daos_test_env['D_IL_POOL'] = self.pool.identifier - daos_test_env['D_IL_CONTAINER'] = self.container.identifier + daos_test_env['D_IL_POOL'] = pool.identifier + daos_test_env['D_IL_CONTAINER'] = container.identifier daos_test_env['D_IL_REPORT'] = '0' daos_test_env['D_IL_MAX_EQ'] = '2' daos_test_env['D_IL_ENFORCE_EXEC_ENV'] = '1' command = [ - self.daos_test, + daos_test, '--test-dir', mount_dir, '--io', diff --git a/src/tests/ftest/datamover/negative.py b/src/tests/ftest/datamover/negative.py index aec63776a7e..3b05e1c8dfd 100644 --- a/src/tests/ftest/datamover/negative.py +++ b/src/tests/ftest/datamover/negative.py @@ -8,6 +8,7 @@ from os.path import join from data_mover_test_base import DataMoverTestBase +from dfuse_utils import get_dfuse, start_dfuse from duns_utils import format_path @@ -60,7 +61,8 @@ def test_dm_bad_params_dcp(self): self.set_tool("DCP") # Start dfuse to hold all pools/containers - self.start_dfuse(self.dfuse_hosts) + dfuse = get_dfuse(self, self.dfuse_hosts) + start_dfuse(self, dfuse) # Create a test pool pool1 = self.create_pool() @@ -69,7 +71,7 @@ def test_dm_bad_params_dcp(self): uns_cont = self.get_container(pool1) # Create a test container - cont1_path = join(self.dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns1') + cont1_path = join(dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns1') cont1 = self.get_container(pool1, path=cont1_path) # Create test files @@ -209,7 +211,8 @@ def test_dm_bad_params_fs_copy(self): self.set_tool("FS_COPY") # Start dfuse to hold all pools/containers - self.start_dfuse(self.dfuse_hosts) + dfuse = get_dfuse(self, self.dfuse_hosts) + start_dfuse(self, dfuse) # Create a test pool pool1 = self.create_pool() @@ -218,7 +221,7 @@ def test_dm_bad_params_fs_copy(self): uns_cont = self.get_container(pool1) # Create a test container - cont1_path = join(self.dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns1') + cont1_path = join(dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns1') cont1 = self.get_container(pool1, path=cont1_path) # Create test files diff --git a/src/tests/ftest/datamover/obj_small.py b/src/tests/ftest/datamover/obj_small.py index b1add0c5911..ed9ba5674b5 100644 --- a/src/tests/ftest/datamover/obj_small.py +++ b/src/tests/ftest/datamover/obj_small.py @@ -101,13 +101,6 @@ def run_dm_obj_small(self, tool): self.num_objs, self.num_dkeys, self.num_akeys_single, self.num_akeys_array, self.akey_sizes, self.akey_extents) - # Must destroy before closing pools - cont1.destroy() - cont2.destroy() - cont3.destroy() - pool1.disconnect() - pool2.disconnect() - @avocado.fail_on(DaosApiError) def test_dm_obj_small_dcp(self): """ diff --git a/src/tests/ftest/datamover/posix_meta_entry.py b/src/tests/ftest/datamover/posix_meta_entry.py index 40268fc7485..bb608c27853 100644 --- a/src/tests/ftest/datamover/posix_meta_entry.py +++ b/src/tests/ftest/datamover/posix_meta_entry.py @@ -6,6 +6,7 @@ from os.path import join from data_mover_test_base import DataMoverTestBase +from dfuse_utils import get_dfuse, start_dfuse from exception_utils import CommandFailure @@ -62,7 +63,8 @@ def run_dm_posix_meta_entry(self, tool): test_desc = self.test_id + " (preserve={})".format(str(preserve_on)) # Start dfuse to hold all pools/containers - self.start_dfuse(self.dfuse_hosts) + dfuse = get_dfuse(self, self.dfuse_hosts) + start_dfuse(self, dfuse) # Create 1 pool pool1 = self.create_pool() @@ -71,7 +73,7 @@ def run_dm_posix_meta_entry(self, tool): cont1 = self.get_container(pool1) daos_src_path = self.new_daos_test_path(False) dfuse_src_path = "{}/{}/{}{}".format( - self.dfuse.mount_dir.value, pool1.uuid, cont1.uuid, daos_src_path) + dfuse.mount_dir.value, pool1.uuid, cont1.uuid, daos_src_path) self.create_data(dfuse_src_path) # Create 1 source posix path with test data @@ -84,7 +86,7 @@ def run_dm_posix_meta_entry(self, tool): # DAOS -> DAOS daos_dst_path = self.new_daos_test_path(False) dfuse_dst_path = "{}/{}/{}{}".format( - self.dfuse.mount_dir.value, pool1.uuid, cont1.uuid, daos_dst_path) + dfuse.mount_dir.value, pool1.uuid, cont1.uuid, daos_dst_path) self.run_datamover( test_desc + "(DAOS->DAOS)", "DAOS", daos_src_path, pool1, cont1, @@ -105,7 +107,7 @@ def run_dm_posix_meta_entry(self, tool): # POSIX -> DAOS daos_dst_path = self.new_daos_test_path(False) dfuse_dst_path = "{}/{}/{}{}".format( - self.dfuse.mount_dir.value, pool1.uuid, cont1.uuid, daos_dst_path) + dfuse.mount_dir.value, pool1.uuid, cont1.uuid, daos_dst_path) self.run_datamover( test_desc + "(POSIX->DAOS)", "POSIX", posix_src_path, None, None, diff --git a/src/tests/ftest/datamover/posix_subsets.py b/src/tests/ftest/datamover/posix_subsets.py index bdce75a0ed8..45e33d9cec9 100644 --- a/src/tests/ftest/datamover/posix_subsets.py +++ b/src/tests/ftest/datamover/posix_subsets.py @@ -7,6 +7,7 @@ from os.path import join from data_mover_test_base import DataMoverTestBase +from dfuse_utils import get_dfuse, start_dfuse class DmvrPosixSubsets(DataMoverTestBase): @@ -49,7 +50,8 @@ def run_dm_posix_subsets(self, tool): self.set_tool(tool) # Start dfuse to hold all pools/containers - self.start_dfuse(self.dfuse_hosts) + dfuse = get_dfuse(self, self.dfuse_hosts) + start_dfuse(self, dfuse) # Create 1 pool pool1 = self.create_pool() @@ -57,13 +59,12 @@ def run_dm_posix_subsets(self, tool): # create dfuse containers to test copying to dfuse subdirectories dfuse_cont1 = self.get_container(pool1) dfuse_cont2 = self.get_container(pool1) - dfuse_src_dir = join(self.dfuse.mount_dir.value, pool1.uuid, dfuse_cont1.uuid) + dfuse_src_dir = join(dfuse.mount_dir.value, pool1.uuid, dfuse_cont1.uuid) # Create a special container to hold UNS entries uns_cont = self.get_container(pool1) # Create two test containers - container1_path = join( - self.dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns1') + container1_path = join(dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns1') container1 = self.get_container(pool1, path=container1_path) container2 = self.get_container(pool1) @@ -128,7 +129,7 @@ def run_dm_posix_subsets(self, tool): dfuse_dst_dir = self.new_posix_test_path( create=False, - parent=join(self.dfuse.mount_dir.value, pool1.uuid, dfuse_cont2.uuid)) + parent=join(dfuse.mount_dir.value, pool1.uuid, dfuse_cont2.uuid)) copy_list.append([ "copy_subsets (dfuse root to new dfuse dir)", ["POSIX", dfuse_src_dir, None, None], diff --git a/src/tests/ftest/datamover/posix_symlinks.py b/src/tests/ftest/datamover/posix_symlinks.py index cdffdd9085e..68d60e4c973 100644 --- a/src/tests/ftest/datamover/posix_symlinks.py +++ b/src/tests/ftest/datamover/posix_symlinks.py @@ -6,6 +6,7 @@ from os.path import join from data_mover_test_base import DataMoverTestBase +from dfuse_utils import get_dfuse, start_dfuse class DmvrPosixSymlinks(DataMoverTestBase): @@ -55,7 +56,8 @@ def run_dm_posix_symlinks(self, tool): self.set_tool(tool) # Start dfuse to hold all pools/containers - self.start_dfuse(self.dfuse_hosts) + dfuse = get_dfuse(self, self.dfuse_hosts) + start_dfuse(self, dfuse) # Create 1 pool pool1 = self.create_pool() @@ -64,19 +66,19 @@ def run_dm_posix_symlinks(self, tool): uns_cont = self.get_container(pool1) # Test links that point forward - container1_path = join(self.dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns1') + container1_path = join(dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns1') container1 = self.get_container(pool1, path=container1_path) self.run_dm_posix_symlinks_fun( pool1, container1, self.create_links_forward, "forward") # Test links that point backward - container2_path = join(self.dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns2') + container2_path = join(dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns2') container2 = self.get_container(pool1, path=container2_path) self.run_dm_posix_symlinks_fun( pool1, container2, self.create_links_backward, "backward") # Test a mix of forward and backward links - container3_path = join(self.dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns3') + container3_path = join(dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns3') container3 = self.get_container(pool1, path=container3_path) self.run_dm_posix_symlinks_fun( pool1, container3, self.create_links_mixed, "mixed") diff --git a/src/tests/ftest/datamover/posix_types.py b/src/tests/ftest/datamover/posix_types.py index 44321355588..0ef85d018a8 100644 --- a/src/tests/ftest/datamover/posix_types.py +++ b/src/tests/ftest/datamover/posix_types.py @@ -6,6 +6,7 @@ from os.path import basename, join from data_mover_test_base import DataMoverTestBase +from dfuse_utils import get_dfuse, start_dfuse from duns_utils import format_path, parse_path @@ -63,7 +64,8 @@ def run_dm_posix_types(self, tool): self.set_tool(tool) # Start dfuse to hold all pools/containers - self.start_dfuse(self.dfuse_hosts) + dfuse = get_dfuse(self, self.dfuse_hosts) + start_dfuse(self, dfuse) # Create 2 pools pool1 = self.create_pool(label='pool1') @@ -73,11 +75,11 @@ def run_dm_posix_types(self, tool): uns_cont = self.get_container(pool1) # Create all other containers - container1_path = join(self.dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns1') + container1_path = join(dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns1') container1 = self.get_container(pool1, path=container1_path, label='container1') - container2_path = join(self.dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns2') + container2_path = join(dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns2') container2 = self.get_container(pool1, path=container2_path, label='container2') - container3_path = join(self.dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns3') + container3_path = join(dfuse.mount_dir.value, pool1.uuid, uns_cont.uuid, 'uns3') container3 = self.get_container(pool2, path=container3_path, label='container3') # Create each source location diff --git a/src/tests/ftest/datamover/serial_large_posix.py b/src/tests/ftest/datamover/serial_large_posix.py index f2d7c336fea..6917097d901 100644 --- a/src/tests/ftest/datamover/serial_large_posix.py +++ b/src/tests/ftest/datamover/serial_large_posix.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' from data_mover_test_base import DataMoverTestBase +from dfuse_utils import get_dfuse, start_dfuse from duns_utils import format_path @@ -54,8 +55,9 @@ def run_dm_serial_large_posix(self, tool): # Use dfuse as a shared intermediate for serialize + deserialize dfuse_cont = self.get_container(pool1) - self.start_dfuse(self.dfuse_hosts, pool1, dfuse_cont) - self.serial_tmp_dir = self.dfuse.mount_dir.value + dfuse = get_dfuse(self, self.dfuse_hosts) + start_dfuse(self, dfuse, pool1, dfuse_cont) + self.serial_tmp_dir = dfuse.mount_dir.value # Serialize/Deserialize cont1 to a new cont2 in pool2 result = self.run_datamover( diff --git a/src/tests/ftest/datamover/serial_small.py b/src/tests/ftest/datamover/serial_small.py index 241f3aa9307..28ce84bee35 100644 --- a/src/tests/ftest/datamover/serial_small.py +++ b/src/tests/ftest/datamover/serial_small.py @@ -88,12 +88,6 @@ def run_dm_serial_small(self, tool): self.num_objs, self.num_dkeys, self.num_akeys_single, self.num_akeys_array, self.akey_sizes, self.akey_extents) - # Must destroy before closing pools - cont1.destroy() - cont2.destroy() - pool1.disconnect() - pool2.disconnect() - @avocado.fail_on(DaosApiError) def test_dm_serial_small_dserialize(self): """ diff --git a/src/tests/ftest/dbench/dbench.py b/src/tests/ftest/dbench/dbench.py index 05081fb4039..fbbb60f3961 100644 --- a/src/tests/ftest/dbench/dbench.py +++ b/src/tests/ftest/dbench/dbench.py @@ -4,13 +4,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from apricot import TestWithServers from ClusterShell.NodeSet import NodeSet from dbench_utils import Dbench -from dfuse_test_base import DfuseTestBase +from dfuse_utils import get_dfuse, start_dfuse from exception_utils import CommandFailure -class DbenchTest(DfuseTestBase): +class DbenchTest(TestWithServers): # pylint: disable=too-few-public-methods """Base Dbench test class. @@ -36,14 +37,18 @@ def test_dbench(self): :avocado: tags=DbenchTest,test_dbench """ - self.add_pool(connect=False) - self.add_container(self.pool) - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + self.log_step('Creating a single pool and container') + pool = self.get_pool(connect=False) + container = self.get_container(pool) + self.log_step('Starting dfuse') + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool, container) + + self.log_step('Running dbench') dbench_cmd = Dbench(self.hostlist_clients, self.tmp) dbench_cmd.get_params(self) - dbench_cmd.directory.update(self.dfuse.mount_dir.value) - + dbench_cmd.directory.update(dfuse.mount_dir.value) try: # Start dfuse dbench_cmd.run() @@ -53,9 +58,4 @@ def test_dbench(self): str(NodeSet.fromlist(dbench_cmd.hosts)), exc_info=error) self.fail("Test was expected to pass but it failed.") - # stop dfuse - self.stop_dfuse() - # destroy container - self.container.destroy() - # destroy pool - self.pool.destroy() + self.log.info('Test passed') diff --git a/src/tests/ftest/deployment/critical_integration.py b/src/tests/ftest/deployment/critical_integration.py index 8afdf18cfcc..c8b9b296f3c 100644 --- a/src/tests/ftest/deployment/critical_integration.py +++ b/src/tests/ftest/deployment/critical_integration.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -93,6 +93,7 @@ def test_passwdlessssh_versioncheck(self): result_client_server = daos_server_version_list[0] == dmg_version_list[0] # libfabric version check + # pylint: disable-next=unsupported-binary-operation all_nodes = self.hostlist_servers | self.hostlist_clients libfabric_version_cmd = "clush -S -b -w {} {}/fi_info --version".format( all_nodes, libfabric_path) diff --git a/src/tests/ftest/dfuse/bash.py b/src/tests/ftest/dfuse/bash.py index aea5fea6c72..e59e9aebb96 100644 --- a/src/tests/ftest/dfuse/bash.py +++ b/src/tests/ftest/dfuse/bash.py @@ -5,11 +5,13 @@ """ import os -from dfuse_test_base import DfuseTestBase +from apricot import TestWithServers +from dfuse_utils import get_dfuse, start_dfuse +from host_utils import get_local_host from run_utils import run_remote -class Cmd(DfuseTestBase): +class DfuseBashCmd(TestWithServers): """Base Cmd test class. :avocado: recursive @@ -54,29 +56,31 @@ def run_bashcmd(self, il_lib=None, compatible_mode=False): env_str = "" # Create a pool if one does not already exist. - self.add_pool(connect=False) - self.add_container(self.pool) - mount_dir = f"/tmp/{self.pool.identifier}_daos_dfuse" - self.start_dfuse(self.hostlist_clients, self.pool, self.container, mount_dir=mount_dir) + self.log_step('Creating a single pool and container') + pool = self.get_pool(connect=False) + container = self.get_container(pool) + + self.log_step('Starting dfuse') + dfuse_hosts = get_local_host() + dfuse = get_dfuse(self, dfuse_hosts) + params = {'mount_dir': f'/tmp/{pool.identifier}_daos_dfuse'} if il_lib is not None: - # unmount dfuse and mount again with caching disabled - self.dfuse.unmount(tries=1) - self.dfuse.update_params(disable_caching=True) - self.dfuse.update_params(disable_wb_cache=True) - self.dfuse.run() + params['disable_caching'] = True + params['disable_wb_cache'] = True + start_dfuse(self, dfuse, pool, container, **params) - fuse_root_dir = self.dfuse.mount_dir.value + fuse_root_dir = dfuse.mount_dir.value abs_dir_path = os.path.join(fuse_root_dir, "test") abs_file_path1 = os.path.join(abs_dir_path, "testfile1.txt") abs_file_path2 = os.path.join(abs_dir_path, "testfile2.txt") - with open(os.path.join(fuse_root_dir, "src.c"), "w") as fd: + with open(os.path.join(fuse_root_dir, "src.c"), "w", encoding="utf-8") as fd: fd.write('#include <stdio.h>\n\nint main(void) {\nprintf("Hello World!");\n}\n') link_name = os.path.join(fuse_root_dir, "link_c") - with open(os.path.join(fuse_root_dir, "src_a.c"), "w") as fd: + with open(os.path.join(fuse_root_dir, "src_a.c"), "w", encoding="utf-8") as fd: fd.write('#include <stdio.h>\n\nvoid fun_a(void) {\nprintf("fun_a()");\n}\n') - with open(os.path.join(fuse_root_dir, "src_b.c"), "w") as fd: + with open(os.path.join(fuse_root_dir, "src_b.c"), "w", encoding="utf-8") as fd: fd.write('#include <stdio.h>\n\nvoid fun_b(void) {\nprintf("fun_b()");\n}\n') # list of commands to be executed. commands = [ @@ -127,21 +131,16 @@ def run_bashcmd(self, il_lib=None, compatible_mode=False): f'{fuse_root_dir}/ --bs=1M --numjobs="4" --ioengine=psync ' "--group_reporting --exitall_on_error --continue_on_error=none", 'fio --readwrite=randwrite --name=test --size="2M" --directory ' - f'{fuse_root_dir}/ --bs=1M --numjobs="1" --ioengine=libaio --iodepth=4' + f'{fuse_root_dir}/ --bs=1M --numjobs="1" --ioengine=libaio --iodepth=16' '--group_reporting --exitall_on_error --continue_on_error=none', f'curl "https://www.google.com" -o {fuse_root_dir}/download.html', ] for cmd in commands: - result = run_remote(self.log, self.hostlist_clients, env_str + cmd) + self.log_step(f'Running command: {cmd}') + result = run_remote(self.log, dfuse_hosts, env_str + cmd) if not result.passed: self.fail(f'"{cmd}" failed on {result.failed_hosts}') - - # stop dfuse - self.stop_dfuse() - # destroy container - self.container.destroy() - # destroy pool - self.pool.destroy() + self.log.info('Test passed') def test_bashcmd(self): """ @@ -154,7 +153,7 @@ def test_bashcmd(self): :avocado: tags=all,daily_regression :avocado: tags=vm :avocado: tags=dfuse,dfs - :avocado: tags=Cmd,test_bashcmd + :avocado: tags=DfuseBashCmd,test_bashcmd """ self.run_bashcmd() @@ -168,8 +167,8 @@ def test_bashcmd_ioil(self): :avocado: tags=all,pr,daily_regression :avocado: tags=vm - :avocado: tags=dfuse,il,dfs - :avocado: tags=Cmd,test_bashcmd_ioil + :avocado: tags=dfuse,dfs,ioil + :avocado: tags=DfuseBashCmd,test_bashcmd_ioil """ self.run_bashcmd(il_lib="libioil.so") @@ -183,7 +182,7 @@ def test_bashcmd_pil4dfs(self): :avocado: tags=all,daily_regression :avocado: tags=vm - :avocado: tags=dfuse,pil4dfs,dfs - :avocado: tags=Cmd,test_bashcmd_pil4dfs + :avocado: tags=dfuse,dfs,pil4dfs + :avocado: tags=DfuseBashCmd,test_bashcmd_pil4dfs """ self.run_bashcmd(il_lib="libpil4dfs.so") diff --git a/src/tests/ftest/dfuse/bash_dcache.py b/src/tests/ftest/dfuse/bash_dcache.py index 55e83bbcd63..306228da283 100644 --- a/src/tests/ftest/dfuse/bash_dcache.py +++ b/src/tests/ftest/dfuse/bash_dcache.py @@ -7,7 +7,9 @@ import os import stat -from dfuse_test_base import DfuseTestBase +from apricot import TestWithServers +from dfuse_utils import get_dfuse, start_dfuse +from host_utils import get_local_host from run_utils import run_remote SCRIPT = """#!/bin/bash @@ -33,7 +35,7 @@ """ -class DFuseBashdcacheTest(DfuseTestBase): +class DFuseBashdcacheTest(TestWithServers): # pylint: disable=wrong-spelling-in-docstring """Base "Bashdcache" test class. @@ -58,18 +60,19 @@ def test_bash_dcache_pil4dfs(self): pool = self.get_pool(connect=False) container = self.get_container(pool) - self.start_dfuse(self.hostlist_clients, pool, container) + dfuse_hosts = get_local_host() + dfuse = get_dfuse(self, dfuse_hosts) + start_dfuse(self, dfuse, pool, container) + fuse_root_dir = dfuse.mount_dir.value - fuse_root_dir = self.dfuse.mount_dir.value - - with open(os.path.join(fuse_root_dir, "sh_dcache.sh"), "w") as fd: + with open(os.path.join(fuse_root_dir, "sh_dcache.sh"), "w", encoding="utf-8") as fd: fd.write(SCRIPT) os.chmod(os.path.join(fuse_root_dir, "sh_dcache.sh"), stat.S_IXUSR | stat.S_IRUSR) cmd = f"cd {fuse_root_dir}; ./sh_dcache.sh" - result = run_remote(self.log, self.hostlist_clients, env_str + cmd) + result = run_remote(self.log, dfuse_hosts, env_str + cmd) if not result.passed: self.fail(f'"{cmd}" failed on {result.failed_hosts}') if result.output[0].stdout[0][:5] != "Hello": @@ -78,6 +81,6 @@ def test_bash_dcache_pil4dfs(self): # Turn on directory caching in bash env_str = env_str + "export D_IL_NO_DCACHE_BASH=0; " - result = run_remote(self.log, self.hostlist_clients, env_str + cmd) + result = run_remote(self.log, dfuse_hosts, env_str + cmd) if result.passed: self.fail(f'"{cmd}" failed on {result.failed_hosts}') diff --git a/src/tests/ftest/dfuse/bash_fd.py b/src/tests/ftest/dfuse/bash_fd.py index a8a39d61a51..925e96e8a9c 100644 --- a/src/tests/ftest/dfuse/bash_fd.py +++ b/src/tests/ftest/dfuse/bash_fd.py @@ -7,7 +7,9 @@ import os import stat -from dfuse_test_base import DfuseTestBase +from apricot import TestWithServers +from dfuse_utils import get_dfuse, start_dfuse +from host_utils import get_local_host from run_utils import run_remote OUTER = """#!/bin/bash @@ -77,7 +79,7 @@ """ -class DFuseFdTest(DfuseTestBase): +class DFuseFdTest(TestWithServers): """Base FdTest test class. :avocado: recursive @@ -98,28 +100,35 @@ def run_bashfd(self, il_lib=None): else: env_str = "" + self.log_step('Creating a single pool and container') pool = self.get_pool(connect=False) container = self.get_container(pool) - self.start_dfuse(self.hostlist_clients, pool, container) - fuse_root_dir = self.dfuse.mount_dir.value + self.log_step('Starting dfuse') + dfuse_hosts = get_local_host() + dfuse = get_dfuse(self, dfuse_hosts) + start_dfuse(self, dfuse, pool, container) + fuse_root_dir = dfuse.mount_dir.value - with open(os.path.join(fuse_root_dir, "bash_fd_inner.sh"), "w") as fd: + self.log_step("Setting up the 'bash_fd_inner.sh' script") + with open(os.path.join(fuse_root_dir, "bash_fd_inner.sh"), "w", encoding="utf-8") as fd: fd.write(INNER) - os.chmod(os.path.join(fuse_root_dir, "bash_fd_inner.sh"), stat.S_IXUSR | stat.S_IRUSR) - with open(os.path.join(fuse_root_dir, "bash_fd_outer.sh"), "w") as fd: + self.log_step("Setting up the 'bash_fd_outer.sh' script") + with open(os.path.join(fuse_root_dir, "bash_fd_outer.sh"), "w", encoding="utf-8") as fd: fd.write(OUTER) - os.chmod(os.path.join(fuse_root_dir, "bash_fd_outer.sh"), stat.S_IXUSR | stat.S_IRUSR) cmd = f"cd {fuse_root_dir}; ./bash_fd_outer.sh" - result = run_remote(self.log, self.hostlist_clients, env_str + cmd) + self.log_step("Executing the 'bash_fd_outer.sh' script") + result = run_remote(self.log, dfuse_hosts, env_str + cmd) if not result.passed: self.fail(f'"{cmd}" failed on {result.failed_hosts}') + self.log.info('Test passed') + def test_bashfd(self): """ @@ -141,7 +150,7 @@ def test_bashfd_ioil(self): :avocado: tags=all,full_regression :avocado: tags=vm - :avocado: tags=dfuse,il,dfs + :avocado: tags=dfuse,dfs,ioil :avocado: tags=DFuseFdTest,test_bashfd_ioil """ self.run_bashfd(il_lib="libioil.so") @@ -154,7 +163,7 @@ def test_bashfd_pil4dfs(self): :avocado: tags=all,full_regression :avocado: tags=vm - :avocado: tags=pil4dfs,dfs + :avocado: tags=dfuse,dfs,pil4dfs :avocado: tags=DFuseFdTest,test_bashfd_pil4dfs """ self.run_bashfd(il_lib="libpil4dfs.so") diff --git a/src/tests/ftest/dfuse/container_type.py b/src/tests/ftest/dfuse/container_type.py index 592bce1cb6e..b14a3d7c4b6 100644 --- a/src/tests/ftest/dfuse/container_type.py +++ b/src/tests/ftest/dfuse/container_type.py @@ -1,13 +1,15 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ + +from apricot import TestWithServers from avocado.core.exceptions import TestFail -from dfuse_test_base import DfuseTestBase +from dfuse_utils import get_dfuse, start_dfuse -class DfuseContainerCheck(DfuseTestBase): +class DfuseContainerCheck(TestWithServers): """Base Dfuse Container check test class. :avocado: recursive @@ -31,40 +33,46 @@ def test_dfuse_container_check(self): :avocado: tags=DfuseContainerCheck,test_dfuse_container_check """ # get test params for cont and pool count - cont_types = self.params.get("cont_types", '/run/container/*') + cont_types = self.params.get('cont_types', '/run/container/*') # Create a pool and start dfuse. - self.add_pool(connect=False) + self.log_step('Creating a single pool') + pool = self.get_pool(connect=False) for cont_type in cont_types: + description = f"{cont_type if cont_type == 'POSIX' else 'non-POSIX'}" # Get container params - self.add_container(self.pool, create=False) + self.log_step(f'Creating a {description} container') + container = self.get_container(pool, create=False) # create container - if cont_type == "POSIX": - self.container.type.update(cont_type) - self.container.create() + if cont_type == 'POSIX': + container.type.update(cont_type) + container.create() # Attempt to mount the dfuse mount point - this should only succeed # with a POSIX container + self.log_step(f'Attempting to mount dfuse with a {description} container') + dfuse = get_dfuse(self, self.hostlist_clients) try: - self.start_dfuse( - self.hostlist_clients, self.pool, self.container) - if cont_type != "POSIX": - self.fail("Non-POSIX type container mounted over dfuse") + start_dfuse(self, dfuse, pool, container) + if cont_type != 'POSIX': + self.fail(f'Dfuse mount succeeded with a {description} container') except TestFail as error: - if cont_type == "POSIX": - self.fail( - "POSIX type container failed dfuse mount: {}".format( - error)) - self.log.info( - "Non-POSIX type container expected to fail dfuse mount") + if cont_type == 'POSIX': + self.log.debug('%s', error) + self.fail(f'Dfuse failed to mount with a {description} container') + self.log.info('Dfuse mount expected to fail with a %s container', description) + + # Verify dfuse is running on the POSIX type container, then stop dfuse + if cont_type == 'POSIX': + self.log_step(f'Verifying dfuse is running with a {description} container') + dfuse.check_running() + self.log_step(f'Stopping dfuse with a {description} container') + dfuse.stop() - # Verify dfuse is running on the POSIX type container - if cont_type == "POSIX": - self.dfuse.check_running() + # Destroy the container for next iteration + self.log_step(f'Destroying a {description} container') + container.destroy(1) - # Stop dfuse and destroy the container for next iteration - if not cont_type == "": - self.stop_dfuse() - self.container.destroy(1) + self.log.info('Test passed') diff --git a/src/tests/ftest/dfuse/daos_build.py b/src/tests/ftest/dfuse/daos_build.py index 29d6b27bee3..03ab81cba1f 100644 --- a/src/tests/ftest/dfuse/daos_build.py +++ b/src/tests/ftest/dfuse/daos_build.py @@ -7,11 +7,12 @@ import os import time -from dfuse_test_base import DfuseTestBase +from apricot import TestWithServers +from dfuse_utils import get_dfuse, start_dfuse from run_utils import run_remote -class DaosBuild(DfuseTestBase): +class DaosBuild(TestWithServers): """Build DAOS over dfuse. :avocado: recursive @@ -132,8 +133,9 @@ def test_dfuse_daos_build_nocache(self): def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False): """Run an actual test from above.""" # Create a pool, container and start dfuse. - self.add_pool(connect=False) - self.add_container(self.pool) + self.log_step('Creating a single pool and container') + pool = self.get_pool(connect=False) + container = self.get_container(pool) cont_attrs = {} @@ -165,7 +167,7 @@ def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False): build_jobs = 5 * 2 remote_env['D_IL_MAX_EQ'] = '0' - self.load_dfuse(self.hostlist_clients, dfuse_namespace) + dfuse = get_dfuse(self, self.hostlist_clients, dfuse_namespace) if cache_mode == 'writeback': cont_attrs['dfuse-data-cache'] = '1m' @@ -179,39 +181,39 @@ def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False): cont_attrs['dfuse-attr-time'] = cache_time cont_attrs['dfuse-dentry-time'] = cache_time cont_attrs['dfuse-ndentry-time'] = cache_time - self.dfuse.disable_wb_cache.value = True + dfuse.disable_wb_cache.value = True elif cache_mode == 'metadata': cont_attrs['dfuse-data-cache'] = '1m' cont_attrs['dfuse-attr-time'] = cache_time cont_attrs['dfuse-dentry-time'] = cache_time cont_attrs['dfuse-ndentry-time'] = cache_time - self.dfuse.disable_wb_cache.value = True + dfuse.disable_wb_cache.value = True elif cache_mode == 'data': build_time *= 2 cont_attrs['dfuse-data-cache'] = '1m' cont_attrs['dfuse-attr-time'] = '0' cont_attrs['dfuse-dentry-time'] = '0' cont_attrs['dfuse-ndentry-time'] = '0' - self.dfuse.disable_wb_cache.value = True + dfuse.disable_wb_cache.value = True elif cache_mode == 'nocache': build_time *= 4 cont_attrs['dfuse-data-cache'] = 'off' cont_attrs['dfuse-attr-time'] = '0' cont_attrs['dfuse-dentry-time'] = '0' cont_attrs['dfuse-ndentry-time'] = '0' - self.dfuse.disable_wb_cache.value = True - self.dfuse.disable_caching.value = True + dfuse.disable_wb_cache.value = True + dfuse.disable_caching.value = True else: - self.fail('Invalid cache_mode: {}'.format(cache_mode)) + self.fail(f'Invalid cache_mode: {cache_mode}') - self.container.set_attr(attrs=cont_attrs) + self.log_step('Starting dfuse') + container.set_attr(attrs=cont_attrs) + start_dfuse(self, dfuse, pool, container) - self.start_dfuse(self.hostlist_clients, self.pool, self.container) - - mount_dir = self.dfuse.mount_dir.value + mount_dir = dfuse.mount_dir.value build_dir = os.path.join(mount_dir, 'daos') - remote_env['PATH'] = '{}:$PATH'.format(os.path.join(mount_dir, 'venv', 'bin')) + remote_env['PATH'] = f"{os.path.join(mount_dir, 'venv', 'bin')}:$PATH" remote_env['VIRTUAL_ENV'] = os.path.join(mount_dir, 'venv') remote_env['COVFILE'] = os.environ['COVFILE'] @@ -226,7 +228,7 @@ def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False): remote_env['D_IL_COMPATIBLE'] = '1' remote_env['D_IL_MAX_EQ'] = '0' - envs = ['export {}={}'.format(env, value) for env, value in remote_env.items()] + envs = [f'export {env}={value}' for env, value in remote_env.items()] preload_cmd = ';'.join(envs) @@ -250,6 +252,7 @@ def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False): timeout = 10 * 60 if cmd.startswith('scons'): timeout = build_time * 60 + self.log_step(f'Running \'{cmd}\' with a {timeout}s timeout') start = time.time() result = run_remote( self.log, self.hostlist_clients, command, verbose=True, timeout=timeout) @@ -272,6 +275,8 @@ def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False): run_remote(self.log, self.hostlist_clients, 'cat {}/config.log'.format(build_dir), timeout=30) if il_lib is not None: - self.fail('{} over dfuse with il in mode {}.\n'.format(fail_type, cache_mode)) + self.fail(f'{fail_type} over dfuse with il in mode {cache_mode}') else: - self.fail('{} over dfuse in mode {}.\n'.format(fail_type, cache_mode)) + self.fail(f'{fail_type} over dfuse in mode {cache_mode}') + + self.log.info('Test passed') diff --git a/src/tests/ftest/dfuse/enospace.py b/src/tests/ftest/dfuse/enospace.py index 4f4c1136baa..52f904acf03 100644 --- a/src/tests/ftest/dfuse/enospace.py +++ b/src/tests/ftest/dfuse/enospace.py @@ -7,10 +7,12 @@ import errno import os -from dfuse_test_base import DfuseTestBase +from apricot import TestWithServers +from dfuse_utils import get_dfuse, start_dfuse +from host_utils import get_local_host -class DfuseEnospace(DfuseTestBase): +class DfuseEnospace(TestWithServers): """Dfuse ENOSPC File base class. :avocado: recursive @@ -34,15 +36,22 @@ def test_dfuse_enospace(self): :avocado: tags=daosio,dfuse :avocado: tags=DfuseEnospace,test_dfuse_enospace """ + dfuse_hosts = get_local_host() + # Create a pool, container and start dfuse. - self.add_pool(connect=False) - self.add_container(self.pool) - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + self.log_step('Creating a single pool with a POSIX container') + pool = self.get_pool(connect=False) + container = self.get_container(pool) + + self.log_step('Starting dfuse') + dfuse = get_dfuse(self, dfuse_hosts) + start_dfuse(self, dfuse, pool, container) # create large file and perform write to it so that if goes out of # space. - target_file = os.path.join(self.dfuse.mount_dir.value, "file.txt") + target_file = os.path.join(dfuse.mount_dir.value, 'file.txt') + self.log_step('Write to file until an error occurs') with open(target_file, 'wb', buffering=0) as fd: # Use a write size of 128. On EL 8 this could be 1MiB, however older kernels @@ -75,3 +84,5 @@ def test_dfuse_enospace(self): except OSError as error: if error.errno != errno.ENOSPC: raise + + self.log.info('Test passed') diff --git a/src/tests/ftest/dfuse/fio_pil4dfs_small.py b/src/tests/ftest/dfuse/fio_pil4dfs_small.py index 5e391325ee8..66ec6b2085e 100644 --- a/src/tests/ftest/dfuse/fio_pil4dfs_small.py +++ b/src/tests/ftest/dfuse/fio_pil4dfs_small.py @@ -6,6 +6,7 @@ import os +from dfuse_utils import get_dfuse, start_dfuse from fio_test_base import FioBase @@ -39,4 +40,10 @@ def test_fio_pil4dfs_small(self): :avocado: tags=FioPil4dfsSmall,test_fio_pil4dfs_small """ self.fio_cmd.env['LD_PRELOAD'] = os.path.join(self.prefix, 'lib64', 'libpil4dfs.so') + pool = self.get_pool(connect=False) + container = self.get_container(pool) + container.set_attr(attrs={'dfuse-direct-io-disable': 'on'}) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool, container) + self.fio_cmd.update_directory(dfuse.mount_dir.value) self.execute_fio() diff --git a/src/tests/ftest/dfuse/fio_small.py b/src/tests/ftest/dfuse/fio_small.py index 69aee6b2507..fea62668f0e 100644 --- a/src/tests/ftest/dfuse/fio_small.py +++ b/src/tests/ftest/dfuse/fio_small.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from dfuse_utils import get_dfuse, start_dfuse from fio_test_base import FioBase @@ -37,4 +38,10 @@ def test_fio_small(self): :avocado: tags=dfuse,fio,checksum,tx :avocado: tags=FioSmall,test_fio_small """ + pool = self.get_pool(connect=False) + container = self.get_container(pool) + container.set_attr(attrs={'dfuse-direct-io-disable': 'on'}) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool, container) + self.fio_cmd.update_directory(dfuse.mount_dir.value) self.execute_fio() diff --git a/src/tests/ftest/dfuse/mu_mount.py b/src/tests/ftest/dfuse/mu_mount.py index 8f2369583d1..32ace91abf7 100644 --- a/src/tests/ftest/dfuse/mu_mount.py +++ b/src/tests/ftest/dfuse/mu_mount.py @@ -6,12 +6,13 @@ import os from getpass import getuser +from apricot import TestWithServers from ClusterShell.NodeSet import NodeSet -from dfuse_test_base import DfuseTestBase +from dfuse_utils import get_dfuse, start_dfuse from run_utils import command_as_user, run_remote -class DfuseMUMount(DfuseTestBase): +class DfuseMUMount(TestWithServers): """Verifies multi-user dfuse mounting :avocado: recursive @@ -35,18 +36,18 @@ def test_dfuse_mu_mount_basic(self): :avocado: tags=dfuse,dfuse_mu,daos_cmd :avocado: tags=DfuseMUMount,test_dfuse_mu_mount_basic """ - self.log.info('Creating a pool and container for dfuse') + self.log_step('Creating a pool and container for dfuse') pool = self.get_pool(connect=False) cont = self.get_container(pool, label='root_cont') # Setup dfuse - self.load_dfuse(self.hostlist_clients) - self.dfuse.update_params(pool=pool.identifier, cont=cont.label.value) - root_dir = self.dfuse.mount_dir.value + dfuse = get_dfuse(self, self.hostlist_clients) + dfuse.update_params(pool=pool.identifier, cont=cont.label.value) + root_dir = dfuse.mount_dir.value # Use a different log file for each user - root_log_file = self.dfuse.env["D_LOG_FILE"] + ".root" - dfuse_user_log_file = self.dfuse.env["D_LOG_FILE"] + "." + getuser() + root_log_file = dfuse.env["D_LOG_FILE"] + ".root" + dfuse_user_log_file = dfuse.env["D_LOG_FILE"] + "." + getuser() # For verifying expected permission failure def _check_fail(result): @@ -54,53 +55,54 @@ def _check_fail(result): if 'DER_NO_PERM' not in stdout: self.fail('Expected mount as root to fail without ACLs in single-user mode') - self.log.info('Verify root cannot mount the container without ACLs in single-user mode') - self.dfuse.update_params(multi_user=False) - self.dfuse.run_user = 'root' - self.dfuse.env["D_LOG_FILE"] = root_log_file - self.dfuse.run(check=False, mount_callback=_check_fail) - self.dfuse.stop() - - self.log.info('Verify root cannot mount the container without ACLs in multi-user mode') - self.dfuse.update_params(multi_user=True) - self.dfuse.run_user = 'root' - self.dfuse.env["D_LOG_FILE"] = root_log_file - self.dfuse.run(check=False, mount_callback=_check_fail) - self.dfuse.stop() - - self.log.info('Mounting dfuse in single-user mode') - self.dfuse.update_params(multi_user=False) - self.dfuse.run_user = None # Current user - self.dfuse.env["D_LOG_FILE"] = dfuse_user_log_file - self.dfuse.run() - - self.log.info('Verify stat as dfuse user in single-user mode succeeds') + self.log_step('Verify root cannot mount the container without ACLs in single-user mode') + dfuse.update_params(multi_user=False) + dfuse.run_user = 'root' + dfuse.env["D_LOG_FILE"] = root_log_file + dfuse.run(check=False, mount_callback=_check_fail) + dfuse.stop() + + self.log_step('Verify root cannot mount the container without ACLs in multi-user mode') + dfuse.update_params(multi_user=True) + dfuse.run_user = 'root' + dfuse.env["D_LOG_FILE"] = root_log_file + dfuse.run(check=False, mount_callback=_check_fail) + dfuse.stop() + + self.log_step('Mounting dfuse in single-user mode') + dfuse.update_params(multi_user=False) + dfuse.run_user = None # Current user + dfuse.env["D_LOG_FILE"] = dfuse_user_log_file + dfuse.run() + + self.log_step('Verify stat as dfuse user in single-user mode succeeds') command = 'stat {}'.format(root_dir) if not run_remote(self.log, self.hostlist_clients, command).passed: self.fail('Failed to stat in single-user mode') - self.log.info('Verify stat as root user in single-user mode fails') + self.log_step('Verify stat as root user in single-user mode fails') command = command_as_user('stat {}'.format(root_dir), 'root') if run_remote(self.log, self.hostlist_clients, command).passed: self.fail('Expected stat to fail as root in single-user mode') - self.log.info('Re-mounting dfuse in multi-user mode') - self.dfuse.stop() - self.dfuse.update_params(multi_user=True) - self.dfuse.run() + self.log_step('Re-mounting dfuse in multi-user mode') + dfuse.stop() + dfuse.update_params(multi_user=True) + dfuse.run() - self.log.info('Verify stat as dfuse user in multi-user mode succeeds') + self.log_step('Verify stat as dfuse user in multi-user mode succeeds') command = 'stat {}'.format(root_dir) if not run_remote(self.log, self.hostlist_clients, command).passed: self.fail('Failed to stat in multi-user mode') - self.log.info('Verify stat as root user in multi-user mode succeeds') + self.log_step('Verify stat as root user in multi-user mode succeeds') command = command_as_user('stat {}'.format(root_dir), 'root') if not run_remote(self.log, self.hostlist_clients, command).passed: self.fail('Failed to stat as root in multi-user mode') # Cleanup leftover dfuse - self.dfuse.stop() + self.log_step('Cleanup leftover dfuse') + dfuse.stop() # Give root permission to read the pool pool.update_acl(False, entry="A::root@:r") @@ -108,19 +110,19 @@ def _check_fail(result): # Give root permission to read the container and access properties cont.update_acl(entry="A::root@:rt") - self.log.info('Verify root can mount the container with ACLs in single-user mode') - self.dfuse.update_params(multi_user=False) - self.dfuse.run_user = 'root' - self.dfuse.env["D_LOG_FILE"] = root_log_file - self.dfuse.run() - self.dfuse.stop() + self.log_step('Verify root can mount the container with ACLs in single-user mode') + dfuse.update_params(multi_user=False) + dfuse.run_user = 'root' + dfuse.env["D_LOG_FILE"] = root_log_file + dfuse.run() + dfuse.stop() - self.log.info('Verify root can mount the container with ACLs in muli-user mode') - self.dfuse.update_params(multi_user=True) - self.dfuse.run_user = 'root' - self.dfuse.env["D_LOG_FILE"] = dfuse_user_log_file - self.dfuse.run() - self.dfuse.stop() + self.log_step('Verify root can mount the container with ACLs in multi-user mode') + dfuse.update_params(multi_user=True) + dfuse.run_user = 'root' + dfuse.env["D_LOG_FILE"] = dfuse_user_log_file + dfuse.run() + dfuse.stop() def test_dfuse_mu_mount_uns(self): """JIRA ID: DAOS-10859. @@ -139,20 +141,19 @@ def test_dfuse_mu_mount_uns(self): """ dfuse_user = getuser() - self.log.info('Creating a pool and container for dfuse') + self.log_step('Creating a pool and container for dfuse') pool1 = self.get_pool(connect=False) cont1 = self.get_container(pool1, label='root_cont') - self.log.info('Starting dfuse with the current user in multi-user mode') - self.load_dfuse(self.hostlist_clients) - self.dfuse.update_params(pool=pool1.identifier, cont=cont1.label.value, multi_user=True) - root_dir = self.dfuse.mount_dir.value - self.dfuse.run() + self.log_step('Starting dfuse with the current user in multi-user mode') + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool1, cont1, multi_user=True) + root_dir = dfuse.mount_dir.value - self.log.info('Creating a second pool') + self.log_step('Creating a second pool') pool2 = self.get_pool(connect=False) - self.log.info('Giving root permission to create containers in both pools') + self.log_step('Giving root permission to create containers in both pools') pool1.update_acl(False, entry="A::root@:rw") pool2.update_acl(False, entry="A::root@:rw") @@ -171,7 +172,7 @@ def _verify_uns(pool_label, cont_label): if not run_remote(self.log, first_client, command).passed: self.fail('Failed to create sub-container as root in multi-user mode') - self.log.info('Verify dfuse user was automatically given ACLs for the new container') + self.log_step('Verify dfuse user was automatically given ACLs for the new container') expected_acl = 'A::{}@:rwt'.format(dfuse_user) self.log.info('Expected ACL: %s', expected_acl) command = command_as_user( @@ -183,22 +184,22 @@ def _verify_uns(pool_label, cont_label): if expected_acl not in stdout: self.fail('Expected ACL for container created by root: {}'.format(expected_acl)) - self.log.info('Verify dfuse user can access the container created by root') + self.log_step('Verify dfuse user can access the container created by root') command = '{} container get-prop --path {}'.format(daos_path, cont_path) if not run_remote(self.log, first_client, command).passed: self.fail('Failed to get sub-container properties in multi-user mode') - self.log.info('Verify dfuse user can read the container created by root') + self.log_step('Verify dfuse user can read the container created by root') command = 'ls -l {}'.format(cont_path) if not run_remote(self.log, first_client, command).passed: self.fail('Failed to read container created by root, as dfuse user') - self.log.info('Verify root can read its own container') + self.log_step('Verify root can read its own container') command = command_as_user('ls -l {}'.format(cont_path), 'root') if not run_remote(self.log, first_client, command).passed: self.fail('Failed to read container created by root, as root') - self.log.info('Revoking ACLs for just dfuse user on the container created by root') + self.log_step('Revoking ACLs for just dfuse user on the container created by root') principal = 'u:{}@'.format(dfuse_user) command = command_as_user( '{} container delete-acl --path {} --principal {}'.format( @@ -207,22 +208,22 @@ def _verify_uns(pool_label, cont_label): if not run_remote(self.log, first_client, command).passed: self.fail('Failed to revoke ACLs on container created by root') - self.log.info('Restarting dfuse to pick up ACL changes') - self.dfuse.stop() - self.dfuse.run() + self.log_step('Restarting dfuse to pick up ACL changes') + dfuse.stop() + dfuse.run() - self.log.info('Verifying dfuse user can no longer read the container through dfuse') + self.log_step('Verifying dfuse user can no longer read the container through dfuse') command = 'ls -l {}'.format(cont_path) if run_remote(self.log, first_client, command).passed: self.fail('Expected ls to fail on container created by root, as dfuse user') - self.log.info('Verifying root can no longer read the container through dfuse') + self.log_step('Verifying root can no longer read the container through dfuse') command = command_as_user('ls -l {}'.format(cont_path), 'root') if run_remote(self.log, first_client, command).passed: self.fail('Expected ls to fail on container created by root, as root') - self.log.info('Verify UNS sub-container create as root - in dfuse pool') + self.log_step('Verify UNS sub-container create as root - in dfuse pool') _verify_uns(pool_label="", cont_label='pool1_root_cont') - self.log.info('Verify UNS sub-container create as root - in different pool') + self.log_step('Verify UNS sub-container create as root - in different pool') _verify_uns(pool_label=pool2.identifier, cont_label='pool2_root_cont') diff --git a/src/tests/ftest/dfuse/mu_perms.py b/src/tests/ftest/dfuse/mu_perms.py index ffe1c91e8b9..a6fc558f157 100644 --- a/src/tests/ftest/dfuse/mu_perms.py +++ b/src/tests/ftest/dfuse/mu_perms.py @@ -71,12 +71,12 @@ def test_dfuse_mu_perms(self): # Run dfuse as dfuse_user dfuse = get_dfuse(self, client) dfuse.run_user = dfuse_user - start_dfuse(self, dfuse, pool=pool, container=cont) + start_dfuse(self, dfuse, pool, cont) # Verify each permission mode and entry type for _mode, _type in product(('simple', 'real'), ('file', 'dir')): path = os.path.join(dfuse.mount_dir.value, 'test_' + _type) - self.log.info('Verifying %s %s permissions on %s', _mode, _type, path) + self.log_step(f'Verifying {_mode} {_type} permissions on {path}') verify_perms_cmd.update_params(path=path, create_type=_type, verify_mode=_mode) verify_perms_cmd.run() self.log.info('Passed %s %s permissions on %s', _mode, _type, path) @@ -93,7 +93,7 @@ def test_dfuse_mu_perms(self): # Verify real permissions for _type in ('file', 'dir'): path = os.path.join(sub_dir, 'test_' + _type) - self.log.info('Verifying real %s permissions on %s', _type, path) + self.log_step(f'Verifying real {_type} permissions on {path}') verify_perms_cmd.update_params(path=path, create_type=_type, verify_mode='real') verify_perms_cmd.run() self.log.info('Passed real %s permissions on %s', _type, path) @@ -111,13 +111,12 @@ def test_dfuse_mu_perms(self): # Verify real permissions for _type in ('file', 'dir'): path = os.path.join(sub_dir, 'test_' + _type) - self.log.info('Verifying real %s permissions on %s', _type, path) + self.log_step(f'Verifying real {_type} permissions on {path}') verify_perms_cmd.update_params(path=path, create_type=_type, verify_mode='real') verify_perms_cmd.run() self.log.info('Passed real %s permissions on %s', _type, path) - # Stop dfuse instances. Needed until containers are cleaned up with with register_cleanup - dfuse.stop() + self.log.info('Test passed') def _create_dir_and_chown(self, client, path, create_as, owner, group=None): """Create a directory and give some user and group ownership. @@ -130,21 +129,21 @@ def _create_dir_and_chown(self, client, path, create_as, owner, group=None): group (str): group to give ownership to """ - self.log.info('Creating directory: %s', path) + self.log_step('Creating directory: %s', path) command = command_as_user('mkdir ' + path, create_as) if not run_remote(self.log, client, command).passed: - self.fail('Failed to create directory: {}'.format(path)) + self.fail(f'Failed to create directory: {path}') if group: - self.log.info('Giving ownership to %s:%s', owner, group) + self.log_step(f'Giving ownership to {owner}:{group}') else: - self.log.info('Giving ownership to %s', owner) + self.log_step(f'Giving ownership to {owner}') command = command_as_user(get_chown_command(user=owner, group=group, file=path), 'root') if not run_remote(self.log, client, command).passed: - self.fail('Failed to give ownership to {}'.format(owner)) - command = command_as_user('stat {}'.format(path), owner) + self.fail(f'Failed to give ownership to {owner}') + command = command_as_user(f'stat {path}', owner) if not run_remote(self.log, client, command).passed: - self.fail('Failed to stat {}'.format(path)) + self.fail(f'Failed to stat {path}') def test_dfuse_mu_perms_cache(self): """Jira ID: DAOS-10858. @@ -248,9 +247,7 @@ def _wait_for_cache_expiration(): path=dfuse2_entry_path, verify_mode='real', perms='000', no_chmod=True) verify_perms_cmd.run() - # Stop dfuse instances. Needed until containers are cleaned up with with register_cleanup - dfuse1.stop() - dfuse2.stop() + self.log.info('Test passed') def run_test_il(self, il_lib=None): """Jira ID: DAOS-10857. @@ -350,7 +347,7 @@ def _verify(use_il, expected_il_messages, expect_der_no_perm): dfuse_entry_path = os.path.join(dfuse.mount_dir.value, entry_type) create_cmd = 'touch' if entry_type == 'file' else 'mkdir' - self.log.info('Creating a test %s in dfuse', entry_type) + self.log_step('Creating a test %s in dfuse', entry_type) command = command_as_user('{} "{}"'.format(create_cmd, dfuse_entry_path), dfuse_user) if not run_remote(self.log, self.hostlist_clients, command).passed: self.fail('Failed to create test {}'.format(entry_type)) @@ -359,7 +356,7 @@ def _verify(use_il, expected_il_messages, expect_der_no_perm): # Revoke POSIX permissions posix_perms = {'file': '600', 'dir': '600'}[entry_type] - self.log.info('Setting %s POSIX permissions to %s', entry_type, posix_perms) + self.log_step(f'Setting {entry_type} POSIX permissions to {posix_perms}') command = command_as_user( 'chmod {} "{}"'.format(posix_perms, dfuse_entry_path), dfuse_user) if not run_remote(self.log, self.hostlist_clients, command).passed: @@ -368,14 +365,14 @@ def _verify(use_il, expected_il_messages, expect_der_no_perm): # Without pool/container ACLs, access is based on POSIX perms, # which the user also doesn't have verify_perms_cmd.update_params(perms=posix_perms) - self.log.info('Verify - no perms - not using IL') + self.log_step('Verify - no perms - not using IL') _verify(use_il=False, expected_il_messages=0, expect_der_no_perm=False) - self.log.info('Verify - no perms - using IL') + self.log_step('Verify - no perms - using IL') _verify(use_il=True, expected_il_messages=2, expect_der_no_perm=False) # Give the user POSIX perms posix_perms = {'file': '606', 'dir': '505'}[entry_type] - self.log.info('Setting %s POSIX permissions to %s', entry_type, posix_perms) + self.log_step(f'Setting {entry_type} POSIX permissions to {posix_perms}') command = command_as_user( 'chmod {} "{}"'.format(posix_perms, dfuse_entry_path), dfuse_user) if not run_remote(self.log, self.hostlist_clients, command).passed: @@ -383,26 +380,26 @@ def _verify(use_il, expected_il_messages, expect_der_no_perm): # With POSIX perms only, access is based on POSIX perms whether using IL or not verify_perms_cmd.update_params(perms=posix_perms) - self.log.info('Verify - POSIX perms only - not using IL') + self.log_step('Verify - POSIX perms only - not using IL') _verify(use_il=False, expected_il_messages=0, expect_der_no_perm=False) - self.log.info('Verify - POSIX perms only - using IL') + self.log_step('Verify - POSIX perms only - using IL') _verify(use_il=True, expected_il_messages=2, expect_der_no_perm=True) # Give the user pool/container ACL perms - self.log.info('Giving %s pool "r" ACL permissions', other_user) + self.log_step('Giving %s pool "r" ACL permissions', other_user) pool.update_acl(use_acl=False, entry="A::{}@:r".format(other_user)) - self.log.info('Giving %s container "rwt" ACL permissions', other_user) + self.log_step('Giving %s container "rwt" ACL permissions', other_user) cont.update_acl(entry="A::{}@:rwt".format(other_user)) # With POSIX perms and ACLs, open is based on POSIX, but IO is based on ACLs - self.log.info('Verify - POSIX and ACL perms - not using IL') + self.log_step('Verify - POSIX and ACL perms - not using IL') _verify(use_il=False, expected_il_messages=0, expect_der_no_perm=False) - self.log.info('Verify - POSIX and ACL perms - using IL') + self.log_step('Verify - POSIX and ACL perms - using IL') _verify(use_il=True, expected_il_messages=4, expect_der_no_perm=False) # Revoke POSIX permissions posix_perms = {'file': '600', 'dir': '00'}[entry_type] - self.log.info('Setting %s POSIX permissions to %s', entry_type, posix_perms) + self.log_step(f'Setting {entry_type} POSIX permissions to {posix_perms}') command = command_as_user( 'chmod {} "{}"'.format(posix_perms, dfuse_entry_path), dfuse_user) if not run_remote(self.log, self.hostlist_clients, command).passed: @@ -410,14 +407,11 @@ def _verify(use_il, expected_il_messages, expect_der_no_perm): # Without POSIX permissions, pool/container ACLs don't matter since open requires POSIX verify_perms_cmd.update_params(perms=posix_perms) - self.log.info('Verify - ACLs only - not using IL') + self.log_step('Verify - ACLs only - not using IL') _verify(use_il=False, expected_il_messages=0, expect_der_no_perm=False) - self.log.info('Verify - ACLs only - using IL') + self.log_step('Verify - ACLs only - using IL') _verify(use_il=True, expected_il_messages=2, expect_der_no_perm=False) - # Stop dfuse instances. Needed until containers are cleaned up with with register_cleanup - dfuse.stop() - def test_dfuse_mu_perms_ioil(self): """ :avocado: tags=all,daily_regression @@ -426,6 +420,7 @@ def test_dfuse_mu_perms_ioil(self): :avocado: tags=DfuseMUPerms,test_dfuse_mu_perms_ioil """ self.run_test_il(il_lib='libioil.so') + self.log.info('Test passed') def test_dfuse_mu_perms_pil4dfs(self): """ @@ -435,3 +430,4 @@ def test_dfuse_mu_perms_pil4dfs(self): :avocado: tags=DfuseMUPerms,test_dfuse_mu_perms_pil4dfs """ self.run_test_il(il_lib='libpil4dfs.so') + self.log.info('Test passed') diff --git a/src/tests/ftest/dfuse/pil4dfs_dcache.py b/src/tests/ftest/dfuse/pil4dfs_dcache.py index 8cb18de389c..683b95a0066 100644 --- a/src/tests/ftest/dfuse/pil4dfs_dcache.py +++ b/src/tests/ftest/dfuse/pil4dfs_dcache.py @@ -7,12 +7,12 @@ import os import re +from apricot import TestWithServers from ClusterShell.NodeSet import NodeSet -from dfuse_test_base import DfuseTestBase -from dfuse_utils import Pil4dfsDcacheCmd +from dfuse_utils import Pil4dfsDcacheCmd, get_dfuse, start_dfuse -class Pil4dfsDcache(DfuseTestBase): +class Pil4dfsDcache(TestWithServers): """Test class Description: Runs set of unit test on pil4dfs directory cache. :avocado: recursive @@ -357,15 +357,21 @@ def setUp(self): super().setUp() def _mount_dfuse(self): - """Mount a DFuse mount point.""" + """Mount a DFuse mount point. + + Returns: + Dfuse: a Dfuse object + """ self.log.info("Creating DAOS pool") - self.add_pool() + pool = self.get_pool() self.log.info("Creating DAOS container") - self.add_container(self.pool) + container = self.get_container(pool) self.log.info("Mounting DFuse mount point") - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool, container) + return dfuse def _update_cmd_env(self, env, mnt, **kwargs): """Update the Pil4dfsDcacheCmd command environment. @@ -446,12 +452,12 @@ def test_pil4dfs_dcache_enabled(self): :avocado: tags=Pil4dfsDcache,test_pil4dfs_dcache_enabled """ self.log_step("Mount a DFuse mount point") - self._mount_dfuse() + dfuse = self._mount_dfuse() self.log.info("Running pil4dfs_dcache command") hostname = self.hostlist_clients[0] host = NodeSet(hostname) - mnt = self.dfuse.mount_dir.value + mnt = dfuse.mount_dir.value cmd = Pil4dfsDcacheCmd(host, self.prefix) self._update_cmd_env(cmd.env, mnt) @@ -480,12 +486,12 @@ def test_pil4dfs_dcache_disabled(self): :avocado: tags=Pil4dfsDcache,test_pil4dfs_dcache_disabled """ self.log_step("Mount a DFuse mount point") - self._mount_dfuse() + dfuse = self._mount_dfuse() self.log.info("Running pil4dfs_dcache command") hostname = self.hostlist_clients[0] host = NodeSet(hostname) - mnt = self.dfuse.mount_dir.value + mnt = dfuse.mount_dir.value cmd = Pil4dfsDcacheCmd(host, self.prefix) env_kwargs = {"D_IL_DCACHE_REC_TIMEOUT": 0} self._update_cmd_env(cmd.env, mnt, **env_kwargs) @@ -515,12 +521,12 @@ def test_pil4dfs_dcache_gc_disabled(self): :avocado: tags=Pil4dfsDcache,test_pil4dfs_dcache_gc_disabled """ self.log_step("Mount a DFuse mount point") - self._mount_dfuse() + dfuse = self._mount_dfuse() self.log_step("Run pil4dfs_dcache command") hostname = self.hostlist_clients[0] host = NodeSet(hostname) - mnt = self.dfuse.mount_dir.value + mnt = dfuse.mount_dir.value cmd = Pil4dfsDcacheCmd(host, self.prefix) env_kwargs = {"D_IL_DCACHE_GC_PERIOD": 0} self._update_cmd_env(cmd.env, mnt, **env_kwargs) diff --git a/src/tests/ftest/dfuse/pil4dfs_fio.py b/src/tests/ftest/dfuse/pil4dfs_fio.py index 49121e38237..c33e9be92dc 100644 --- a/src/tests/ftest/dfuse/pil4dfs_fio.py +++ b/src/tests/ftest/dfuse/pil4dfs_fio.py @@ -7,14 +7,15 @@ import json import os +from apricot import TestWithServers from ClusterShell.NodeSet import NodeSet from cpu_utils import CpuInfo -from dfuse_test_base import DfuseTestBase +from dfuse_utils import get_dfuse, start_dfuse from fio_utils import FioCommand from general_utils import bytes_to_human -class Pil4dfsFio(DfuseTestBase): +class Pil4dfsFio(TestWithServers): """Test class Description: Runs Fio with in small config. :avocado: recursive @@ -26,7 +27,6 @@ def __init__(self, *args, **kwargs): """Initialize a FioPil4dfs object.""" super().__init__(*args, **kwargs) - self.fio_cmd = None self.fio_params = {"thread": "", "blocksize": "", "size": ""} self.fio_numjobs = 0 self.fio_cpus_allowed = "" @@ -57,24 +57,10 @@ def setUp(self): def _create_container(self): """Created a DAOS POSIX container""" self.log.info("Creating pool") - self.assertIsNone(self.pool, "Unexpected pool before starting test") - self.add_pool() + pool = self.get_pool() self.log.info("Creating container") - self.assertIsNone(self.container, "Unexpected container before starting test") - self.add_container(self.pool) - - def _destroy_container(self): - """Destroy DAOS POSIX container previously created""" - if self.container is not None: - self.log.debug("Destroying container %s", str(self.container)) - self.destroy_containers(self.container) - self.container = None - - if self.pool is not None: - self.log.debug("Destroying pool %s", str(self.pool)) - self.destroy_pools(self.pool) - self.pool = None + return self.get_container(pool) def _get_bandwidth(self, fio_result, rw): """Returns FIO bandwidth of a given I/O pattern @@ -101,42 +87,40 @@ def _run_fio_dfuse(self): dict: Read and Write bandwidths of the FIO command. """ - self._create_container() - self.log.info("Mounting DFuse mount point") - self.start_dfuse(self.hostlist_clients, self.pool, self.container) - self.log.debug("Mounted DFuse mount point %s", str(self.dfuse)) - - self.fio_cmd = FioCommand() - self.fio_cmd.get_params(self) - self.fio_cmd.update( - "global", "directory", self.dfuse.mount_dir.value, - f"fio --name=global --directory={self.dfuse.mount_dir.value}") - self.fio_cmd.update("global", "ioengine", "psync", "fio --name=global --ioengine='psync'") - self.fio_cmd.update( + container = self._create_container() + + self.log_step("Mounting DFuse mount point") + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, container.pool, container) + self.log.debug("Mounted DFuse mount point %s", str(dfuse)) + + fio_cmd = FioCommand() + fio_cmd.get_params(self) + fio_cmd.update_directory(dfuse.mount_dir.value) + fio_cmd.update("global", "ioengine", "psync", "fio --name=global --ioengine='psync'") + fio_cmd.update( "global", "numjobs", self.fio_numjobs, f"fio --name=global --numjobs={self.fio_numjobs}") - self.fio_cmd.update( + fio_cmd.update( "global", "cpus_allowed", self.fio_cpus_allowed, f"fio --name=global --cpus_allowed={self.fio_cpus_allowed}") - self.fio_cmd.env['LD_PRELOAD'] = os.path.join(self.prefix, 'lib64', 'libpil4dfs.so') - self.fio_cmd.hosts = self.hostlist_clients + fio_cmd.env['LD_PRELOAD'] = os.path.join(self.prefix, 'lib64', 'libpil4dfs.so') + fio_cmd.hosts = self.hostlist_clients bws = {} for rw in Pil4dfsFio._FIO_RW_NAMES: - self.fio_cmd.update("job", "rw", rw, f"fio --name=job --rw={rw}") + fio_cmd.update("job", "rw", rw, f"fio --name=job --rw={rw}") params = ", ".join(f"{name}={value}" for name, value in self.fio_params.items()) self.log.info("Running FIO command: rw=%s, %s", rw, params) - self.log.debug( - "FIO command: LD_PRELOAD=%s %s", self.fio_cmd.env['LD_PRELOAD'], str(self.fio_cmd)) - result = self.fio_cmd.run() + self.log.debug("FIO command: LD_PRELOAD=%s %s", fio_cmd.env['LD_PRELOAD'], str(fio_cmd)) + result = fio_cmd.run() bws[rw] = self._get_bandwidth(result, rw) self.log.debug("DFuse bandwidths for %s: %s", rw, bws[rw]) - if self.dfuse is not None: - self.log.debug("Stopping DFuse mount point %s", str(self.dfuse)) - self.stop_dfuse() - self._destroy_container() + dfuse.stop() + container.destroy() + container.pool.destroy() return bws @@ -147,38 +131,36 @@ def _run_fio_dfs(self): dict: Read and Write bandwidths of the FIO command. """ - self._create_container() + container = self._create_container() - self.fio_cmd = FioCommand() - self.fio_cmd.get_params(self) - self.fio_cmd.update("global", "ioengine", "dfs", "fio --name=global --ioengine='dfs'") - self.fio_cmd.update( + fio_cmd = FioCommand() + fio_cmd.get_params(self) + fio_cmd.update("global", "ioengine", "dfs", "fio --name=global --ioengine='dfs'") + fio_cmd.update( "global", "numjobs", self.fio_numjobs, f"fio --name=global --numjobs={self.fio_numjobs}") - self.fio_cmd.update( + fio_cmd.update( "global", "cpus_allowed", self.fio_cpus_allowed, f"fio --name=global --cpus_allowed={self.fio_cpus_allowed}") # NOTE DFS ioengine options must come after the ioengine that defines them is selected. - self.fio_cmd.update( - "job", "pool", self.pool.uuid, - f"fio --name=job --pool={self.pool.uuid}") - self.fio_cmd.update( - "job", "cont", self.container.uuid, - f"fio --name=job --cont={self.container.uuid}") - self.fio_cmd.hosts = self.hostlist_clients + fio_cmd.update( + "job", "pool", container.pool.uuid, f"fio --name=job --pool={container.pool.uuid}") + fio_cmd.update("job", "cont", container.uuid, f"fio --name=job --cont={container.uuid}") + fio_cmd.hosts = self.hostlist_clients bws = {} for rw in Pil4dfsFio._FIO_RW_NAMES: - self.fio_cmd.update("job", "rw", rw, f"fio --name=job --rw={rw}") + fio_cmd.update("job", "rw", rw, f"fio --name=job --rw={rw}") params = ", ".join(f"{name}={value}" for name, value in self.fio_params.items()) self.log.info("Running FIO command: rw=%s, %s", rw, params) - self.log.debug("FIO command: %s", str(self.fio_cmd)) - result = self.fio_cmd.run() + self.log.debug("FIO command: %s", str(fio_cmd)) + result = fio_cmd.run() bws[rw] = self._get_bandwidth(result, rw) self.log.debug("DFS bandwidths for %s: %s", rw, bws[rw]) - self._destroy_container() + container.destroy() + container.pool.destroy() return bws diff --git a/src/tests/ftest/dfuse/read.py b/src/tests/ftest/dfuse/read.py index b01b6655d19..607b8f99f9f 100644 --- a/src/tests/ftest/dfuse/read.py +++ b/src/tests/ftest/dfuse/read.py @@ -5,11 +5,12 @@ import time -from dfuse_test_base import DfuseTestBase +from apricot import TestWithServers +from dfuse_utils import get_dfuse, start_dfuse from run_utils import run_remote -class DFusePreReadTest(DfuseTestBase): +class DFusePreReadTest(TestWithServers): """Base ReadTest test class. :avocado: recursive """ @@ -31,7 +32,7 @@ def test_dfuse_pre_read(self): pool = self.get_pool(connect=False) container = self.get_container(pool) - self.load_dfuse(self.hostlist_clients, None) + dfuse = get_dfuse(self, self.hostlist_clients) cont_attrs = {} @@ -42,9 +43,9 @@ def test_dfuse_pre_read(self): container.set_attr(attrs=cont_attrs) - self.start_dfuse(self.hostlist_clients, pool, container) + start_dfuse(self, dfuse, pool, container) - fuse_root_dir = self.dfuse.mount_dir.value + fuse_root_dir = dfuse.mount_dir.value # make a directory to run the test from. Pre-read is based on previous access to a # directory so this needs to be evicted after the write and before the test so the @@ -77,7 +78,7 @@ def test_dfuse_pre_read(self): time.sleep(1) # Sample the stats, later on we'll check this. - data = self.dfuse.get_stats() + data = dfuse.get_stats() # Check that the inode has been evicted, and there's been no reads so far. self.assertEqual(data["inodes"], 1, "Incorrect number of active nodes") @@ -92,7 +93,7 @@ def test_dfuse_pre_read(self): if not result.passed: self.fail(f'"{cmd}" failed on {result.failed_hosts}') - data = self.dfuse.get_stats() + data = dfuse.get_stats() # pre_read requests are a subset of reads so for this test we should verify that they are # equal, and non-zero. @@ -111,7 +112,7 @@ def test_dfuse_pre_read(self): if not result.passed: self.fail(f'"{cmd}" failed on {result.failed_hosts}') - data = self.dfuse.get_stats() + data = dfuse.get_stats() # pre_read requests are a subset of reads so for this test we should verify that they are # equal, and non-zero. diff --git a/src/tests/ftest/dfuse/root_container.py b/src/tests/ftest/dfuse/root_container.py index efce567ea01..8daed813027 100644 --- a/src/tests/ftest/dfuse/root_container.py +++ b/src/tests/ftest/dfuse/root_container.py @@ -1,61 +1,21 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ -from dfuse_test_base import DfuseTestBase +import os + +from apricot import TestWithServers +from dfuse_utils import get_dfuse, start_dfuse from run_utils import run_remote -class RootContainerTest(DfuseTestBase): +class RootContainerTest(TestWithServers): """Base Dfuse Container check test class. :avocado: recursive """ - def __init__(self, *args, **kwargs): - """Initialize a RootContainerTest object.""" - super().__init__(*args, **kwargs) - self.pool = [] - self.container = [] - self.tmp_file_count = self.params.get("tmp_file_count", '/run/container/*') - self.cont_count = self.params.get("cont_count", '/run/container/*') - self.tmp_file_size = self.params.get("tmp_file_size", '/run/container/*') - self.tmp_file_name = self.params.get("tmp_file_name", '/run/container/*') - # device where the pools and containers are created - self.device = "scm" - - def setUp(self): - """Set up each test case.""" - # Start the servers and agents - super().setUp() - self.dfuse_hosts = None - - def _create_pool(self): - """Add a new TestPool object to the list of pools. - - Returns: - TestPool: the newly added pool - - """ - self.pool.append(self.get_pool(connect=False)) - return self.pool[-1] - - def _create_cont(self, pool, **params): - """Add a new TestContainer object to the list of containers. - - Args: - pool (TestPool): pool object - params (dict, optional): name/value of container attributes to update - - Returns: - TestContainer: the newly added container - - """ - container = self.get_container(pool, **params) - self.container.append(container) - return container - def test_dfuse_root_container(self): """Jira ID: DAOS-3782. @@ -75,40 +35,79 @@ def test_dfuse_root_container(self): :avocado: tags=container,dfuse :avocado: tags=RootContainerTest,test_dfuse_root_container """ - # Create a pool and start dfuse. - pool = self._create_pool() - container = self._create_cont(pool) - self.dfuse_hosts = self.agent_managers[0].hosts - # mount fuse - self.start_dfuse(self.dfuse_hosts, pool, container) - # Create another container and add it as sub container under - # root container - sub_container = str(self.dfuse.mount_dir.value + "/cont0") - container = self._create_cont(pool, path=sub_container) + cont_count = self.params.get("cont_count", '/run/container/*') + pool_count = self.params.get("pool_count", "/run/pool/*") + tmp_file_name = self.params.get("tmp_file_name", '/run/container/*') + tmp_file_count = self.params.get("tmp_file_count", '/run/container/*') + tmp_file_size = self.params.get("tmp_file_size", '/run/container/*') + device = "scm" + + # Create a pool and container. + self.log_step("Create a pool and a root container") + pool = self.get_pool(connect=False) + container = self.get_container(pool) + + # Start and mount fuse + dfuse_hosts = self.agent_managers[0].hosts + self.log_step("Start and mount dfuse using the root container") + dfuse = get_dfuse(self, dfuse_hosts) + start_dfuse(self, dfuse, pool, container) + + # Create another container and add it as sub container under the root container + self.log_step("Add another container as a sub container under the root container") + sub_container = str(dfuse.mount_dir.value + "/cont0") + self.get_container(pool, path=sub_container) + # Insert files into root container - self.insert_files_and_verify("") + self.log_step("Insert files into the root container") + self.insert_files_and_verify( + dfuse_hosts, dfuse.mount_dir.value, tmp_file_count, tmp_file_name, tmp_file_size) + # Insert files into sub container - self.insert_files_and_verify("cont0") + self.log_step("Insert files into the sub container") + self.insert_files_and_verify( + dfuse_hosts, os.path.join(dfuse.mount_dir.value, "cont0"), tmp_file_count, + tmp_file_name, tmp_file_size) + # Create 100 sub containers and verify the temp files - self.verify_create_delete_containers(pool, 100) - self.verify_multi_pool_containers() + self.verify_create_delete_containers( + pool, device, 100, dfuse_hosts, dfuse.mount_dir.value, tmp_file_count, tmp_file_size, + tmp_file_size) + self.verify_multi_pool_containers( + pool_count, cont_count, dfuse_hosts, dfuse.mount_dir.value, tmp_file_count, + tmp_file_name, tmp_file_size) - def verify_multi_pool_containers(self): + self.log.info("Test Passed") + + def verify_multi_pool_containers(self, pool_count, cont_count, hosts, mount_dir, tmp_file_count, + tmp_file_name, tmp_file_size): """Verify multiple pools and containers. Create several pools and containers and mount it under the root container and verify they're accessible. + + Args: + pool_count (int): number of pools to create + cont_count (int): number of containers to create in each pool + hosts (NodeSet): Hosts on which to run the commands + mount_dir (str): dfuse mount directory + tmp_file_count (int): number of temporary files + tmp_file_name (str): base name for temporary files + tmp_file_size (int): size of temporary files """ - pool_count = self.params.get("pool_count", "/run/pool/*") + self.log_step( + f"Create {pool_count} pools with {cont_count} containers mounted under the root " + "container and insert files into each new container") for idx in range(pool_count): - pool = self._create_pool() - for jdx in range(self.cont_count): - cont_name = "/cont_{}{}".format(idx, jdx) - sub_cont = str(self.dfuse.mount_dir.value + cont_name) - self._create_cont(pool=pool, path=sub_cont) - self.insert_files_and_verify(cont_name) - - def verify_create_delete_containers(self, pool, cont_count): + pool = self.get_pool(connect=False) + for jdx in range(cont_count): + sub_container = str(mount_dir + f"/cont_{idx}{jdx}") + self.get_container(pool=pool, path=sub_container) + self.insert_files_and_verify( + hosts, sub_container, tmp_file_count, tmp_file_name, tmp_file_size) + + def verify_create_delete_containers(self, pool, device, cont_count, hosts, mount_dir, + tmp_file_count, tmp_file_name, tmp_file_size): """Verify multiple pools and containers creation and deletion. Create multiple containers and multiple multi-mb files in each of @@ -117,76 +116,87 @@ def verify_create_delete_containers(self, pool, cont_count): Destroy half of the containers and verify the space usage is reclaimed. Args: + pool (TestPool): pool in which to create the containers + device (str): device where the pools and containers are created cont_count (int): Number of containers to be created. + hosts (NodeSet): Hosts on which to run the commands + tmp_file_count (int): number of temporary files + tmp_file_name (str): base name for temporary files + tmp_file_size (int): size of temporary files """ self.log.info("Verifying multiple container create delete") - pool_space_before = pool.get_pool_free_space(self.device) + self.log_step(f"Create {cont_count} new sub containers and insert files") + pool_space_before = pool.get_pool_free_space(device) self.log.info("Pool space before = %s", pool_space_before) + containers = [] for idx in range(cont_count): - sub_cont = str(self.dfuse.mount_dir.value + "/cont{}".format(idx + 1)) - self._create_cont(pool, path=sub_cont) - self.insert_files_and_verify("cont{}".format(idx + 1)) - expected = pool_space_before - \ - cont_count * self.tmp_file_count * self.tmp_file_size - pool_space_after = pool.get_pool_free_space(self.device) + sub_cont = str(mount_dir + f"/cont{idx + 1}") + containers.append(self.get_container(pool, path=sub_cont)) + self.insert_files_and_verify( + hosts, os.path.join(mount_dir, f"cont{idx + 1}"), tmp_file_count, tmp_file_name, + tmp_file_size) + + expected = pool_space_before - cont_count * tmp_file_count * tmp_file_size + self.log_step( + "Verify the pool free space <= {expected} after creating {cont_count} containers") + pool_space_after = pool.get_pool_free_space(device) self.log.info("Pool space <= Expected") self.log.info("%s <= %s", pool_space_after, expected) self.assertTrue(pool_space_after <= expected) - self.log.info("Destroying half of the containers = %s", cont_count // 2) + + self.log_step(f"Destroy half of the {cont_count} new sub containers ({cont_count // 2})") for _ in range(cont_count // 2): - self.container[-1].destroy(1) - self.container.pop() - expected = pool_space_after + \ - ((cont_count // 2) * self.tmp_file_count * self.tmp_file_size) - pool_space_after_cont_destroy = \ - pool.get_pool_free_space(self.device) + containers[-1].destroy(1) + containers.pop() + + expected = pool_space_after + ((cont_count // 2) * tmp_file_count * tmp_file_size) + self.log_step( + "Verify the pool free space >= {expected} after destroying half of the containers") + pool_space_after_cont_destroy = pool.get_pool_free_space(device) self.log.info("After container destroy") self.log.info("Free Pool space >= Expected") self.log.info("%s >= %s", pool_space_after_cont_destroy, expected) self.assertTrue(pool_space_after_cont_destroy >= expected) - def insert_files_and_verify(self, container_name): + def insert_files_and_verify(self, hosts, cont_dir, tmp_file_count, tmp_file_name, + tmp_file_size): """Verify inserting files into a specific container. Insert files into the specific container and verify they're navigable and accessible. Args: - container_name: Name of the POSIX Container - file_name_prefix: Prefix of the file name that will be created - no_of_files: Number of files to be created iteratively + hosts (NodeSet): Hosts on which to run the commands + cont_dir (str): container directory + tmp_file_count (int): number of temporary files + tmp_file_name (str): base name for temporary files + tmp_file_size (int): size of temporary files """ - cont_dir = self.dfuse.mount_dir.value - if container_name: - cont_dir = "{}/{}".format(cont_dir, container_name) - cmds = [] ls_cmds = [] - for idx in range(self.tmp_file_count): + for idx in range(tmp_file_count): # Create 40 MB files - file_name = "{}{}".format(self.tmp_file_name, idx + 1) - cmd = "head -c {} /dev/urandom > {}/{}".format( - self.tmp_file_size, cont_dir, file_name) - ls_cmds.append("ls {}".format(file_name)) + file_name = f"{tmp_file_name}{idx + 1}" + cmd = f"head -c {tmp_file_size} /dev/urandom > {cont_dir}/{file_name}" + ls_cmds.append(f"ls {file_name}") cmds.append(cmd) - self._execute_cmd(";".join(cmds)) + self._execute_cmd(";".join(cmds), hosts) cmds = [] # Run ls to verify the temp files are actually created - cmds = ["cd {}".format(cont_dir)] + cmds = [f"cd {cont_dir}"] cmds.extend(ls_cmds) - self._execute_cmd(";".join(cmds)) + self._execute_cmd(";".join(cmds), hosts) - def _execute_cmd(self, cmd): + def _execute_cmd(self, cmd, hosts): """Execute command on the host clients. Args: cmd (str): Command to run + hosts (NodeSet): hosts on which to run the command """ - result = run_remote(self.log, self.dfuse_hosts, cmd, timeout=30) + result = run_remote(self.log, hosts, cmd, timeout=30) if not result.passed: - self.log.error( - "Error running '%s' on the following hosts: %s", cmd, result.failed_hosts) - self.fail("Test was expected to pass but it failed.\n") + self.fail(f"Error running '{cmd}' on {str(result.failed_hosts)}") diff --git a/src/tests/ftest/dfuse/simul.py b/src/tests/ftest/dfuse/simul.py index aef80b5d651..5179ad0f5b4 100644 --- a/src/tests/ftest/dfuse/simul.py +++ b/src/tests/ftest/dfuse/simul.py @@ -1,19 +1,19 @@ """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -from avocado import fail_on -from dfuse_test_base import DfuseTestBase +from apricot import TestWithServers +from dfuse_utils import get_dfuse, start_dfuse from env_modules import load_mpi from exception_utils import MPILoadError -from general_utils import DaosTestError, run_command +from general_utils import run_command -class PosixSimul(DfuseTestBase): +class PosixSimul(TestWithServers): """Tests a posix container with simul. From : https://github.com/LLNL/simul @@ -69,7 +69,6 @@ class PosixSimul(DfuseTestBase): :avocado: recursive """ - @fail_on(DaosTestError) def run_simul(self, include=None, exclude=None, raise_exception=True): """Run simul. @@ -91,39 +90,35 @@ def run_simul(self, include=None, exclude=None, raise_exception=True): simul_path = self.params.get("simul_path", "/run/*", "") # Create a pool - self.log.info("Create a pool") - self.add_pool() + self.log_step("Create a pool") + pool = self.get_pool() # Create a container - self.log.info("Create container") - self.add_container(self.pool) + self.log_step("Create container") + container = self.get_container(pool) # Setup dfuse - dfuse_hosts = self.agent_managers[0].hosts - dfuse_mount_dir = self.params.get("mount_dir", '/run/dfuse/*') - self.start_dfuse(dfuse_hosts, self.pool, self.container) - self.dfuse.check_running() + self.log_step("Start dfuse") + dfuse = get_dfuse(self, self.agent_managers[0].hosts) + start_dfuse(self, dfuse, pool, container) + dfuse.check_running() # The use of MPI here is to run in parallel all simul tests on a single host. if not load_mpi(mpi_type): raise MPILoadError(mpi_type) # Run simul - sumil_cmd = os.path.join(simul_path, "simul") + simul_cmd = os.path.join(simul_path, "simul") if include and not exclude: - cmd = "{0} -vv -d {1} -i {2}".format(sumil_cmd, dfuse_mount_dir, include) + cmd = "{0} -vv -d {1} -i {2}".format(simul_cmd, dfuse.mount_dir.value, include) elif exclude and not include: - cmd = "{0} -vv -d {1} -e {2}".format(sumil_cmd, dfuse_mount_dir, exclude) + cmd = "{0} -vv -d {1} -e {2}".format(simul_cmd, dfuse.mount_dir.value, exclude) else: + cmd = None # appease pylint self.fail("##Both include and exclude tests are selected both or empty.") - self.log.info("Running simul on %s", mpi_type) - try: - result = run_command(cmd, output_check="combined", raise_exception=raise_exception) - finally: - self.stop_dfuse() - - return result + self.log_step("Running simul on %s", mpi_type) + return run_command(cmd, output_check="combined", raise_exception=raise_exception) def test_posix_simul(self): """Test simul. @@ -134,6 +129,7 @@ def test_posix_simul(self): :avocado: tags=PosixSimul,test_posix_simul """ self.run_simul(exclude="9,18,30,39,40") + self.log.info('Test passed') def test_posix_expected_failures(self): """Test simul, expected failures. @@ -154,4 +150,5 @@ def test_posix_expected_failures(self): self.log.info("Test %s was expected to fail, but passed", test) failed.append(test) if failed: - self.fail("Simul tests {} expected to failed, but passed".format(", ".join(failed))) + self.fail(f"Simul tests {', '.join(failed)} expected to failed, but passed") + self.log.info('Test passed') diff --git a/src/tests/ftest/dfuse/sparse_file.py b/src/tests/ftest/dfuse/sparse_file.py index 6b8521614b1..ef31c3816e6 100644 --- a/src/tests/ftest/dfuse/sparse_file.py +++ b/src/tests/ftest/dfuse/sparse_file.py @@ -8,6 +8,7 @@ from getpass import getuser import paramiko +from dfuse_utils import get_dfuse, start_dfuse from general_utils import get_remote_file_size from ior_test_base import IorTestBase @@ -50,14 +51,15 @@ def test_sparsefile(self): # Create a pool, container and start dfuse. self.create_pool() self.create_cont() - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, self.pool, self.container) # get scm space before write self.space_before = self.pool.get_pool_free_space("nvme") # create large file and perform write to it so that if goes out of # space. - sparse_file = os.path.join(self.dfuse.mount_dir.value, 'sparsefile.txt') + sparse_file = os.path.join(dfuse.mount_dir.value, 'sparsefile.txt') self.execute_cmd("touch {}".format(sparse_file)) self.log.info("File size (in bytes) before truncate: %s", get_remote_file_size(self.hostlist_clients[0], sparse_file)) diff --git a/src/tests/ftest/erasurecode/online_rebuild.yaml b/src/tests/ftest/erasurecode/online_rebuild.yaml index f5ea4768df7..0f2133b08ec 100644 --- a/src/tests/ftest/erasurecode/online_rebuild.yaml +++ b/src/tests/ftest/erasurecode/online_rebuild.yaml @@ -34,6 +34,11 @@ pool: container: type: POSIX control_method: daos +daos: + container: + destroy: + env_vars: + - CRT_TIMEOUT=10 ior: api: "DFS" client_processes: diff --git a/src/tests/ftest/erasurecode/rebuild_fio.py b/src/tests/ftest/erasurecode/rebuild_fio.py index ffd2d90aa1c..b68d058d927 100644 --- a/src/tests/ftest/erasurecode/rebuild_fio.py +++ b/src/tests/ftest/erasurecode/rebuild_fio.py @@ -1,28 +1,23 @@ ''' - (C) Copyright 2019-2023 Intel Corporation. + (C) Copyright 2019-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent ''' +import queue +import threading import time -from ec_utils import ErasureCodeFio +from dfuse_utils import get_dfuse, start_dfuse +from fio_test_base import FioBase -class EcodFioRebuild(ErasureCodeFio): - # pylint: disable=protected-access +class EcodFioRebuild(FioBase): """Test class Description: Runs Fio with EC object type over POSIX and verify on-line, off-line for rebuild and verify the data. :avocado: recursive """ - def __init__(self, *args, **kwargs): - """Initialize a EcodFioRebuild object.""" - super().__init__(*args, **kwargs) - self.set_online_rebuild = False - self.rank_to_kill = None - self.read_option = self.params.get("rw_read", "/run/fio/test/read_write/*") - def execution(self, rebuild_mode): """Execute test. @@ -31,29 +26,42 @@ def execution(self, rebuild_mode): """ aggregation_threshold = self.params.get("threshold", "/run/pool/aggregation/*") aggregation_timeout = self.params.get("aggr_timeout", "/run/pool/aggregation/*") + read_option = self.params.get("rw_read", "/run/fio/test/read_write/*") + + engine_count = self.server_managers[0].get_config_value("engines_per_host") + server_count = len(self.hostlist_servers) * engine_count + rank_to_kill = server_count - 1 + # 1. Disable aggregation self.log_step("Disable aggregation") - self.pool.disable_aggregation() + pool = self.get_pool() + pool.disable_aggregation() - # 2.a Kill last server rank first - self.log_step("Start fio and kill the last server") - self.rank_to_kill = self.server_count - 1 + # Start dfuse + self.log_step('Starting dfuse') + container = self.get_container(pool) + container.set_attr(attrs={'dfuse-direct-io-disable': 'on'}) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool, container) + + # Write the Fio data and kill the last server rank if rebuild_mode is on-line if 'on-line' in rebuild_mode: - # Enabled on-line rebuild for the test - self.set_online_rebuild = True - # 2.b Write the Fio data and kill server if rebuild_mode is on-line - self.start_online_fio() + self.log_step(f"Start fio and stop the last server rank ({rank_to_kill})") + self.start_online_fio(dfuse.mount_dir.value, rank_to_kill) + else: + self.log_step("Start fio and leave all servers running") + self.start_online_fio(dfuse.mount_dir.value, None) - # 3. Get initial total free space (scm+nvme) + # Get initial total free space (scm+nvme) self.log_step("Get initial total free space (scm+nvme)") - init_free_space = self.pool.get_total_free_space(refresh=True) + init_free_space = pool.get_total_free_space(refresh=True) - # 4. Enable aggregation + # Enable aggregation self.log_step("Enable aggregation") - self.pool.enable_aggregation() + pool.enable_aggregation() - # 5. Get total space consumed (scm+nvme) after aggregation enabled, verify and wait until - # aggregation triggered, maximum 3 minutes. + # Get total space consumed (scm+nvme) after aggregation enabled, verify and wait until + # aggregation triggered, maximum 3 minutes. self.log_step("Verify the Fio write finish without any error") start_time = time.time() timed_out = False @@ -61,7 +69,7 @@ def execution(self, rebuild_mode): self.log_step("Verify and wait until aggregation triggered") while not aggr_triggered and not timed_out: # Check if current free space exceeds threshold - free_space = self.pool.get_total_free_space(refresh=True) + free_space = pool.get_total_free_space(refresh=True) difference = free_space - init_free_space aggr_triggered = difference >= aggregation_threshold self.log.debug("Total Free space: initial=%s, current=%s, difference=%s", @@ -72,36 +80,86 @@ def execution(self, rebuild_mode): if not aggr_triggered and not timed_out: time.sleep(1) if timed_out: - self.fail("Aggregation not observed within {} seconds".format(aggregation_timeout)) + self.fail(f"Aggregation not observed within {aggregation_timeout} seconds") # ec off-line rebuild fio if 'off-line' in rebuild_mode: - self.log_step("Stop rank for ec off-line rebuild fio") - self.server_managers[0].stop_ranks( - [self.server_count - 1], self.d_log, force=True) + self.log_step(f"Stop the last server rank ({rank_to_kill}) for ec off-line rebuild fio") + self.server_managers[0].stop_ranks([rank_to_kill], self.d_log, force=True) - # 6. Adding unlink option for final read command + # Adding unlink option for final read command self.log_step("Adding unlink option for final read command") - if int(self.container.properties.value.split(":")[1]) == 1: - self.fio_cmd._jobs['test'].unlink.value = 1 + if int(container.properties.value.split(":")[1]) == 1: + self.fio_cmd._jobs['test'].unlink.value = 1 # pylint: disable=protected-access - # 7. Read and verify the original data. + # Read and verify the original data. self.log_step("Read and verify the original data.") - self.fio_cmd._jobs['test'].rw.value = self.read_option + self.fio_cmd._jobs['test'].rw.value = read_option # pylint: disable=protected-access self.fio_cmd.run() - # 8. If RF is 2 kill one more server and validate the data is not corrupted. - self.log_step("If RF is 2 kill one more server and validate the data is not corrupted.") - if int(self.container.properties.value.split(":")[1]) == 2: - self.fio_cmd._jobs['test'].unlink.value = 1 - self.log.info("RF is 2,So kill another server and verify data") + # If RF is 2 kill one more server and validate the data is not corrupted. + if int(container.properties.value.split(":")[1]) == 2: # Kill one more server rank - self.server_managers[0].stop_ranks([self.server_count - 2], self.d_log, force=True) + rank_to_kill = server_count - 2 + self.log_step(f"Kill one more server rank {rank_to_kill} when RF=2") + self.fio_cmd._jobs['test'].unlink.value = 1 # pylint: disable=protected-access + self.server_managers[0].stop_ranks([rank_to_kill], self.d_log, force=True) + # Read and verify the original data. + self.log_step(f"Verify the data is not corrupted after stopping rank {rank_to_kill}.") self.fio_cmd.run() # Pre-teardown: make sure rebuild is done before too-quickly trying to destroy container. - self.pool.wait_for_rebuild_to_end() + pool.wait_for_rebuild_to_end() + + self.log.info("Test passed") + + def start_online_fio(self, directory, rank_to_kill=None): + """Run Fio operation with thread in background. + + Trigger the server failure while Fio is running + + Args: + directory (str): directory to use with the fio command + rank_to_kill (int, optional): the server rank to kill while IO operation is in progress. + Set to None to leave all servers running during IO. Defaults to None. + """ + results_queue = queue.Queue() + + # Create the Fio run thread + job = threading.Thread( + target=self.write_single_fio_dataset, + kwargs={"directory": directory, "results": results_queue}) + + # Launch the Fio thread + job.start() + + # Kill the server rank while IO operation in progress + if rank_to_kill is not None: + time.sleep(30) + self.server_managers[0].stop_ranks([rank_to_kill], self.d_log, force=True) + + # Wait to finish the thread + job.join() + + # Verify the queue result and make sure test has no failure + while not results_queue.empty(): + if results_queue.get() == "FAIL": + self.fail("Error running fio as a thread") + + def write_single_fio_dataset(self, directory, results): + """Run Fio Benchmark. + + Args: + directory (str): directory to use with the fio command + results (queue): queue for returning thread results + """ + try: + self.fio_cmd.update_directory(directory) + self.execute_fio() + results.put("PASS") + except Exception: # pylint: disable=broad-except + results.put("FAIL") def test_ec_online_rebuild_fio(self): """Jira ID: DAOS-7320. diff --git a/src/tests/ftest/erasurecode/restart.py b/src/tests/ftest/erasurecode/restart.py index 959c2dcee2a..4b37e517156 100644 --- a/src/tests/ftest/erasurecode/restart.py +++ b/src/tests/ftest/erasurecode/restart.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -122,7 +122,6 @@ def execution(self, agg_check=None): "hosts": self.hostlist_clients, "path": self.workdir, "slots": None, - "group": self.server_group, "pool": self.pool, "container": None, "processes": processes, diff --git a/src/tests/ftest/erasurecode/truncate.py b/src/tests/ftest/erasurecode/truncate.py index dc8ff91f67a..bb570755bc8 100644 --- a/src/tests/ftest/erasurecode/truncate.py +++ b/src/tests/ftest/erasurecode/truncate.py @@ -1,10 +1,11 @@ ''' - (C) Copyright 2019-2023 Intel Corporation. + (C) Copyright 2019-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent ''' import os +from dfuse_utils import get_dfuse, start_dfuse from fio_test_base import FioBase from general_utils import get_remote_file_size, run_pcmd @@ -40,11 +41,16 @@ def test_ec_truncate(self): fname = self.params.get("names", '/run/fio/*') # Write the file using Fio - self.execute_fio(stop_dfuse=False) + pool = self.get_pool(connect=False) + container = self.get_container(pool) + container.set_attr(attrs={'dfuse-direct-io-disable': 'on'}) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool, container) + self.fio_cmd.update_directory(dfuse.mount_dir.value) + self.execute_fio() # Get the fuse file name. - testfile = "{}.0.0".format(os.path.join(self.dfuse.mount_dir.value, - fname[0])) + testfile = "{}.0.0".format(os.path.join(dfuse.mount_dir.value, fname[0])) original_fs = int(self.fio_cmd._jobs['test'].size.value) # Read and verify the original data. @@ -56,8 +62,8 @@ def test_ec_truncate(self): self.assertEqual(original_fs, file_size) # Truncate the original file which will extend the size of file. - result = run_pcmd(self.hostlist_clients, "truncate -s {} {}" - .format(truncate_size, testfile)) + result = run_pcmd( + self.hostlist_clients, "truncate -s {} {}".format(truncate_size, testfile)) if result[0]["exit_status"] == 1: self.fail("Failed to truncate file {}".format(testfile)) @@ -69,8 +75,8 @@ def test_ec_truncate(self): self.fio_cmd.run() # Truncate the original file and shrink to original size. - result = run_pcmd(self.hostlist_clients, "truncate -s {} {}" - .format(original_fs, testfile)) + result = run_pcmd( + self.hostlist_clients, "truncate -s {} {}".format(original_fs, testfile)) if result[0]["exit_status"] == 1: self.fail("Failed to truncate file {}".format(testfile)) diff --git a/src/tests/ftest/fault_injection/ec.py b/src/tests/ftest/fault_injection/ec.py index 873ddd1741c..f4bcbc37a38 100644 --- a/src/tests/ftest/fault_injection/ec.py +++ b/src/tests/ftest/fault_injection/ec.py @@ -3,6 +3,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' +from dfuse_utils import get_dfuse, start_dfuse from fio_test_base import FioBase from ior_test_base import IorTestBase @@ -51,4 +52,10 @@ def test_ec_fio_fault(self): :avocado: tags=ec,ec_array,ec_fio_fault,faults,fio :avocado: tags=EcodFaultInjection,test_ec_fio_fault """ + pool = self.get_pool(connect=False) + container = self.get_container(pool) + container.set_attr(attrs={'dfuse-direct-io-disable': 'on'}) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool, container) + self.fio_cmd.update_directory(dfuse.mount_dir.value) self.execute_fio() diff --git a/src/tests/ftest/io/macsio_test.py b/src/tests/ftest/io/macsio_test.py index 01daf179712..14b7595381f 100644 --- a/src/tests/ftest/io/macsio_test.py +++ b/src/tests/ftest/io/macsio_test.py @@ -3,12 +3,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from dfuse_test_base import DfuseTestBase +from dfuse_utils import get_dfuse, start_dfuse from general_utils import list_to_str from macsio_test_base import MacsioTestBase -class MacsioTest(DfuseTestBase, MacsioTestBase): +class MacsioTest(MacsioTestBase): """Test class Description: Runs a basic MACSio test. :avocado: recursive @@ -33,17 +33,18 @@ def test_macsio(self): processes = self.params.get("processes", "/run/macsio/*", len(self.hostlist_clients)) # Create a pool - self.add_pool() - self.pool.display_pool_daos_space() + self.log_step('Create a single pool') + pool = self.get_pool() + pool.display_pool_daos_space() # Create a container - self.add_container(self.pool) + self.log_step('Create a single container') + container = self.get_container(pool) # Run macsio - self.log.info("Running MACSio") + self.log_step("Running MACSio") status = self.macsio.check_results( - self.run_macsio( - self.pool.uuid, list_to_str(self.pool.svc_ranks), processes, self.container.uuid), + self.run_macsio(pool.uuid, list_to_str(pool.svc_ranks), processes, container.uuid), self.hostlist_clients) if status: self.log.info("Test passed") @@ -68,26 +69,29 @@ def test_macsio_daos_vol(self): processes = self.params.get("processes", "/run/macsio/*", len(self.hostlist_clients)) # Create a pool - self.add_pool() - self.pool.display_pool_daos_space() + self.log_step('Create a single pool') + pool = self.get_pool() + pool.display_pool_daos_space() # Create a container - self.add_container(self.pool) + self.log_step('Create a single container') + container = self.get_container(pool) # Create dfuse mount point - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + self.log_step('Starting dfuse') + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool, container) # VOL needs to run from a file system that supports xattr. Currently # nfs does not have this attribute so it was recommended to create and # use a dfuse dir and run vol tests from there. - self.job_manager.working_dir.value = self.dfuse.mount_dir.value + self.job_manager.working_dir.value = dfuse.mount_dir.value # Run macsio - self.log.info("Running MACSio with DAOS VOL connector") + self.log_step("Running MACSio with DAOS VOL connector") status = self.macsio.check_results( self.run_macsio( - self.pool.uuid, list_to_str(self.pool.svc_ranks), processes, self.container.uuid, - plugin_path), + pool.uuid, list_to_str(pool.svc_ranks), processes, container.uuid, plugin_path), self.hostlist_clients) if status: self.log.info("Test passed") diff --git a/src/tests/ftest/io/parallel_io.py b/src/tests/ftest/io/parallel_io.py index b181da47c6c..7be497dd28f 100644 --- a/src/tests/ftest/io/parallel_io.py +++ b/src/tests/ftest/io/parallel_io.py @@ -10,6 +10,7 @@ import time from getpass import getuser +from dfuse_utils import get_dfuse, start_dfuse from exception_utils import CommandFailure from fio_test_base import FioBase from ior_test_base import IorTestBase @@ -25,7 +26,6 @@ class ParallelIo(FioBase, IorTestBase): def __init__(self, *args, **kwargs): """Initialize a ParallelIo object.""" super().__init__(*args, **kwargs) - self.dfuse = None self.cont_count = None self.pool_count = None self.statvfs_info_initial = None @@ -76,7 +76,7 @@ def statvfs_pool(self, path): return statvfs_list - def verify_aggregation(self, reduced_space, count): + def verify_aggregation(self, dfuse, reduced_space, count): """Verify aggregation. Verify if expected space is returned for each pool after containers @@ -84,6 +84,7 @@ def verify_aggregation(self, reduced_space, count): otherwise exit the test with a failure. Args: + dfuse (Dfuse): the dfuse object reduced_space (int): expected space to be returned count (int): aggregation index """ @@ -101,8 +102,7 @@ def verify_aggregation(self, reduced_space, count): self.statvfs_after_cont_destroy) self.fail("Aggregation did not complete as expected") time.sleep(60) - self.statvfs_after_cont_destroy = self.statvfs_pool( - self.dfuse.mount_dir.value) + self.statvfs_after_cont_destroy = self.statvfs_pool(dfuse.mount_dir.value) counter += 1 def test_parallelio(self): @@ -133,14 +133,16 @@ def test_parallelio(self): # Create a pool and start dfuse. self.create_pool() - self.start_dfuse(self.hostlist_clients, self.pool[0], None) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, self.pool[0]) + # create multiple containers self.add_container_qty(self.cont_count, self.pool[0]) # check if all the created containers can be accessed and perform # io on each container using fio in parallel for _, cont in enumerate(self.container): - dfuse_cont_dir = self.dfuse.mount_dir.value + "/" + cont.uuid + dfuse_cont_dir = dfuse.mount_dir.value + "/" + cont.uuid cmd = "ls -a {}".format(dfuse_cont_dir) # execute bash cmds result = run_remote(self.log, self.hostlist_clients, cmd, timeout=30) @@ -148,8 +150,8 @@ def test_parallelio(self): self.fail("Error running '{}' on the following hosts: {}".format( cmd, result.failed_hosts)) # run fio on all containers - thread = threading.Thread(target=self.execute_fio, args=( - self.dfuse.mount_dir.value + "/" + cont.uuid, False)) + self.fio_cmd.update_directory(os.path.join(dfuse.mount_dir.value, cont.uuid)) + thread = threading.Thread(target=self.execute_fio) threads.append(thread) thread.start() @@ -162,19 +164,18 @@ def test_parallelio(self): self.container[0].destroy(1) # check dfuse if it is running fine - self.dfuse.check_running() + dfuse.check_running() # try accessing destroyed container, it should fail try: - self.execute_fio( - self.dfuse.mount_dir.value + "/" + container_to_destroy, False) + self.fio_cmd.update_directory(os.path.join(dfuse.mount_dir.value, container_to_destroy)) + self.execute_fio() self.fail( - "Fio was able to access destroyed container: {}".format( - self.container[0].uuid)) + "Fio was able to access destroyed container: {}".format(self.container[0].uuid)) except CommandFailure: self.log.info("fio failed as expected") # check dfuse is still running after attempting to access deleted container - self.dfuse.check_running() + dfuse.check_running() def test_multipool_parallelio(self): """Jira ID: DAOS-3775. @@ -220,10 +221,11 @@ def test_multipool_parallelio(self): pool_job.join() # start dfuse. - self.start_dfuse(self.hostlist_clients, None, None) + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse) # record free space using statvfs before any data is written. - self.statvfs_info_initial = self.statvfs_pool(self.dfuse.mount_dir.value) + self.statvfs_info_initial = self.statvfs_pool(dfuse.mount_dir.value) # Create 10 containers for each pool. Container create process cannot # be parallelized as different container create could complete at @@ -237,7 +239,7 @@ def test_multipool_parallelio(self): # using ior. This process of performing io is done in parallel for # all containers using threads. for pool_count, pool in enumerate(self.pool): - dfuse_pool_dir = str(self.dfuse.mount_dir.value + "/" + pool.uuid) + dfuse_pool_dir = str(dfuse.mount_dir.value + "/" + pool.uuid) for counter in range(self.cont_count): cont_num = (pool_count * self.cont_count) + counter dfuse_cont_dir = str(dfuse_pool_dir + "/" + self.container[cont_num].uuid) @@ -260,8 +262,7 @@ def test_multipool_parallelio(self): job.join() # Record free space after io - self.statvfs_before_cont_destroy = self.statvfs_pool( - self.dfuse.mount_dir.value) + self.statvfs_before_cont_destroy = self.statvfs_pool(dfuse.mount_dir.value) # Destroy half of the containers from each pool pfinal = 0 @@ -279,8 +280,7 @@ def test_multipool_parallelio(self): destroy_job.join() # Record free space after container destroy. - self.statvfs_after_cont_destroy = self.statvfs_pool( - self.dfuse.mount_dir.value) + self.statvfs_after_cont_destroy = self.statvfs_pool(dfuse.mount_dir.value) # Calculate the expected space to be returned after containers # are destroyed. diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index 2f168891d37..ad5b034301d 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -347,8 +347,8 @@ def _run(self, args): group.update_test_yaml( logger, args.scm_size, args.scm_mount, args.extra_yaml, args.timeout_multiplier, args.override, args.verbose, args.include_localhost) - except (RunException, YamlException): - message = "Error modifying the test yaml files" + except (RunException, YamlException) as e: + message = "Error modifying the test yaml files: {}".format(e) status |= self.get_exit_status(1, message, "Setup", sys.exc_info()) except StorageException: message = "Error detecting storage information for test yaml files" @@ -370,7 +370,7 @@ def _run(self, args): logger, self.result, self.repeat, self.slurm_setup, args.sparse, args.failfast, not args.disable_stop_daos, args.archive, args.rename, args.jenkinslog, core_files, args.logs_threshold, args.user_create, code_coverage, self.job_results_dir, - self.logdir) + self.logdir, args.clear_mounts) # Convert the test status to a launch.py status status |= summarize_run(logger, self.mode, test_status) @@ -438,6 +438,28 @@ def __arg_type_find_size(val): return val +def __arg_type_mount_point(val): + """Parse a mount point argument. + + The mount point does not need to exist on this host. + + Args: + val (str): the mount point to parse + + Raises: + ArgumentTypeError: if the value is not a string starting with '/' + + Returns: + str: the mount point + """ + try: + if not val.startswith(os.sep): + raise ValueError(f'Mount point does not start with {os.sep}') + except Exception as err: # pylint: disable=broad-except + raise ArgumentTypeError(f'Invalid mount point: {val}') from err + return val + + def main(): """Launch DAOS functional tests.""" # Parse the command line arguments @@ -507,6 +529,12 @@ def main(): "-a", "--archive", action="store_true", help="archive host log files in the avocado job-results directory") + parser.add_argument( + "-c", "--clear_mounts", + action="append", + default=[], + type=__arg_type_mount_point, + help="mount points to remove before running each test") parser.add_argument( "-dsd", "--disable_stop_daos", action="store_true", @@ -705,6 +733,9 @@ def main(): args.slurm_install = True args.slurm_setup = True args.user_create = True + args.clear_mounts.append("/mnt/daos") + args.clear_mounts.append("/mnt/daos0") + args.clear_mounts.append("/mnt/daos1") # Setup the Launch object launch = Launch(args.name, args.mode, args.slurm_install, args.slurm_setup) diff --git a/src/tests/ftest/mpiio/llnl_mpi4py.py b/src/tests/ftest/mpiio/llnl_mpi4py.py index 0ec142c6d7d..b9c30726d9b 100644 --- a/src/tests/ftest/mpiio/llnl_mpi4py.py +++ b/src/tests/ftest/mpiio/llnl_mpi4py.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import glob import os import site @@ -27,7 +28,7 @@ def get_test_repo(self, name): """ test_repo = self.params.get(name, '/run/test_repo/') # DAOS-15602: Always check the python3-6 install for test sources. - for packages in site.getsitepackages() + ["/usr/lib64/python3.6/site-packages"]: + for packages in site.getsitepackages() + glob.glob("/usr/lib64/python3.*/site-packages"): test_path = os.path.join(packages, test_repo) if os.path.exists(test_path): return test_path diff --git a/src/tests/ftest/pool/bad_query.py b/src/tests/ftest/pool/bad_query.py index 1426037102b..cc50cddc762 100644 --- a/src/tests/ftest/pool/bad_query.py +++ b/src/tests/ftest/pool/bad_query.py @@ -1,5 +1,5 @@ ''' - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent ''' @@ -52,8 +52,8 @@ def test_query(self): self.add_pool() # trash the pool handle value + handle_sav = self.pool.pool.handle if not handle == 'VALID': - handle_sav = self.pool.pool.handle self.pool.pool.handle = handle try: diff --git a/src/tests/ftest/pool/destroy_rebuild.py b/src/tests/ftest/pool/destroy_rebuild.py index 8f9d1d4adf7..753e8fb009e 100644 --- a/src/tests/ftest/pool/destroy_rebuild.py +++ b/src/tests/ftest/pool/destroy_rebuild.py @@ -59,7 +59,7 @@ def test_destroy_while_rebuilding(self): # Get the pool leader rank pool.set_query_data() - leader_rank = pool.query_data["response"]["leader"] + leader_rank = pool.query_data["response"]["svc_ldr"] if leader_rank in ap_ranks: ap_ranks.remove(leader_rank) elif leader_rank in non_ap_ranks: diff --git a/src/tests/ftest/pool/list_verbose.py b/src/tests/ftest/pool/list_verbose.py index d794f004789..eac2aadc190 100644 --- a/src/tests/ftest/pool/list_verbose.py +++ b/src/tests/ftest/pool/list_verbose.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -49,8 +49,10 @@ def create_expected(self, pool, scm_free, nvme_free, scm_imbalance, if scm_size is None: scm_size = pool.scm_size.value * rank_count + scm_size = int(scm_size) if nvme_size is None: nvme_size = pool.nvme_size.value * rank_count + nvme_size = int(nvme_size) targets_total = self.server_managers[0].get_config_value("targets") * rank_count @@ -59,17 +61,37 @@ def create_expected(self, pool, scm_free, nvme_free, scm_imbalance, upgrade_layout_ver = p_query["response"]["upgrade_layout_ver"] return { + "query_mask": "rebuild,space", + "state": state, "uuid": pool.uuid.lower(), "label": pool.label.value, - "svc_ldr": 0, + "total_targets": targets_total, + "active_targets": targets_total - targets_disabled, + "total_engines": rank_count, + "disabled_targets": targets_disabled, + "svc_ldr": pool.svc_leader, "svc_reps": pool.svc_ranks, - "state": state, - "targets_total": targets_total, - "targets_disabled": targets_disabled, "upgrade_layout_ver": upgrade_layout_ver, "pool_layout_ver": pool_layout_ver, - "query_error_msg": "", - "query_status_msg": "", + "rebuild": { + "status": 0, + "state": rebuild_state, + "objects": 0, + "records": 0 + }, + # NB: tests should not expect min/max/mean values + "tier_stats": [ + { + "total": scm_size, + "free": scm_free, + "media_type": "scm", + }, + { + "total": nvme_size, + "free": nvme_free, + "media_type": "nvme", + }, + ], "usage": [ { "tier_name": "SCM", @@ -78,12 +100,11 @@ def create_expected(self, pool, scm_free, nvme_free, scm_imbalance, "imbalance": scm_imbalance }, { - "tier_name": "NVMe", + "tier_name": "NVME", "size": nvme_size, "free": nvme_free, "imbalance": nvme_imbalance }], - "rebuild_state": rebuild_state } @staticmethod @@ -112,7 +133,7 @@ def get_scm_nvme_free_imbalance(pool_list_out): scm_size = usage["size"] scm_free = usage["free"] scm_imbalance = usage["imbalance"] - elif usage["tier_name"] == "NVMe": + elif usage["tier_name"] == "NVME": nvme_free = usage["free"] nvme_imbalance = usage["imbalance"] @@ -171,6 +192,12 @@ def verify_pool_lists(self, targets_disabled, scm_size, nvme_size, state, rebuil expected_pools = [] actual_pools = self.get_dmg_command().get_pool_list_all(verbose=True) + for pool in actual_pools: + del pool['version'] # not easy to calculate expected value, could cause flaky tests + for tier in pool["tier_stats"]: # expected values are tricky to calculate + del tier['min'] + del tier['max'] + del tier['mean'] # Get free and imbalance from actual so that we can use them in expected. free_data = self.get_scm_nvme_free_imbalance(actual_pools) @@ -334,7 +361,7 @@ def verify_used_imbalance(self, storage): # 1. Create a pool of 80GB. self.pool = [] - if storage == "NVMe": + if storage == "NVME": self.pool.append(self.get_pool(namespace="/run/pool_both/*")) nvme_size = [None] else: @@ -412,7 +439,7 @@ def test_used_imbalance(self): """ errors = [] self.log.debug("---------- NVMe test ----------") - errors.extend(self.verify_used_imbalance("NVMe")) + errors.extend(self.verify_used_imbalance("NVME")) self.log.debug("---------- SCM test ----------") errors.extend(self.verify_used_imbalance("SCM")) diff --git a/src/tests/ftest/pool/svc.py b/src/tests/ftest/pool/svc.py index 181e300cf40..6375206fab8 100644 --- a/src/tests/ftest/pool/svc.py +++ b/src/tests/ftest/pool/svc.py @@ -33,7 +33,7 @@ def check_leader(self, previous_leader=None, expect_change=True): self.pool.use_label = True try: - current_leader = int(self.pool.query_data["response"]["leader"]) + current_leader = int(self.pool.query_data["response"]["svc_ldr"]) except KeyError as error: self.log.error("self.pool.query_data: %s", self.pool.query_data) self.fail( diff --git a/src/tests/ftest/pool/verify_space.py b/src/tests/ftest/pool/verify_space.py index 9c1f3be704f..c4ccba5e046 100644 --- a/src/tests/ftest/pool/verify_space.py +++ b/src/tests/ftest/pool/verify_space.py @@ -112,12 +112,13 @@ def _write_data(self, description, ior_kwargs, container, block_size): except CommandFailure as error: self.fail(f'IOR write to {description} failed, {error}') - def _get_system_pool_size(self, description, scm_mounts): + def _get_system_pool_size(self, description, scm_mounts, server_hosts): """Get the pool size information from the df system command. Args: description (str): pool description scm_mounts (list): mount points used by the engine ranks + server_hosts (NodeSet): hosts running servers from which to collect the mount sizes Returns: dict: the df command information per server rank @@ -126,7 +127,7 @@ def _get_system_pool_size(self, description, scm_mounts): self.log_step(f'Collect system-level DAOS mount information for {description}') fields = ('source', 'size', 'used', 'avail', 'pcent', 'target') command = f"df -BG --output={','.join(fields)} | grep -E '{'|'.join(scm_mounts)}'" - result = run_remote(self.log, self.server_managers[0].hosts, command, stderr=True) + result = run_remote(self.log, server_hosts, command, stderr=True) if not result.passed: self.log.debug('########## DEBUG ##########') run_remote(self.log, result.failed_hosts, 'df -a') @@ -139,7 +140,7 @@ def _get_system_pool_size(self, description, scm_mounts): for rank in self.server_managers[0].get_host_ranks(data.hosts): system_pool_size[rank] = { field: info[index] for index, field in enumerate(fields)} - if len(system_pool_size) != len(self.server_managers[0].hosts): + if len(system_pool_size) != len(server_hosts): self.fail(f'Error obtaining system pool data for all hosts: {system_pool_size}') return system_pool_size @@ -179,17 +180,19 @@ def _compare_system_pool_size(self, pool_size, compare_methods): if not overall: self.fail(f"Error detected in system pools size for {pool_size[-1]['label']}") - def _check_pool_size(self, description, pool_size, scm_mounts, compare_methods): + def _check_pool_size(self, description, pool_size, scm_mounts, server_hosts, compare_methods): """Check the system pool size information reports as expected. Args: description (str): pool description pool_size (list): the list of pool size information scm_mounts (list): mount points used by the engine ranks + server_hosts (NodeSet): hosts running servers from which to collect the mount sizes compare_methods (list): a list of compare methods to execute per rank """ pool_size.append( - {'label': description, 'data': self._get_system_pool_size(description, scm_mounts)}) + {'label': description, + 'data': self._get_system_pool_size(description, scm_mounts, server_hosts)}) self._compare_system_pool_size(pool_size, compare_methods) def test_verify_pool_space(self): @@ -231,12 +234,14 @@ def test_verify_pool_space(self): } pools = [] pool_size = [] + server_hosts = self.server_managers[0].hosts.copy() # (1) Collect initial system information # - System available space should equal the free space description = 'initial configuration w/o pools' self._check_pool_size( - description, pool_size, scm_mounts, [compare_initial, compare_initial, compare_initial]) + description, pool_size, scm_mounts, server_hosts, + [compare_initial, compare_initial, compare_initial]) dmg.storage_query_usage() # (2) Create a single pool on a rank 0 @@ -244,7 +249,8 @@ def test_verify_pool_space(self): description = 'a single pool on rank 0' pools.extend(self._create_pools(description, [0])) self._check_pool_size( - description, pool_size, scm_mounts, [compare_reduced, compare_equal, compare_equal]) + description, pool_size, scm_mounts, server_hosts, + [compare_reduced, compare_equal, compare_equal]) self._query_pool_size(description, pools[0:1]) # (3) Write various amounts of data to the single pool on a single engine @@ -254,7 +260,8 @@ def test_verify_pool_space(self): self._write_data(description, ior_kwargs, container, block_size) data_label = f'{description} after writing data using a {block_size} block size' self._check_pool_size( - data_label, pool_size, scm_mounts, [compare_equal, compare_equal, compare_equal]) + data_label, pool_size, scm_mounts, server_hosts, + [compare_equal, compare_equal, compare_equal]) self._query_pool_size(data_label, pools[0:1]) dmg.storage_query_usage() @@ -263,7 +270,8 @@ def test_verify_pool_space(self): description = 'multiple pools on rank 1' pools.extend(self._create_pools(description, ['1_a', '1_b', '1_c'])) self._check_pool_size( - description, pool_size, scm_mounts, [compare_equal, compare_reduced, compare_equal]) + description, pool_size, scm_mounts, server_hosts, + [compare_equal, compare_reduced, compare_equal]) self._query_pool_size(description, pools[1:4]) # (5) Write various amounts of data to the multiple pools on rank 1 @@ -274,7 +282,8 @@ def test_verify_pool_space(self): self._write_data(description, ior_kwargs, container, block_size) data_label = f'{description} after writing data using a {block_size} block size' self._check_pool_size( - data_label, pool_size, scm_mounts, [compare_equal, compare_equal, compare_equal]) + data_label, pool_size, scm_mounts, server_hosts, + [compare_equal, compare_equal, compare_equal]) self._query_pool_size(data_label, pools[1 + index:2 + index]) dmg.storage_query_usage() @@ -283,7 +292,8 @@ def test_verify_pool_space(self): description = 'a single pool on ranks 1 & 2' pools.extend(self._create_pools(description, ['1_2'])) self._check_pool_size( - description, pool_size, scm_mounts, [compare_equal, compare_reduced, compare_reduced]) + description, pool_size, scm_mounts, server_hosts, + [compare_equal, compare_reduced, compare_reduced]) self._query_pool_size(description, pools[4:5]) # (7) Write various amounts of data to the single pool on ranks 1 & 2 @@ -294,7 +304,8 @@ def test_verify_pool_space(self): self._write_data(description, ior_kwargs, container, block_size) data_label = f'{description} after writing data using a {block_size} block size' self._check_pool_size( - data_label, pool_size, scm_mounts, [compare_equal, compare_equal, compare_equal]) + data_label, pool_size, scm_mounts, server_hosts, + [compare_equal, compare_equal, compare_equal]) self._query_pool_size(data_label, pools[4:5]) dmg.storage_query_usage() @@ -303,7 +314,8 @@ def test_verify_pool_space(self): description = 'a single pool on all ranks' pools.extend(self._create_pools(description, ['0_1_2'])) self._check_pool_size( - description, pool_size, scm_mounts, [compare_reduced, compare_reduced, compare_reduced]) + description, pool_size, scm_mounts, server_hosts, + [compare_reduced, compare_reduced, compare_reduced]) self._query_pool_size(description, pools[5:6]) # (9) Write various amounts of data to the single pool on all ranks @@ -313,19 +325,25 @@ def test_verify_pool_space(self): self._write_data(description, ior_kwargs, container, block_size) data_label = f'{description} after writing data using a {block_size} block size' self._check_pool_size( - data_label, pool_size, scm_mounts, [compare_equal, compare_equal, compare_equal]) + data_label, pool_size, scm_mounts, server_hosts, + [compare_equal, compare_equal, compare_equal]) self._query_pool_size(data_label, pools[5:6]) dmg.storage_query_usage() # (10) Stop one of the servers for a pool spanning many servers + # - With MD on SSD the control plane will unmount /mnt/daos0 when the rank is stopped description = 'all pools after stopping rank 1' self.log_step(f'Checking {description}', True) self.server_managers[0].stop_ranks([1], self.d_log) status = self.server_managers[0].verify_expected_states() if not status['expected']: self.fail("Rank 1 was not stopped") + if self.server_managers[0].manager.job.using_control_metadata: + # Don't checking the mount point size on the host where the rank was stopped + server_hosts.remove(self.server_managers[0].ranks[1]) self._check_pool_size( - description, pool_size, scm_mounts, [compare_equal, compare_equal, compare_equal]) + description, pool_size, scm_mounts, server_hosts, + [compare_equal, compare_equal, compare_equal]) for index, pool in enumerate(pools): self.log_step( ' '.join(['Query pool information for', str(pool), 'after stopping rank 1'])) diff --git a/src/tests/ftest/pool/verify_space.yaml b/src/tests/ftest/pool/verify_space.yaml index e5a790097b4..b88dcf2a3f3 100644 --- a/src/tests/ftest/pool/verify_space.yaml +++ b/src/tests/ftest/pool/verify_space.yaml @@ -1,7 +1,9 @@ hosts: test_servers: 3 test_clients: 1 + timeout: 600 + server_config: # Test will require code changes if more than one rank/engine per host engines_per_host: 1 @@ -9,27 +11,36 @@ server_config: 0: storage: auto system_ram_reserved: 1 + pool_rank_0: size: 100G target_list: [0] + pool_rank_1_a: size: 75G target_list: [1] + pool_rank_1_b: size: 85G target_list: [1] + pool_rank_1_c: size: 95G target_list: [1] + pool_rank_1_2: size: 200G target_list: [1, 2] + pool_rank_0_1_2: size: 175G target_list: [0, 1, 2] + container: type: POSIX control_method: daos + register_cleanup: False + ior: api: DFS dfs_chunk: 1048576 diff --git a/src/tests/ftest/rebuild/basic.py b/src/tests/ftest/rebuild/basic.py index 2d7b0e723c1..787e0f922e5 100644 --- a/src/tests/ftest/rebuild/basic.py +++ b/src/tests/ftest/rebuild/basic.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -23,11 +23,10 @@ def run_rebuild_test(self, pool_quantity): """ # Get the test parameters pools = [] - self.container = [] - daos_cmd = self.get_daos_command() + containers = [] for _ in range(pool_quantity): pools.append(self.get_pool(create=False)) - self.container.append(self.get_container(pools[-1], create=False)) + containers.append(self.get_container(pools[-1], create=False)) rank = self.params.get("rank", "/run/testparams/*") obj_class = self.params.get("object_class", "/run/testparams/*") @@ -55,12 +54,12 @@ def run_rebuild_test(self, pool_quantity): # Create containers in each pool and fill them with data rs_obj_nr = [] rs_rec_nr = [] - for container in self.container: + for container in containers: container.create() container.write_objects(rank, obj_class) # Determine how many objects will need to be rebuilt - for container in self.container: + for container in containers: target_rank_lists = container.get_target_rank_lists(" prior to rebuild") rebuild_qty = container.get_target_rank_count(rank, target_rank_lists) rs_obj_nr.append(rebuild_qty) @@ -101,12 +100,10 @@ def run_rebuild_test(self, pool_quantity): self.assertTrue(status, "Error confirming pool info after rebuild") # Verify the data after rebuild - for index, pool in enumerate(pools): - daos_cmd.container_set_prop( - pool=pool.uuid, cont=self.container[index].uuid, prop="status", value="healthy") - if self.container[index].object_qty.value != 0: - self.assertTrue( - self.container[index].read_objects(), "Data verification error after rebuild") + for container in containers: + container.set_prop(prop="status", value="healthy") + if container.object_qty.value != 0: + self.assertTrue(container.read_objects(), "Data verification error after rebuild") self.log.info("Test Passed") def test_simple_rebuild(self): @@ -120,7 +117,7 @@ def test_simple_rebuild(self): :avocado: tags=all,daily_regression :avocado: tags=vm - :avocado: tags=rebuild,pool,rebuild_tests,daos_cmd + :avocado: tags=rebuild,pool,daos_cmd :avocado: tags=RbldBasic,test_simple_rebuild """ self.run_rebuild_test(1) @@ -136,7 +133,7 @@ def test_multipool_rebuild(self): :avocado: tags=all,daily_regression :avocado: tags=vm - :avocado: tags=rebuild,pool,rebuild_tests + :avocado: tags=rebuild,pool :avocado: tags=RbldBasic,test_multipool_rebuild """ - self.run_rebuild_test(self.params.get("quantity", "/run/testparams/*")) + self.run_rebuild_test(self.params.get("pool_quantity", "/run/testparams/*")) diff --git a/src/tests/ftest/rebuild/basic.yaml b/src/tests/ftest/rebuild/basic.yaml index dad276257e8..31d203f3036 100644 --- a/src/tests/ftest/rebuild/basic.yaml +++ b/src/tests/ftest/rebuild/basic.yaml @@ -2,9 +2,10 @@ # required quantity is indicated by the placeholders hosts: test_servers: 6 -timeout: 900 +timeout: 600 server_config: name: daos_server + crt_timeout: 10 engines_per_host: 1 engines: 0: @@ -19,13 +20,14 @@ server_config: scm_mount: /mnt/daos system_ram_reserved: 1 pool: - scm_size: 1G - control_method: dmg + size: 1G pool_query_timeout: 30 properties: rd_fac:2 container: akey_size: 5 dkey_size: 5 + debug: true + properties: rd_fac:2 data: !mux small_data: data_size: 32 @@ -33,6 +35,9 @@ container: data_size: 250 objects: !mux zero_objects: + # Don't run duplicate variants when there are zero objects + !filter-only : "/run/container/data/small_data" # yamllint disable-line rule:colons + !filter-only : "/run/container/records/one_record" # yamllint disable-line rule:colons object_qty: 0 one_object: object_qty: 1 @@ -43,8 +48,6 @@ container: record_qty: 1 ten_records: record_qty: 10 - debug: true - properties: rd_fac:2 testparams: ranks: !mux rank1: @@ -53,5 +56,5 @@ testparams: rank: 2 rank3: rank: 3 - quantity: 2 + pool_quantity: 2 object_class: OC_RP_3G1 diff --git a/src/tests/ftest/recovery/container_list_consolidation.py b/src/tests/ftest/recovery/container_list_consolidation.py index 340fce41c68..45c9762f95c 100644 --- a/src/tests/ftest/recovery/container_list_consolidation.py +++ b/src/tests/ftest/recovery/container_list_consolidation.py @@ -27,14 +27,14 @@ def test_orphan_container(self): but doesn't appear with daos commands. 3. Check that the container doesn't appear with daos command. 4. Stop servers. - 5. Use ddb to verify that the container is left in shards. + 5. Use ddb to verify that the container is left in shards (PMEM only). 6. Enable the checker. 7. Set policy to --all-interactive. 8. Start the checker and query the checker until the fault is detected. 9. Repair by selecting the destroy option. 10. Query the checker until the fault is repaired. 11. Disable the checker. - 12. Run the ddb command and verify that the container is removed from shard. + 12. Run the ddb command and verify that the container is removed from shard (PMEM only). Jira ID: DAOS-12287 @@ -43,13 +43,11 @@ def test_orphan_container(self): :avocado: tags=recovery,cat_recov,container_list_consolidation :avocado: tags=ContainerListConsolidationTest,test_orphan_container """ - # 1. Create a pool and a container. self.log_step("Create a pool and a container") pool = self.get_pool(connect=False) container = self.get_container(pool=pool) expected_uuid = container.uuid.lower() - # 2. Inject fault to cause orphan container. self.log_step("Inject fault to cause orphan container.") daos_command = self.get_daos_command() daos_command.faults_container( @@ -57,49 +55,45 @@ def test_orphan_container(self): location="DAOS_CHK_CONT_ORPHAN") container.skip_cleanup() - # 3. Check that the container doesn't appear with daos command. self.log_step("Check that the container doesn't appear with daos command.") pool_list = daos_command.pool_list_containers(pool=pool.identifier) errors = [] if pool_list["response"]: errors.append(f"Container appears with daos command! {pool_list}") - # 4. Stop servers. self.log_step("Stop servers.") dmg_command = self.get_dmg_command() dmg_command.system_stop() - # 5. Use ddb to verify that the container is left in shards. - self.log_step("Use ddb to verify that the container is left in shards.") - scm_mount = self.server_managers[0].get_config_value("scm_mount") - ddb_command = DdbCommand( - server_host=NodeSet(self.hostlist_servers[0]), path=self.bin, - mount_point=scm_mount, pool_uuid=pool.uuid, - vos_file=self.get_vos_file_path(pool=pool)) - cmd_result = ddb_command.list_component() - ls_out = "\n".join(cmd_result[0]["stdout"]) - uuid_regex = r"([0-f]{8}-[0-f]{4}-[0-f]{4}-[0-f]{4}-[0-f]{12})" - match = re.search(uuid_regex, ls_out) - if match is None: - self.fail("Unexpected output from ddb command, unable to parse.") - self.log.info("Container UUID from ddb ls = %s", match.group(1)) - - # UUID if found. Verify that it's the container UUID of the container we created. - actual_uuid = match.group(1) - if actual_uuid != expected_uuid: - msg = "Unexpected container UUID! Expected = {}; Actual = {}".format( - expected_uuid, actual_uuid) - errors.append(msg) - - # 6. Enable the checker. + self.log_step("Use ddb to verify that the container is left in shards (PMEM only).") + vos_file = self.get_vos_file_path(pool=pool) + if vos_file: + # We're using a PMEM cluster. + scm_mount = self.server_managers[0].get_config_value("scm_mount") + ddb_command = DdbCommand( + server_host=NodeSet(self.hostlist_servers[0]), path=self.bin, + mount_point=scm_mount, pool_uuid=pool.uuid, vos_file=vos_file) + cmd_result = ddb_command.list_component() + ls_out = "\n".join(cmd_result[0]["stdout"]) + uuid_regex = r"([0-f]{8}-[0-f]{4}-[0-f]{4}-[0-f]{4}-[0-f]{12})" + match = re.search(uuid_regex, ls_out) + if match is None: + self.fail("Unexpected output from ddb command, unable to parse.") + self.log.info("Container UUID from ddb ls = %s", match.group(1)) + + # UUID is found. Verify that it's the container UUID of the container we created. + actual_uuid = match.group(1) + if actual_uuid != expected_uuid: + msg = "Unexpected container UUID! Expected = {}; Actual = {}".format( + expected_uuid, actual_uuid) + errors.append(msg) + self.log_step("Enable the checker.") dmg_command.check_enable(stop=False) - # 7. Set policy to --all-interactive. self.log_step("Set policy to --all-interactive.") dmg_command.check_set_policy(all_interactive=True) - # 8. Start the checker and query the checker until the fault is detected. self.log_step("Start and query the checker until the fault is detected.") seq_num = None # Start checker. @@ -116,36 +110,34 @@ def test_orphan_container(self): if not seq_num: self.fail("Checker didn't detect any fault!") - # 9. Repair by selecting the destroy option, 0. msg = ("Repair with option 0; Destroy the orphan container to release space " "[suggested].") self.log_step(msg) dmg_command.check_repair(seq_num=seq_num, action=0) - # 10. Query the checker until the fault is repaired. self.log_step("Query the checker until the fault is repaired.") repair_report = self.wait_for_check_complete()[0] - # Verify that the repair report has expected message "Discard the container". action_message = repair_report["act_msgs"][0] exp_msg = "Discard the container" errors = [] if exp_msg not in action_message: errors.append(f"{exp_msg} not in {action_message}!") - # 11. Disable the checker. self.log_step("Disable the checker.") dmg_command.check_disable(start=False) - # 12. Run the ddb command and verify that the container is removed from shard. - self.log_step( - "Run the ddb command and verify that the container is removed from shard.") - cmd_result = ddb_command.list_component() - ls_out = "\n".join(cmd_result[0]["stdout"]) - uuid_regex = r"([0-f]{8}-[0-f]{4}-[0-f]{4}-[0-f]{4}-[0-f]{12})" - match = re.search(uuid_regex, ls_out) - if match: - errors.append("Container UUID is found in shard! Checker didn't remove it.") + if vos_file: + # ddb requires the vos file. PMEM cluster only. + msg = ("Run the ddb command and verify that the container is removed from shard " + "(PMEM only).") + self.log_step(msg) + cmd_result = ddb_command.list_component() + ls_out = "\n".join(cmd_result[0]["stdout"]) + uuid_regex = r"([0-f]{8}-[0-f]{4}-[0-f]{4}-[0-f]{4}-[0-f]{12})" + match = re.search(uuid_regex, ls_out) + if match: + errors.append("Container UUID is found in shard! Checker didn't remove it.") # Start server to prepare for the cleanup. try: @@ -155,3 +147,7 @@ def test_orphan_container(self): self.log.error(error) finally: report_errors(test=self, errors=errors) + + # Remove container object so that tearDown will not try to destroy the non-existent + # container. + container.skip_cleanup() diff --git a/src/tests/ftest/recovery/ddb.py b/src/tests/ftest/recovery/ddb.py index 784d13b70a4..8447db1873a 100644 --- a/src/tests/ftest/recovery/ddb.py +++ b/src/tests/ftest/recovery/ddb.py @@ -148,10 +148,12 @@ def test_recovery_ddb_ls(self): # Find the vos file name. e.g., /mnt/daos0/<pool_uuid>/vos-0. scm_mount = self.server_managers[0].get_config_value("scm_mount") + vos_file = self.get_vos_file_path(pool=self.pool) + if vos_file is None: + self.fail("vos file wasn't found in {}/{}".format(scm_mount, self.pool.uuid.lower())) ddb_command = DdbCommand( server_host=NodeSet(self.hostlist_servers[0]), path=self.bin, - mount_point=scm_mount, pool_uuid=self.pool.uuid, - vos_file=self.get_vos_file_path(pool=self.pool)) + mount_point=scm_mount, pool_uuid=self.pool.uuid, vos_file=vos_file) errors = [] @@ -343,9 +345,11 @@ def test_recovery_ddb_rm(self): dmg_command.system_stop() # 3. Find the vos file name. + scm_mount = self.server_managers[0].get_config_value("scm_mount") vos_file = self.get_vos_file_path(pool=self.pool) + if vos_file is None: + self.fail("vos file wasn't found in {}/{}".format(scm_mount, self.pool.uuid.lower())) host = NodeSet(self.hostlist_servers[0]) - scm_mount = self.server_managers[0].get_config_value("scm_mount") ddb_command = DdbCommand( server_host=host, path=self.bin, mount_point=scm_mount, pool_uuid=self.pool.uuid, vos_file=vos_file) @@ -486,9 +490,11 @@ def test_recovery_ddb_load(self): dmg_command.system_stop() # 4. Find the vos file name. + scm_mount = self.server_managers[0].get_config_value("scm_mount") vos_file = self.get_vos_file_path(pool=self.pool) + if vos_file is None: + self.fail("vos file wasn't found in {}/{}".format(scm_mount, self.pool.uuid.lower())) host = NodeSet(self.hostlist_servers[0]) - scm_mount = self.server_managers[0].get_config_value("scm_mount") ddb_command = DdbCommand( server_host=host, path=self.bin, mount_point=scm_mount, pool_uuid=self.pool.uuid, vos_file=vos_file) @@ -575,9 +581,11 @@ def test_recovery_ddb_dump_value(self): dmg_command.system_stop() # 4. Find the vos file name. + scm_mount = self.server_managers[0].get_config_value("scm_mount") vos_file = self.get_vos_file_path(pool=self.pool) + if vos_file is None: + self.fail("vos file wasn't found in {}/{}".format(scm_mount, self.pool.uuid.lower())) host = NodeSet(self.hostlist_servers[0]) - scm_mount = self.server_managers[0].get_config_value("scm_mount") ddb_command = DdbCommand( server_host=host, path=self.bin, mount_point=scm_mount, pool_uuid=self.pool.uuid, vos_file=vos_file) diff --git a/src/tests/ftest/recovery/pool_membership.py b/src/tests/ftest/recovery/pool_membership.py index 47325940ffb..d9a29377d6e 100644 --- a/src/tests/ftest/recovery/pool_membership.py +++ b/src/tests/ftest/recovery/pool_membership.py @@ -214,7 +214,7 @@ def test_orphan_pool_shard(self): "Checker didn't fix orphan pool shard! msg = {}".format(query_msg)) # 8. Disable the checker. - self.log_step("Disable and start the checker.") + self.log_step("Disable checker.") dmg_command.check_disable() # 9. Call dmg storage query usage to verify that the pool usage is back to the @@ -288,7 +288,7 @@ def test_dangling_pool_map(self): "Checker didn't fix orphan pool shard! msg = {}".format(query_msg)) # 6. Disable the checker. - self.log_step("Disable and start the checker.") + self.log_step("Disable checker.") dmg_command.check_disable() # 7. Verify that the pool has one less target. diff --git a/src/tests/ftest/scripts/setup_nodes.sh b/src/tests/ftest/scripts/setup_nodes.sh index b8fd2851e60..a812e717071 100755 --- a/src/tests/ftest/scripts/setup_nodes.sh +++ b/src/tests/ftest/scripts/setup_nodes.sh @@ -43,30 +43,6 @@ cat /etc/security/limits.d/80_daos_limits.conf ulimit -a echo \"/var/tmp/core.%e.%t.%p\" > /proc/sys/kernel/core_pattern" sudo rm -f /var/tmp/core.* -if [ "${HOSTNAME%%.*}" != "$FIRST_NODE" ]; then - if grep /mnt/daos\ /proc/mounts; then - sudo umount /mnt/daos - else - if [ ! -d /mnt/daos ]; then - sudo mkdir -p /mnt/daos - fi - fi - - tmpfs_size=16777216 - memsize="$(sed -ne '/MemTotal:/s/.* \([0-9][0-9]*\) kB/\1/p' \ - /proc/meminfo)" - if [ "$memsize" -gt "32000000" ]; then - # make it twice as big on the hardware cluster - tmpfs_size=$((tmpfs_size*2)) - fi - sudo ed <<EOF /etc/fstab -\$a -tmpfs /mnt/daos tmpfs rw,relatime,size=${tmpfs_size}k 0 0 # added by ftest.sh -. -wq -EOF - sudo mount /mnt/daos -fi # make sure to set up for daos_agent. The test harness will take care of # creating the /var/run/daos_{agent,server} directories when needed. diff --git a/src/tests/ftest/security/cont_acl.py b/src/tests/ftest/security/cont_acl.py index 729784b9620..74c4ae34124 100644 --- a/src/tests/ftest/security/cont_acl.py +++ b/src/tests/ftest/security/cont_acl.py @@ -1,22 +1,27 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ import os import security_test_base as secTestBase +from agent_utils import include_local_host from cont_security_test_base import ContSecurityTestBase from pool_security_test_base import PoolSecurityTestBase -class DaosContainterSecurityTest(ContSecurityTestBase, PoolSecurityTestBase): +class DaosContainerSecurityTest(ContSecurityTestBase, PoolSecurityTestBase): # pylint: disable=too-few-public-methods,too-many-ancestors """Test daos_container user acls. :avocado: recursive """ + def _get_acl_file_name(self): + return os.path.join(self.tmp, self.params.get("acl_file_name", "/run/container_acl/*", + "cont_test_acl.txt")) + def test_container_user_acl(self): """ Description: @@ -56,7 +61,7 @@ def test_container_user_acl(self): :avocado: tags=all,full_regression :avocado: tags=vm :avocado: tags=security,container,container_acl,cont_user_sec,cont_group_sec,cont_sec - :avocado: tags=DaosContainterSecurityTest,test_container_user_acl + :avocado: tags=DaosContainerSecurityTest,test_container_user_acl """ # (1)Setup @@ -70,12 +75,10 @@ def test_container_user_acl(self): property_name, property_value = self.params.get( "property", "/run/container_acl/*") secTestBase.add_del_user( - self.hostlist_clients, "useradd", new_test_user) + include_local_host(self.hostlist_clients), "useradd", new_test_user) secTestBase.add_del_user( - self.hostlist_clients, "groupadd", new_test_group) - acl_file_name = os.path.join( - self.tmp, self.params.get( - "acl_file_name", "/run/container_acl/*", "cont_test_acl.txt")) + include_local_host(self.hostlist_clients), "groupadd", new_test_group) + acl_file_name = self._get_acl_file_name() test_user = self.params.get( "testuser", "/run/container_acl/daos_user/*") test_user_type = secTestBase.get_user_type(test_user) diff --git a/src/tests/ftest/security/cont_acl.yaml b/src/tests/ftest/security/cont_acl.yaml index 7c9d5c6bb46..799a0eddc6f 100644 --- a/src/tests/ftest/security/cont_acl.yaml +++ b/src/tests/ftest/security/cont_acl.yaml @@ -1,7 +1,7 @@ # change host names to your reserved nodes, the # required quantity is indicated by the placeholders hosts: - test_servers: 2 + test_servers: 1 test_clients: 1 timeout: 1200 server_config: diff --git a/src/tests/ftest/security/cont_owner.py b/src/tests/ftest/security/cont_owner.py new file mode 100644 index 00000000000..7935a6b9bc5 --- /dev/null +++ b/src/tests/ftest/security/cont_owner.py @@ -0,0 +1,127 @@ +""" + (C) Copyright 2024 Intel Corporation. + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import os + +import security_test_base as secTestBase +from cont_security_test_base import ContSecurityTestBase +from pool_security_test_base import PoolSecurityTestBase + + +class DaosContainerOwnerTest(ContSecurityTestBase, PoolSecurityTestBase): + # pylint: disable=too-few-public-methods,too-many-ancestors + """Test daos_container user acls. + + :avocado: recursive + """ + + def _create_cont_with_acl(self, cont_type): + # Set up an ACL that will allow us to reclaim the container + acl_file_name = os.path.join(self.tmp, "cont_test_owner_acl.txt") + acl_entries = [ + secTestBase.acl_entry("user", self.current_user, "rwdaAtTo"), + ] + secTestBase.create_acl_file(acl_file_name, acl_entries) + + # Set up the pool and container. + self.add_pool() + self.container = self.create_container_with_daos(self.pool, acl_file=acl_file_name, + cont_type=cont_type) + + def test_container_set_owner_no_check_non_posix(self): + """ + Description: + Verify that daos container set-owner --no-check flag ignores missing user for non-POSIX + container. + + :avocado: tags=all,full_regression + :avocado: tags=vm + :avocado: tags=security,container,cont_sec,cont_set_owner + :avocado: tags=DaosContainerOwnerTest + :avocado: tags=test_container_set_owner_no_check_non_posix + """ + fake_user = "fakeuser" + fake_grp = "fakegroup" + der_nonexist = '-1005' + + self._create_cont_with_acl(cont_type="python") + + # Attempt to change ownership to a fake user + with self.container.no_exception(): + result = self.container.set_owner(user=fake_user, group=None) + self.verify_daos_pool_cont_result(result, "set owner to fake user", "fail", der_nonexist) + + # Attempt to change ownership to fake group + with self.container.no_exception(): + result = self.container.set_owner(user=None, group=fake_grp) + self.verify_daos_pool_cont_result(result, "set owner to fake group", "fail", der_nonexist) + + # Using UID not allowed for non-POSIX + with self.container.no_exception(): + result = self.container.set_owner(user=fake_user, uid=123, no_check=True) + self.verify_daos_pool_cont_result(result, "set owner with uid", "fail", + 'for POSIX containers only') + + # Using GID not allowed for non-POSIX + with self.container.no_exception(): + result = self.container.set_owner(group=fake_grp, gid=123, no_check=True) + self.verify_daos_pool_cont_result(result, "set owner with gid", "fail", + 'for POSIX containers only') + + # Allow changing to fake user and group with no-check + with self.container.no_exception(): + result = self.container.set_owner(user=fake_user, group=fake_grp, no_check=True) + self.verify_daos_pool_cont_result(result, "set owner with no-check", "pass", None) + + def test_container_set_owner_no_check_posix(self): + """ + Description: + Verify that daos container set-owner --no-check flag works with uid/gid flags for + POSIX containers + + :avocado: tags=all,full_regression + :avocado: tags=vm + :avocado: tags=security,container,cont_sec,cont_set_owner + :avocado: tags=DaosContainerOwnerTest + :avocado: tags=test_container_set_owner_no_check_posix + """ + fake_user = "fakeuser" + fake_grp = "fakegroup" + der_nonexist = '-1005' + + self._create_cont_with_acl(cont_type="posix") + + # Attempt to change ownership to a fake user + with self.container.no_exception(): + result = self.container.set_owner(user=fake_user, group=None) + self.verify_daos_pool_cont_result(result, "set owner to fake user", "fail", der_nonexist) + + # Attempt to change ownership to fake group + with self.container.no_exception(): + result = self.container.set_owner(user=None, group=fake_grp) + self.verify_daos_pool_cont_result(result, "set owner to fake group", "fail", der_nonexist) + + # No-check alone is not allowed for POSIX containers + with self.container.no_exception(): + result = self.container.set_owner(user=fake_user, no_check=True) + self.verify_daos_pool_cont_result(result, "set owner to user with no-check", "fail", + 'requires --uid') + with self.container.no_exception(): + result = self.container.set_owner(group=fake_grp, no_check=True) + self.verify_daos_pool_cont_result(result, "set owner to group with no-check", "fail", + 'requires --gid') + + # No-check flag missing, but uid/gid supplied + with self.container.no_exception(): + result = self.container.set_owner(user=fake_user, uid=123, group=fake_grp, gid=456) + self.verify_daos_pool_cont_result(result, "set owner with uid/gid without no-check", "fail", + '--no-check is required') + + # Supply new uid and gid with --no-check + with self.container.no_exception(): + result = self.container.set_owner(user=fake_user, uid=123, group=fake_grp, gid=456, + no_check=True) + self.verify_daos_pool_cont_result(result, "set owner with uid/gid with no-check", + "pass", None) diff --git a/src/tests/ftest/security/cont_owner.yaml b/src/tests/ftest/security/cont_owner.yaml new file mode 100644 index 00000000000..4a086144711 --- /dev/null +++ b/src/tests/ftest/security/cont_owner.yaml @@ -0,0 +1,22 @@ +hosts: + test_servers: 1 + test_clients: 1 +timeout: 180 +server_config: + name: daos_server + port: 10001 + engines_per_host: 1 + engines: + 0: + targets: 4 + nr_xs_helpers: 0 + storage: + 0: + class: ram + scm_mount: /mnt/daos + scm_size: 4 + system_ram_reserved: 1 +pool: + size: 1G +container: + control_method: daos diff --git a/src/tests/ftest/server/replay.py b/src/tests/ftest/server/replay.py index 5b0124065f7..09df22ddf1c 100644 --- a/src/tests/ftest/server/replay.py +++ b/src/tests/ftest/server/replay.py @@ -7,7 +7,7 @@ import time from apricot import TestWithServers -from dfuse_utils import get_dfuse, start_dfuse, stop_dfuse +from dfuse_utils import get_dfuse, start_dfuse from general_utils import join from ior_utils import read_data, write_data from test_utils_pool import add_pool @@ -137,7 +137,7 @@ def test_replay_posix(self): ior = write_data(self, container, dfuse=dfuse) self.log_step('After the read has completed, unmount dfuse') - stop_dfuse(self, dfuse) + dfuse.stop() self.stop_engines() self.restart_engines() diff --git a/src/tests/ftest/soak/harassers.yaml b/src/tests/ftest/soak/harassers.yaml index cd59cbdfe51..c02a0ae90ce 100644 --- a/src/tests/ftest/soak/harassers.yaml +++ b/src/tests/ftest/soak/harassers.yaml @@ -136,6 +136,7 @@ ior_harasser: transfer_size: - '32k' - '1m' + - '4k' segment_count: 1 dfs_oclass: - ["EC_2P2GX", "RP_3GX"] diff --git a/src/tests/ftest/soak/smoke.yaml b/src/tests/ftest/soak/smoke.yaml index f6fe56dd3ec..ca1d4fb7a4c 100644 --- a/src/tests/ftest/soak/smoke.yaml +++ b/src/tests/ftest/soak/smoke.yaml @@ -116,6 +116,7 @@ ior_smoke: - '32k' - '128k' - '1m' + - '4k' segment_count: 1 dfs_oclass: - ["EC_2P1GX", "RP_2GX"] diff --git a/src/tests/ftest/soak/stress.yaml b/src/tests/ftest/soak/stress.yaml index e4b59b6b852..15a6a3033a3 100644 --- a/src/tests/ftest/soak/stress.yaml +++ b/src/tests/ftest/soak/stress.yaml @@ -126,6 +126,7 @@ ior_stress: transfer_size: - '1m' - '32k' + - '4k' dfs_oclass: - ["SX","SX"] - ["EC_2P1GX", "RP_2GX"] @@ -201,7 +202,7 @@ vpic_stress: oclass: - ["EC_2P1GX", "RP_2GX"] lammps_stress: - job_timeout: 45 + job_timeout: 55 nodesperjob: - 8 taskspernode: diff --git a/src/tests/ftest/tags.py b/src/tests/ftest/tags.py index d7f9347524e..4158f53141f 100755 --- a/src/tests/ftest/tags.py +++ b/src/tests/ftest/tags.py @@ -57,14 +57,14 @@ def __iter__(self): for item in self.__mapping.items(): yield deepcopy(item) - def methods(self): - """Get a mapping of methods to tags. + def __methods(self): + """Iterate over each method name and its tags. Yields: (str, set): method name and tags """ - for _, classes in self.__mapping.items(): - for _, methods in classes.items(): + for classes in self.__mapping.values(): + for methods in classes.values(): for method_name, tags in methods.items(): yield (method_name, tags) @@ -140,7 +140,7 @@ def is_test_subset(self, tags1, tags2): """ tests1 = set(self.__tags_to_tests(tags1)) tests2 = set(self.__tags_to_tests(tags2)) - return tests1.issubset(tests2) + return tests1 and tests2 and tests1.issubset(tests2) def __tags_to_tests(self, tags): """Convert a list of tags to the tests they would run. @@ -149,7 +149,7 @@ def __tags_to_tests(self, tags): tags (list): list of sets of tags """ tests = [] - for method_name, test_tags in self.methods(): + for method_name, test_tags in self.__methods(): for tag_set in tags: if tag_set.issubset(test_tags): tests.append(method_name) @@ -262,7 +262,7 @@ def sorted_tags(tags): return new_tags -def run_linter(paths=None): +def run_linter(paths=None, verbose=False): """Run the ftest tag linter. Args: @@ -307,17 +307,21 @@ def run_linter(paths=None): non_unique_classes = list(name for name, num in all_classes.items() if num > 1) non_unique_methods = list(name for name, num in all_methods.items() if num > 1) - print('ftest tag lint') + def _print_verbose(*args): + if verbose: + print(*args) + + _print_verbose('ftest tag lint') def _error_handler(_list, message, required=True): """Exception handler for each class of failure.""" _list_len = len(_list) req_str = '(required)' if required else '(optional)' - print(f' {req_str} {_list_len} {message}') + _print_verbose(f' {req_str} {_list_len} {message}') if _list_len == 0: return None for _test in _list: - print(f' {_test}') + _print_verbose(f' {_test}') if _list_len > 3: remaining = _list_len - 3 _list = _list[:3] + [f"... (+{remaining})"] @@ -525,6 +529,37 @@ def print_verbose(*args): [set(['foo1']), set(['foo2'])]) == ['test_1', 'test_2'] assert tag_map._FtestTagMap__tags_to_tests([set(['foo1', 'class_1'])]) == ['test_1'] + print_step('__methods') + assert list(tag_map._FtestTagMap__methods()) == [ + ('test_1', {'class_1', 'test_1', 'foo1'}), ('test_2', {'class_2', 'test_2', 'foo2'})] + + print_step('unique_tags') + assert tag_map.unique_tags() == set(['class_1', 'test_1', 'foo1', 'class_2', 'test_2', 'foo2']) + assert tag_map.unique_tags(exclude=['/foo1']) == set(['class_2', 'test_2', 'foo2']) + assert tag_map.unique_tags(exclude=['/foo2']) == set(['class_1', 'test_1', 'foo1']) + assert tag_map.unique_tags(exclude=['/foo1', '/foo2']) == set() + + print_step('minimal_tags') + assert tag_map.minimal_tags('/foo1') == set(['class_1']) + assert tag_map.minimal_tags('/foo2') == set(['class_2']) + + print_step('is_test_subset') + assert tag_map.is_test_subset([set(['test_1'])], [set(['test_1'])]) + assert tag_map.is_test_subset([set(['test_1'])], [set(['class_1'])]) + assert tag_map.is_test_subset([set(['test_1', 'foo1'])], [set(['class_1'])]) + assert not tag_map.is_test_subset([set(['test_1'])], [set(['test_2'])]) + assert not tag_map.is_test_subset([set(['test_1'])], [set(['class_2'])]) + assert not tag_map.is_test_subset([set(['test_1', 'foo1'])], [set(['class_2'])]) + assert not tag_map.is_test_subset([set(['test_1']), set(['test_2'])], [set(['class_1'])]) + assert tag_map.is_test_subset([set(['class_2'])], [set(['test_1']), set(['test_2'])]) + assert not tag_map.is_test_subset([set(['fake'])], [set(['class_2'])]) + + print_step('__init__') + # Just a smoke test to verify the map can parse real files + tag_map = FtestTagMap(all_python_files(FTEST_DIR)) + expected_tags = set(['test_harness_config', 'test_ior_small', 'test_dfuse_mu_perms']) + assert len(tag_map.unique_tags().intersection(expected_tags)) == len(expected_tags) + print('Ftest Tags Utility Unit Tests PASSED') @@ -556,7 +591,7 @@ def main(): if args.command == "lint": try: - run_linter(args.paths) + run_linter(args.paths, args.verbose) except LintFailure as err: print(err) sys.exit(1) diff --git a/src/tests/ftest/telemetry/pool_svc_metrics.py b/src/tests/ftest/telemetry/pool_svc_metrics.py new file mode 100644 index 00000000000..8a34964bfd8 --- /dev/null +++ b/src/tests/ftest/telemetry/pool_svc_metrics.py @@ -0,0 +1,130 @@ +''' + (C) Copyright 2024 Intel Corporation. + + SPDX-License-Identifier: BSD-2-Clause-Patent +''' +import time + +from telemetry_test_base import TestWithTelemetry + +MAP_VERSION_METRIC = "engine_pool_svc_map_version" +SVC_LEADER_METRIC = "engine_pool_svc_leader" +DEGRADED_RANKS_METRIC = "engine_pool_svc_degraded_ranks" + + +class PoolServiceMetrics(TestWithTelemetry): + """Verify pool service metric values. + + :avocado: recursive + """ + + def collect_svc_telemetry(self, pool_uuid): + """Collect the pool service metric values. + + Args: + pool_uuid (str): The UUID of the pool to collect metrics from. + + Returns: + dict: A dict of the current pool service leader's metrics. + """ + def _map_version(rank_metrics): + if rank_metrics is None or MAP_VERSION_METRIC not in rank_metrics: + return 0 + return rank_metrics[MAP_VERSION_METRIC] + + def _pool_rank_metrics(host_metrics, pool_uuid): + self.log.debug("collecting rank metrics for pool: %s", pool_uuid) + pm = {} + for hm in host_metrics.values(): + for k, v in hm.items(): + try: + for m in v['metrics']: + if m['labels']['pool'].casefold() != str(pool_uuid).casefold(): + continue + rank = m['labels']['rank'] + if rank not in pm: + pm[rank] = {} + pm[rank][k] = m['value'] + except KeyError: + continue + return pm + + metrics_list = ",".join(self.telemetry.ENGINE_POOL_SVC_METRICS) + host_metrics = self.telemetry.get_metrics(metrics_list) + self.log.debug("host metrics: %s", host_metrics) + pr_metrics = _pool_rank_metrics(host_metrics, pool_uuid) + self.log.debug("pool rank metrics: %s", pr_metrics) + leader_metrics = {} + for metrics in pr_metrics.values(): + if _map_version(metrics) > _map_version(leader_metrics): + leader_metrics = metrics + + self.log.debug("Pool service leader metrics: %s", leader_metrics) + return leader_metrics + + def test_pool_service_metrics(self): + """Test that pool service telemetry is updated as expected. + + :avocado: tags=all,daily_regression + :avocado: tags=vm + :avocado: tags=telemetry + :avocado: tags=PoolServiceMetrics,test_pool_service_metrics + """ + self.log_step("Create pool for testing.") + pool = self.get_pool(connect=True) + + self.log_step("Collect pool service metrics prior to making changes.") + initial_metrics = self.collect_svc_telemetry(pool.uuid) + self.assertTrue(MAP_VERSION_METRIC in initial_metrics, + f"initial metrics don't contain {MAP_VERSION_METRIC} (no leader?)") + self.assertTrue(initial_metrics[MAP_VERSION_METRIC] == 1, + "initial pool service map version is not 1") + self.assertTrue(initial_metrics[DEGRADED_RANKS_METRIC] == 0, + "initial pool service degraded rank count is not 0") + + restart_rank = initial_metrics[SVC_LEADER_METRIC] + self.log_step(f"Stop pool service leader rank: {restart_rank}") + self.server_managers[0].stop_ranks(ranks=[restart_rank], daos_log=self.d_log) + + self.log_step("Verify the pool service leader rank has stopped successfully.") + failed_ranks = self.server_managers[0].check_rank_state( + ranks=[restart_rank], valid_states=["stopped", "excluded"], max_checks=15) + if failed_ranks: + self.fail(f"Rank {restart_rank} didn't stop!") + + def _wait_for_telemetry(test): + metrics = self.collect_svc_telemetry(pool.uuid) + while True: + try: + if test(metrics): + return metrics + except KeyError: + pass + + self.log.info("waiting for pool telemetry to update") + time.sleep(5) + metrics = self.collect_svc_telemetry(pool.uuid) + + self.log_step("Wait for rank exclusion to show up in the telemetry.") + metrics = _wait_for_telemetry(lambda m: m[MAP_VERSION_METRIC] > 1) + + self.log_step("Verify that the pool service telemetry has updated.") + self.assertTrue(metrics[DEGRADED_RANKS_METRIC] == 1, + "pool service degraded rank count should be 1") + + self.log_step("Restart the stopped rank.") + self.server_managers[0].start_ranks(ranks=[restart_rank], daos_log=self.d_log) + + self.log_step("Verify the desired rank restarted successfully.") + failed_ranks = self.server_managers[0].check_rank_state( + ranks=[restart_rank], valid_states=["joined"], max_checks=15) + if failed_ranks: + self.fail(f"Rank {restart_rank} didn't start!") + + self.log_step("Reintegrate failed rank back into the pool.") + pool.reintegrate(restart_rank) + + self.log_step("Wait for reintegration to show up in the telemetry.") + metrics = _wait_for_telemetry(lambda m: m[DEGRADED_RANKS_METRIC] == 0) + + self.log_step("Test passed.") diff --git a/src/tests/ftest/telemetry/pool_svc_metrics.yaml b/src/tests/ftest/telemetry/pool_svc_metrics.yaml new file mode 100644 index 00000000000..c7275aae1d0 --- /dev/null +++ b/src/tests/ftest/telemetry/pool_svc_metrics.yaml @@ -0,0 +1,17 @@ +hosts: + test_servers: 3 +timeout: 300 +server_config: + name: daos_server + engines_per_host: 1 + engines: + 0: + targets: 4 + nr_xs_helpers: 0 + storage: + 0: + class: ram + scm_mount: /mnt/daos + system_ram_reserved: 1 +pool: + scm_size: 1G diff --git a/src/tests/ftest/telemetry/wal_metrics.py b/src/tests/ftest/telemetry/wal_metrics.py index 9e7224a302e..6c1d0c6a247 100644 --- a/src/tests/ftest/telemetry/wal_metrics.py +++ b/src/tests/ftest/telemetry/wal_metrics.py @@ -19,9 +19,9 @@ class WalMetrics(TestWithTelemetry): def test_wal_commit_metrics(self): """JIRA ID: DAOS-11626. - The WAL commit metrics is per-pool metrics, it includes 'wal_sz', 'wal_qd' and 'wal_waiters' - (see vos_metrics_alloc() in src/vos/vos_common.c). WAL commit metrics are updated on each - local transaction (for example, transaction for a update request, etc.) + The WAL commit metrics is per-pool metrics, it includes 'wal_sz', 'wal_qd', 'wal_waiters' + and 'wal_dur' (see vos_metrics_alloc() in src/vos/vos_common.c). WAL commit metrics are + updated on each local transaction (for example, transaction for a update request, etc.) Test steps: 1) Create a pool @@ -41,10 +41,12 @@ def test_wal_commit_metrics(self): 'Collect WAL commit metrics after creating a pool (dmg telemetry metrics query)') ranges = self.telemetry.collect_data(wal_metrics) for metric in list(ranges): - if '_sz' in metric and not metric.endswith('_mean') and not metric.endswith('_stddev'): + if (('_sz' in metric or '_dur' in metric) + and not metric.endswith('_mean') and not metric.endswith('_stddev')): for label in ranges[metric]: if self.server_managers[0].manager.job.using_control_metadata: - # The min/max/actual size should be greater than 0 for MD on SSD + # The min/max/actual values of the size and duration metrics should be + # greater than 0 for MD on SSD ranges[metric][label] = [1] else: ranges[metric][label] = [0, 0] @@ -136,7 +138,7 @@ def test_wal_checkpoint_metrics(self): :avocado: tags=telemetry :avocado: tags=WalMetrics,test_wal_checkpoint_metrics """ - frequency = 5 + frequency = 10 wal_metrics = list(self.telemetry.ENGINE_POOL_CHECKPOINT_METRICS) self.log_step('Creating a pool with check pointing disabled (dmg pool create)') @@ -146,10 +148,10 @@ def test_wal_checkpoint_metrics(self): 'Collect WAL checkpoint metrics after creating a pool w/o check pointing ' '(dmg telemetry metrics query)') ranges = self.telemetry.collect_data(wal_metrics) - for metric, values in ranges.items(): - for label in values: + for metric in list(ranges): + for label in ranges[metric]: # Initially all metrics should be 0 - values[label] = [0, 0] + ranges[metric][label] = [0, 0] self.log_step( 'Verifying WAL checkpoint metrics are all 0 after creating a pool w/o check pointing') @@ -163,28 +165,31 @@ def test_wal_checkpoint_metrics(self): 'Collect WAL checkpoint metrics after creating a pool w/ check pointing ' '(dmg telemetry metrics query)') ranges = self.telemetry.collect_data(wal_metrics) - for metric, values in ranges.items(): - for label in values: + for metric in list(ranges): + for label in ranges[metric]: uuid = pool.uuid if uuid in label and self.server_managers[0].manager.job.using_control_metadata: - if '_dirty_chunks' in metric: + if '_sumsquares' in metric: + # Check point sum squares should be > 0 after pool create for MD on SSD + ranges[metric][label] = [1] + elif '_dirty_chunks' in metric: # Check point dirty chunks should be 0-300 after pool create for MD on SSD - values[label] = [0, 300] + ranges[metric][label] = [0, 300] elif '_dirty_pages' in metric: # Check point dirty pages should be 0-3 after pool create for MD on SSD - values[label] = [0, 3] + ranges[metric][label] = [0, 3] elif '_duration' in metric: # Check point duration should be 0-1,000,000 after pool create for MD on SSD - values[label] = [0, 1000000] + ranges[metric][label] = [0, 1000000] elif '_iovs_copied' in metric: # Check point iovs copied should be >= 0 after pool create for MD on SSD - values[label] = [0] + ranges[metric][label] = [0] elif '_wal_purged' in metric: # Check point wal purged should be >= 0 after pool create for MD on SSD - values[label] = [0] + ranges[metric][label] = [0] else: # All metrics for the pool w/o check pointing or w/o MD on SSD should be 0 - values[label] = [0, 0] + ranges[metric][label] = [0, 0] self.log_step('Verifying WAL check point metrics after creating a pool w/ check pointing') if not self.telemetry.verify_data(ranges): self.fail('WAL replay metrics verification failed after pool w/ check pointing create') @@ -202,12 +207,12 @@ def test_wal_checkpoint_metrics(self): self.log_step('Collect WAL checkpoint metrics after check pointing is complete') self.telemetry.collect_data(wal_metrics) if self.server_managers[0].manager.job.using_control_metadata: - for metric, values in ranges.items(): - for label in values: + for metric in list(ranges): + for label in ranges[metric]: if pool.uuid in label: if '_wal_purged' in metric: # Check point wal purged should be > 0 after check point for MD on SSD - values[label] = [1] + ranges[metric][label] = [1] self.log_step( 'Verify WAL checkpoint metrics after check pointing is complete ' '(dmg telemetry metrics query)') diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index aa9949499c3..ec99e3c1889 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -1821,7 +1821,7 @@ def add_container(self, pool, namespace=CONT_NAMESPACE, create=True, daos=None, """ self.container = self.get_container(pool, namespace, create, daos, **params) - def add_container_qty(self, quantity, pool, namespace=None, create=True): + def add_container_qty(self, quantity, pool, namespace=CONT_NAMESPACE, create=True): """Add multiple containers to the test case. This method requires self.container to be defined as a list. diff --git a/src/tests/ftest/util/configuration_utils.py b/src/tests/ftest/util/configuration_utils.py index 689ac8456c4..df333808be0 100644 --- a/src/tests/ftest/util/configuration_utils.py +++ b/src/tests/ftest/util/configuration_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2019-2023 Intel Corporation. + (C) Copyright 2019-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -452,10 +452,12 @@ def get(self, key, path=None, default=None): # Multiple configuration-specific matching paths for the key - # no other way to determine which value to use multiple_matches = True + value = None else: # Multiple matching paths w/o an active configuration for the key - # no way to determine which value to use multiple_matches = True + value = None # Report an AvocadoParam-style exception for multiple key matches if multiple_matches: diff --git a/src/tests/ftest/util/cont_security_test_base.py b/src/tests/ftest/util/cont_security_test_base.py index ce9c3bf93a5..28e714052a5 100644 --- a/src/tests/ftest/util/cont_security_test_base.py +++ b/src/tests/ftest/util/cont_security_test_base.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -40,13 +40,14 @@ def setUp(self): self.co_prop = self.params.get("container_properties", "/run/container/*") self.dmg = self.get_dmg_command() - def create_container_with_daos(self, pool, acl_type=None, acl_file=None): + def create_container_with_daos(self, pool, acl_type=None, acl_file=None, cont_type=None): """Create a container with the daos tool. Args: pool (TestPool): Pool object. acl_type (str, optional): valid or invalid. acl_file (str, optional): acl file + cont_type (str, optional): container type Returns: TestContainer: the new container @@ -61,7 +62,7 @@ def create_container_with_daos(self, pool, acl_type=None, acl_file=None): acl_file = os.path.join(self.tmp, "acl_{}.txt".format(acl_type)) try: - return self.get_container(pool, acl_file=acl_file) + return self.get_container(pool, type=cont_type, acl_file=acl_file) except TestFail as error: if acl_type != "invalid": raise DaosTestError("Could not create expected container ") from error diff --git a/src/tests/ftest/util/container_rf_test_base.py b/src/tests/ftest/util/container_rf_test_base.py index 99e49c1532c..313534a45fb 100644 --- a/src/tests/ftest/util/container_rf_test_base.py +++ b/src/tests/ftest/util/container_rf_test_base.py @@ -180,6 +180,9 @@ def execute_cont_rf_test(self, create_container=True, mode=None): self.execute_pool_verify(" after rebuild") self.log.info("==>(7)Check for container data if the container is healthy.") self.verify_container_data() + else: + self.container.close() + self.container.skip_cleanup() self.log.info("Test passed") elif mode == "cont_rf_enforcement": self.log.info("Container rd_fac test passed") diff --git a/src/tests/ftest/util/daos_utils.py b/src/tests/ftest/util/daos_utils.py index 96166ce8294..e309cd091c4 100644 --- a/src/tests/ftest/util/daos_utils.py +++ b/src/tests/ftest/util/daos_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -592,7 +592,8 @@ def container_get_prop(self, pool, cont, properties=None): return self._get_json_result( ("container", "get-prop"), pool=pool, cont=cont, prop=props) - def container_set_owner(self, pool, cont, user, group): + def container_set_owner(self, pool, cont, user=None, group=None, uid=None, gid=None, + no_check=False): """Call daos container set-owner. Args: @@ -600,6 +601,9 @@ def container_set_owner(self, pool, cont, user, group): cont (str): container UUID or label user (str): New-user who will own the container. group (str): New-group who will own the container. + uid (int): with no_check=True, UID to use for user on POSIX container + gid (int): with no_check=True, GID to use for group on POSIX container + no_check (bool): Skip checking if user and group exist locally Returns: CmdResult: Object that contains exit status, stdout, and other @@ -611,7 +615,7 @@ def container_set_owner(self, pool, cont, user, group): """ return self._get_result( ("container", "set-owner"), - pool=pool, cont=cont, user=user, group=group) + pool=pool, cont=cont, user=user, group=group, uid=uid, gid=gid, no_check=no_check) def container_set_attr(self, pool, cont, attrs, sys_name=None): """Call daos container set-attr. diff --git a/src/tests/ftest/util/daos_utils_base.py b/src/tests/ftest/util/daos_utils_base.py index 855d17b9bab..bc1eee84ddc 100644 --- a/src/tests/ftest/util/daos_utils_base.py +++ b/src/tests/ftest/util/daos_utils_base.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -438,6 +438,9 @@ def __init__(self): super().__init__("set-owner") self.user = FormattedParameter("--user={}") self.group = FormattedParameter("--group={}") + self.uid = FormattedParameter("--uid={}") + self.gid = FormattedParameter("--gid={}") + self.no_check = FormattedParameter("--no-check", False) class SetPropSubCommand(CommonContainerSubCommand): """Defines an object for the daos container set-prop command.""" diff --git a/src/tests/ftest/util/data_mover_test_base.py b/src/tests/ftest/util/data_mover_test_base.py index bfed9571c32..2fbe3e6e193 100644 --- a/src/tests/ftest/util/data_mover_test_base.py +++ b/src/tests/ftest/util/data_mover_test_base.py @@ -12,6 +12,7 @@ from command_utils_base import BasicParameter, EnvironmentVariables from data_mover_utils import (ContClone, DcpCommand, DdeserializeCommand, DserializeCommand, DsyncCommand, FsCopy, uuid_from_obj) +from dfuse_utils import get_dfuse, start_dfuse from duns_utils import format_path from exception_utils import CommandFailure from general_utils import create_string_buffer, get_log_file @@ -995,8 +996,8 @@ def run_dm_activities_with_ior(self, tool, pool, cont, create_dataset=False): tool (str): specify the tool name to be used pool (TestPool): source pool object cont (TestContainer): source container object - create_dataset (bool): boolean to create initial set of - data using ior. Defaults to False. + create_dataset (bool, optional): boolean to create initial set of data using ior. + Defaults to False. """ # Set the tool to use self.set_tool(tool) @@ -1009,6 +1010,9 @@ def run_dm_activities_with_ior(self, tool, pool, cont, create_dataset=False): cont2 = self.get_container(pool, oclass=self.ior_cmd.dfs_oclass.value) # perform various datamover activities + daos_path = None + read_back_cont = None + read_back_pool = None if tool == 'CONT_CLONE': result = self.run_datamover( self.test_id + " (cont to cont2)", @@ -1021,7 +1025,8 @@ def run_dm_activities_with_ior(self, tool, pool, cont, create_dataset=False): pool2 = self.get_pool() # Use dfuse as a shared intermediate for serialize + deserialize dfuse_cont = self.get_container(pool, oclass=self.ior_cmd.dfs_oclass.value) - self.start_dfuse(self.dfuse_hosts, pool, dfuse_cont) + self.dfuse = get_dfuse(self, self.dfuse_hosts) + start_dfuse(self, self.dfuse, pool, dfuse_cont) self.serial_tmp_dir = self.dfuse.mount_dir.value # Serialize/Deserialize container 1 to a new cont2 in pool2 diff --git a/src/tests/ftest/util/dfuse_test_base.py b/src/tests/ftest/util/dfuse_test_base.py deleted file mode 100644 index df1f6c8d721..00000000000 --- a/src/tests/ftest/util/dfuse_test_base.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -(C) Copyright 2020-2022 Intel Corporation. - -SPDX-License-Identifier: BSD-2-Clause-Patent -""" -from agent_utils import include_local_host -from apricot import TestWithServers -from dfuse_utils import get_dfuse, start_dfuse -from exception_utils import CommandFailure - - -class DfuseTestBase(TestWithServers): - """Runs Dfuse test suites. - - :avocado: recursive - """ - - def __init__(self, *args, **kwargs): - """Initialize a TestWithServers object.""" - super().__init__(*args, **kwargs) - self.dfuse = None - - def setUp(self): - """Set up the test case.""" - super().setUp() - # using localhost as client if client list is empty - if not self.hostlist_clients: - self.hostlist_clients = include_local_host(None) - - def stop_job_managers(self): - """Stop the test job manager followed by dfuse. - - Returns: - list: a list of exceptions raised stopping the agents - - """ - error_list = super().stop_job_managers() - try: - self.stop_dfuse() - except CommandFailure as error: - error_list.append("Error stopping dfuse: {}".format(error)) - return error_list - - def load_dfuse(self, hosts, namespace=None): - """Create a DfuseCommand object - - Args: - hosts (NodeSet): hosts on which to start Dfuse - namespace (str, optional): dfuse namespace. Defaults to None - """ - self.dfuse = get_dfuse(self, hosts, namespace) - - def start_dfuse(self, hosts, pool=None, container=None, **params): - """Create a DfuseCommand object and use it to start Dfuse. - - Args: - hosts (NodeSet): hosts on which to start Dfuse - pool (TestPool, optional): pool to mount. Defaults to None - container (TestContainer, optional): container to mount. Defaults to None - params (Object, optional): Dfuse command arguments to update - """ - if self.dfuse is None: - self.load_dfuse(hosts) - start_dfuse(self, self.dfuse, pool=pool, container=container, **params) - - def stop_dfuse(self): - """Stop Dfuse and unset the DfuseCommand object.""" - if self.dfuse: - self.dfuse.stop() - self.dfuse = None diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index c31aa74198b..c509fbf1541 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -631,6 +631,7 @@ def pool_create(self, scm_size, uid=None, gid=None, nvme_size=None, data["status"] = output["status"] data["uuid"] = output["response"]["uuid"] data["svc"] = ",".join([str(svc) for svc in output["response"]["svc_reps"]]) + data["leader"] = output["response"]["svc_ldr"] data["ranks"] = ",".join([str(r) for r in output["response"]["tgt_ranks"]]) data["scm_per_rank"] = output["response"]["tier_bytes"][0] data["nvme_per_rank"] = output["response"]["tier_bytes"][1] @@ -662,7 +663,7 @@ def pool_query(self, pool, show_enabled=False, show_disabled=False): # "total_engines": 1, # "disabled_targets": 0, # "version": 1, - # "leader": 0, + # "svc_ldr": 0, # "rebuild": { # "status": 0, # "state": "idle", @@ -838,10 +839,8 @@ def pool_list(self, no_query=False, verbose=False): # "svc_reps": [ # 0 # ], - # "targets_total": 8, - # "targets_disabled": 0, - # "query_error_msg": "", - # "query_status_msg": "", + # "total_targets": 8, + # "disabled_targets": 0, # "usage": [ # { # "tier_name": "SCM", diff --git a/src/tests/ftest/util/ec_utils.py b/src/tests/ftest/util/ec_utils.py index 7b49629f913..54ccda3b9aa 100644 --- a/src/tests/ftest/util/ec_utils.py +++ b/src/tests/ftest/util/ec_utils.py @@ -11,8 +11,7 @@ from apricot import TestWithServers from daos_utils import DaosCommand from exception_utils import CommandFailure -from fio_test_base import FioBase -from general_utils import DaosTestError, run_pcmd +from general_utils import DaosTestError from mdtest_test_base import MdtestBase from nvme_utils import ServerFillUp from pydaos.raw import DaosApiError @@ -375,8 +374,11 @@ def read_single_type_dataset(self, results=None, parity=1): def start_online_single_operation(self, operation, parity=1): """Do Write/Read operation with single data type. + Raises: + ValueError: if operation is invalid + Args: - operation (str): Write/Read operation + operation (str): WRITE or READ operation """ # Create the single data Write/Read threads if operation == 'WRITE': @@ -386,6 +388,8 @@ def start_online_single_operation(self, operation, parity=1): job = threading.Thread(target=self.read_single_type_dataset, kwargs={"results": self.out_queue, "parity": parity}) + else: + raise ValueError(f'Invalid operation: {operation}') # Launch the single data write/read thread job.start() @@ -462,96 +466,3 @@ def start_online_mdtest(self): while not self.out_queue.empty(): if self.out_queue.get() == "Mdtest Failed": self.fail("FAIL") - - -class ErasureCodeFio(FioBase): - """Class to use for EC testing with Fio Benchmark.""" - - def __init__(self, *args, **kwargs): - """Initialize a FioBase object.""" - super().__init__(*args, **kwargs) - self.server_count = None - self.set_online_rebuild = False - self.rank_to_kill = None - - def setUp(self): - """Set up each test case.""" - super().setUp() - engine_count = self.server_managers[0].get_config_value("engines_per_host") - self.server_count = len(self.hostlist_servers) * engine_count - - # Create Pool - self.add_pool() - self.out_queue = queue.Queue() - - def stop_job_managers(self): - """Cleanup dfuse in case of test failure.""" - error_list = [] - dfuse_cleanup_cmd = ["pkill dfuse --signal KILL", - "fusermount3 -uz {}".format(self.dfuse.mount_dir.value)] - - for cmd in dfuse_cleanup_cmd: - results = run_pcmd(self.hostlist_clients, cmd) - for result in results: - if result["exit_status"] != 0: - error_list.append("Errors detected during cleanup cmd %s on node %s", - cmd, str(result["hosts"])) - error_list.extend(super().stop_job_managers()) - return error_list - - def write_single_fio_dataset(self, results): - """Run Fio Benchmark. - - Args: - results (queue): queue for returning thread results - """ - try: - self.execute_fio(stop_dfuse=False) - if results is not None: - results.put("PASS") - except (CommandFailure, DaosApiError, DaosTestError): - if results is not None: - results.put("FAIL") - raise - - def start_online_fio(self): - """Run Fio operation with thread in background. - - Trigger the server failure while Fio is running - """ - # Create the Fio run thread - job = threading.Thread(target=self.write_single_fio_dataset, - kwargs={"results": self.out_queue}) - - # Launch the Fio thread - job.start() - - # Kill the server rank while IO operation in progress - if self.set_online_rebuild: - time.sleep(30) - # Kill the server rank - if self.rank_to_kill is not None: - self.server_managers[0].stop_ranks([self.rank_to_kill], - self.d_log, - force=True) - - # Wait to finish the thread - job.join() - - # Verify the queue result and make sure test has no failure - while not self.out_queue.empty(): - if self.out_queue.get() == "FAIL": - self.fail("FAIL") - - def check_aggregation_status(self, quick_check=True, attempt=20): - """EC Aggregation triggered status. - - Args: - quick_check (bool): Return immediately when Aggregation starts for any storage type. - attempt (int): Number of attempts to do pool query at interval of 5 seconds. - default is 20 attempts. - - Returns: - dict: Storage Aggregation stats SCM/NVMe True/False. - """ - return check_aggregation_status(self.log, self.pool, quick_check, attempt) diff --git a/src/tests/ftest/util/file_count_test_base.py b/src/tests/ftest/util/file_count_test_base.py index 88e1ceb1f27..e7ef302bed8 100644 --- a/src/tests/ftest/util/file_count_test_base.py +++ b/src/tests/ftest/util/file_count_test_base.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -27,13 +27,12 @@ def add_containers(self, oclass=None): container = self.get_container(self.pool, create=False) # don't include oclass in daos cont cmd; include rd_fac based on the class if oclass: + properties = container.properties.value container.oclass.update(oclass) redundancy_factor = extract_redundancy_factor(oclass) rd_fac = 'rd_fac:{}'.format(str(redundancy_factor)) - properties = container.properties.value - cont_properties = (",").join(filter(None, [properties, rd_fac])) - if cont_properties is not None: - container.properties.update(cont_properties) + properties = (",").join(filter(None, [properties, rd_fac])) + container.properties.update(properties) container.create() return container diff --git a/src/tests/ftest/util/fio_test_base.py b/src/tests/ftest/util/fio_test_base.py index 4044d71c847..93c802718c3 100644 --- a/src/tests/ftest/util/fio_test_base.py +++ b/src/tests/ftest/util/fio_test_base.py @@ -1,13 +1,13 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ -from dfuse_test_base import DfuseTestBase +from apricot import TestWithServers from fio_utils import FioCommand -class FioBase(DfuseTestBase): +class FioBase(TestWithServers): """Base fio class. :avocado: recursive @@ -34,37 +34,7 @@ def setUp(self): self.processes = self.params.get("np", '/run/fio/client_processes/*') self.manager = self.params.get("manager", '/run/fio/*', "MPICH") - def execute_fio(self, directory=None, stop_dfuse=True): - """Runner method for Fio. - - Args: - directory (str): path for fio run dir - stop_dfuse (bool): Flag to stop or not stop dfuse as part of this method. - """ - # Create a pool if one does not already exist - if self.pool is None: - self.add_pool(connect=False) - - # start dfuse if api is POSIX - if self.fio_cmd.api.value == "POSIX": - if directory: - self.fio_cmd.update( - "global", "directory", directory, - "fio --name=global --directory") - else: - self.add_container(self.pool) - - # Instruct dfuse to disable direct-io for this container - self.container.set_attr(attrs={'dfuse-direct-io-disable': 'on'}) - - self.start_dfuse(self.hostlist_clients, self.pool, self.container) - self.fio_cmd.update( - "global", "directory", self.dfuse.mount_dir.value, - "fio --name=global --directory") - - # Run Fio + def execute_fio(self): + """Runner method for Fio.""" self.fio_cmd.hosts = self.hostlist_clients self.fio_cmd.run() - - if stop_dfuse: - self.stop_dfuse() diff --git a/src/tests/ftest/util/fio_utils.py b/src/tests/ftest/util/fio_utils.py index 1a511257afd..983f2255841 100644 --- a/src/tests/ftest/util/fio_utils.py +++ b/src/tests/ftest/util/fio_utils.py @@ -135,6 +135,14 @@ def update(self, job_name, param_name, value, description=None): else: self.log.error("Invalid job name: %s", job_name) + def update_directory(self, directory): + """Helper method for setting Fio directory command line option. + + Args: + directory (str): fio directory argument value + """ + self.update("global", "directory", directory, "fio --name=global --directory") + @property def command_with_params(self): """Get the command with all of its defined parameters as a string. diff --git a/src/tests/ftest/util/ior_test_base.py b/src/tests/ftest/util/ior_test_base.py index 0e1d1bdbf8d..8290dd254ed 100644 --- a/src/tests/ftest/util/ior_test_base.py +++ b/src/tests/ftest/util/ior_test_base.py @@ -1,19 +1,21 @@ """ -(C) Copyright 2018-2023 Intel Corporation. +(C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ import os +from apricot import TestWithServers from ClusterShell.NodeSet import NodeSet -from dfuse_test_base import DfuseTestBase +from dfuse_utils import get_dfuse, start_dfuse from exception_utils import CommandFailure from general_utils import get_random_string, pcmd +from host_utils import get_local_host from ior_utils import IorCommand from job_manager_utils import get_job_manager -class IorTestBase(DfuseTestBase): +class IorTestBase(TestWithServers): """Base IOR test class. :avocado: recursive @@ -31,6 +33,7 @@ def __init__(self, *args, **kwargs): self.container = None self.ior_timeout = None self.ppn = None + self.dfuse = None def setUp(self): """Set up each test case.""" @@ -47,6 +50,10 @@ def setUp(self): self.subprocess = self.params.get("subprocess", '/run/ior/*', False) self.ior_timeout = self.params.get("ior_timeout", '/run/ior/*', None) + # Use the local host as a client for hostfile/dfuse if the client list is empty + if not self.hostlist_clients: + self.hostlist_clients = get_local_host() + def create_pool(self): """Create a TestPool object to use with ior.""" # Get the pool params and create a pool @@ -118,7 +125,7 @@ def run_ior_with_pool(self, intercept=None, display_space=True, test_file_suffix # start dfuse if api is POSIX or HDF5 with vol connector if (self.ior_cmd.api.value == "POSIX" or plugin_path) and not self.dfuse: # Initialize dfuse instance - self.load_dfuse(self.hostlist_clients) + self.dfuse = get_dfuse(self, self.hostlist_clients) # Default mount_dir to value in dfuse instance mount_dir = mount_dir or self.dfuse.mount_dir.value # Add a substring in case of HDF5-VOL @@ -126,7 +133,7 @@ def run_ior_with_pool(self, intercept=None, display_space=True, test_file_suffix sub_dir = get_random_string(5) mount_dir = os.path.join(mount_dir, sub_dir) # Connect to the pool, create container and then start dfuse - self.start_dfuse(self.hostlist_clients, self.pool, self.container, mount_dir=mount_dir) + start_dfuse(self, self.dfuse, self.pool, self.container, mount_dir=mount_dir) # setup test file for POSIX or HDF5 with vol connector if self.ior_cmd.api.value == "POSIX" or plugin_path: @@ -144,8 +151,9 @@ def run_ior_with_pool(self, intercept=None, display_space=True, test_file_suffix fail_on_warning=fail_on_warning, out_queue=out_queue, env=env) finally: - if stop_dfuse: - self.stop_dfuse() + if stop_dfuse and self.dfuse: + self.dfuse.stop() + self.dfuse = None return out diff --git a/src/tests/ftest/util/launch_utils.py b/src/tests/ftest/util/launch_utils.py index e8dc3326c14..f1c359d7f4f 100644 --- a/src/tests/ftest/util/launch_utils.py +++ b/src/tests/ftest/util/launch_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2022-2023 Intel Corporation. + (C) Copyright 2022-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -26,6 +26,8 @@ from util.user_utils import get_group_id, get_user_groups, groupadd, useradd, userdel from util.yaml_utils import YamlUpdater, get_yaml_data +D_TM_SHARED_MEMORY_KEY = 0x10242048 + class LaunchException(Exception): """Exception for launch.py execution.""" @@ -317,14 +319,19 @@ def __init__(self, avocado, launch_result, total_tests, total_repeats, tag_filte self.local_host = get_local_host() def prepare(self, logger, test_log_file, test, repeat, user_create, slurm_setup, control_host, - partition_hosts): + partition_hosts, clear_mounts): """Prepare the test for execution. Args: logger (Logger): logger for the messages produced by this method + test_log_file (str): the log file for this test test (TestInfo): the test information repeat (str): the test repetition sequence, e.g. '1/10' user_create (bool): whether to create extra test users defined by the test + slurm_setup (bool): whether to setup slurm before running the test + control_host (NodeSet): slurm control hosts + partition_hosts (NodeSet): slurm partition hosts + clear_mounts (list): mount points to remove before the test Returns: int: status code: 0 = success, 128 = failure @@ -354,6 +361,11 @@ def prepare(self, logger, test_log_file, test, repeat, user_create, slurm_setup, if status: return status + # Remove existing mount points on each test host + status = self._clear_mount_points(logger, test, clear_mounts) + if status: + return status + # Generate certificate files for the test return self._generate_certs(logger) @@ -457,9 +469,9 @@ def _setup_host_information(self, logger, test, slurm_setup, control_host, parti Args: logger (Logger): logger for the messages produced by this method test (TestInfo): the test information - slurm_setup (bool): - control_host (NodeSet): - partition_hosts (NodeSet): + slurm_setup (bool): whether to setup slurm before running the test + control_host (NodeSet): slurm control hosts + partition_hosts (NodeSet): slurm partition hosts Returns: int: status code: 0 = success, 128 = failure @@ -663,6 +675,117 @@ def _query_create_user(logger, hosts, user, gid=None, create=False): if not useradd(logger, hosts, user, gid, test_env.user_dir, True).passed: raise LaunchException(f'Error creating user {user}') + def _clear_mount_points(self, logger, test, clear_mounts): + """Remove existing mount points on each test host. + + Args: + logger (Logger): logger for the messages produced by this method + test (TestInfo): the test information + clear_mounts (list): mount points to remove before the test + + Returns: + int: status code: 0 = success, 128 = failure + """ + if not clear_mounts: + return 0 + + logger.debug("-" * 80) + hosts = test.host_info.all_hosts + logger.debug("Clearing existing mount points on %s: %s", hosts, clear_mounts) + command = f" df --type=tmpfs --output=target | grep -E '^({'|'.join(clear_mounts)})$'" + find_result = run_remote(logger, hosts, command) + mount_point_hosts = {} + for data in find_result.output: + if not data.passed: + continue + for line in data.stdout: + if line not in mount_point_hosts: + mount_point_hosts[line] = NodeSet() + mount_point_hosts[line].add(data.hosts) + + for mount_point, mount_hosts in mount_point_hosts.items(): + if not self._remove_super_blocks(logger, mount_hosts, mount_point): + message = "Error removing superblocks for existing mount points" + self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) + return 128 + + if not self._remove_shared_memory_segments(logger, hosts): + message = "Error removing shared memory segments for existing mount points" + self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) + return 128 + + for mount_point, mount_hosts in mount_point_hosts.items(): + if not self._remove_mount_point(logger, mount_hosts, mount_point): + message = "Error removing existing mount points" + self.test_result.fail_test(logger, "Prepare", message, sys.exc_info()) + return 128 + + return 0 + + def _remove_super_blocks(self, logger, hosts, mount_point): + """Remove the super blocks from the specified mount point. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to remove the super blocks + mount_point (str): mount point from which to remove the super blocks + + Returns: + bool: True if successful; False otherwise + """ + logger.debug("Clearing existing super blocks on %s", hosts) + command = f"sudo rm -fr {mount_point}/*" + return run_remote(logger, hosts, command).passed + + def _remove_shared_memory_segments(self, logger, hosts): + """Remove existing shared memory segments. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to remove the shared memory segments + + Returns: + bool: True if successful; False otherwise + """ + logger.debug("Clearing existing shared memory segments on %s", hosts) + daos_engine_keys = [hex(D_TM_SHARED_MEMORY_KEY + index) for index in range(4)] + result = run_remote(logger, hosts, "ipcs -m") + keys_per_host = {} + for data in result.output: + if not data.passed: + continue + for line in data.stdout: + info = re.split(r"\s+", line) + if info[0] not in daos_engine_keys: + # Skip processing lines not listing a shared memory segment + continue + if info[0] not in keys_per_host: + keys_per_host[info[0]] = NodeSet() + keys_per_host[info[0]].add(data.hosts) + for key, key_hosts in keys_per_host.items(): + logger.debug("Clearing shared memory segment %s on %s:", key, key_hosts) + if not run_remote(logger, key_hosts, f"sudo ipcrm -M {key}").passed: + return False + return True + + def _remove_mount_point(self, logger, hosts, mount_point): + """Remove the mount point. + + Args: + logger (Logger): logger for the messages produced by this method + hosts (NodeSet): hosts on which to remove the mount point + mount_point (str): mount point from which to remove the mount point + + Returns: + bool: True if successful; False otherwise + """ + logger.debug("Clearing mount point %s on %s:", mount_point, hosts) + commands = [f"sudo umount -f {mount_point}", f"sudo rm -fr {mount_point}"] + for command in commands: + if not run_remote(logger, hosts, command).passed: + return False + return True + def _generate_certs(self, logger): """Generate the certificates for the test. @@ -1036,15 +1159,17 @@ def _setup_application_directory(self, logger, result): run_local(logger, f"ls -al '{self._test_env.app_dir}'") return 0 - def run_tests(self, logger, result, repeat, setup, sparse, fail_fast, stop_daos, archive, + def run_tests(self, logger, result, repeat, slurm_setup, sparse, fail_fast, stop_daos, archive, rename, jenkins_log, core_files, threshold, user_create, code_coverage, - job_results_dir, logdir): + job_results_dir, logdir, clear_mounts): # pylint: disable=too-many-arguments """Run all the tests. Args: logger (Logger): logger for the messages produced by this method - mode (str): launch mode + result (Results): object tracking the result of the test + repeat (int): number of times to repeat the test + slurm_setup (bool): whether to setup slurm before running the test sparse (bool): whether or not to display the shortened avocado test output fail_fast (bool): whether or not to fail the avocado run command upon the first failure stop_daos (bool): whether or not to stop daos servers/clients after the test @@ -1055,6 +1180,9 @@ def run_tests(self, logger, result, repeat, setup, sparse, fail_fast, stop_daos, threshold (str): optional upper size limit for test log files user_create (bool): whether to create extra test users defined by the test code_coverage (CodeCoverage): bullseye code coverage + job_results_dir (str): avocado job-results directory + logdir (str): base directory in which to place the log file + clear_mounts (list): mount points to remove before each test Returns: int: status code indicating any issues running tests @@ -1084,8 +1212,8 @@ def run_tests(self, logger, result, repeat, setup, sparse, fail_fast, stop_daos, # Prepare the hosts to run the tests step_status = runner.prepare( - logger, test_log_file, test, loop, user_create, setup, self._control, - self._partition_hosts) + logger, test_log_file, test, loop, user_create, slurm_setup, self._control, + self._partition_hosts, clear_mounts) if step_status: # Do not run this test - update its failure status to interrupted return_code |= step_status diff --git a/src/tests/ftest/util/mdtest_test_base.py b/src/tests/ftest/util/mdtest_test_base.py index b17cd3d90d7..10d92ac4636 100644 --- a/src/tests/ftest/util/mdtest_test_base.py +++ b/src/tests/ftest/util/mdtest_test_base.py @@ -6,13 +6,14 @@ import os -from dfuse_test_base import DfuseTestBase +from apricot import TestWithServers +from dfuse_utils import get_dfuse, start_dfuse from exception_utils import CommandFailure from job_manager_utils import get_job_manager from mdtest_utils import MdtestCommand -class MdtestBase(DfuseTestBase): +class MdtestBase(TestWithServers): """Base mdtest class. :avocado: recursive @@ -27,6 +28,13 @@ def __init__(self, *args, **kwargs): self.hostfile_clients_slots = None self.subprocess = False + # We should not be using these as class level variables, but are needed until the + # execute_mdtest() method can be redesigned to pass in these arguments instead of + # optionally defining them + self.pool = None + self.container = None + self.dfuse = None + def setUp(self): """Set up each test case.""" # obtain separate logs @@ -83,7 +91,8 @@ def execute_mdtest(self, out_queue=None, display_space=True): # start dfuse if api is POSIX if self.mdtest_cmd.api.value == "POSIX": - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + self.dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, self.dfuse, self.pool, self.container) self.mdtest_cmd.test_dir.update(self.dfuse.mount_dir.value) # Run Mdtest @@ -99,7 +108,10 @@ def execute_mdtest(self, out_queue=None, display_space=True): self.container.skip_cleanup() # Need to set self.container to None to force a creation of a new container self.container = None - self.stop_dfuse() + + if self.dfuse is not None: + self.dfuse.stop() + self.dfuse = None return out diff --git a/src/tests/ftest/util/nvme_utils.py b/src/tests/ftest/util/nvme_utils.py index 76f5e8138e3..ac53ebe1ae9 100644 --- a/src/tests/ftest/util/nvme_utils.py +++ b/src/tests/ftest/util/nvme_utils.py @@ -223,6 +223,7 @@ def calculate_ior_block_size(self, percent, storage_type): free_space = self.pool.get_pool_daos_space()["s_total"][1] self.ior_local_cmd.transfer_size.value = self.ior_nvme_xfersize else: + free_space = None # To appease pylint self.fail('Provide storage type (SCM/NVMe) to be filled') # Get the block size based on the capacity to be filled. For example @@ -298,13 +299,13 @@ def create_pool_max_size(self, scm=False, nvme=False, percentage=96): if nvme or scm: sizes = self.get_max_storage_sizes(percentage) - # If NVMe is True get the max NVMe size from servers - if nvme: - self.pool.nvme_size.update(str(sizes[1])) + # If NVMe is True get the max NVMe size from servers + if nvme: + self.pool.nvme_size.update(str(sizes[1])) - # If SCM is True get the max SCM size from servers - if scm: - self.pool.scm_size.update(str(sizes[0])) + # If SCM is True get the max SCM size from servers + if scm: + self.pool.scm_size.update(str(sizes[0])) # Create the Pool self.pool.create() diff --git a/src/tests/ftest/util/osa_utils.py b/src/tests/ftest/util/osa_utils.py index 8626eb23476..a4a13abbab5 100644 --- a/src/tests/ftest/util/osa_utils.py +++ b/src/tests/ftest/util/osa_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -123,12 +123,14 @@ def remove_pool_dir(self, ip_addr=None, port_num=None): elif port_num == str(expected_ports[1]): port_val = 1 else: + port_val = None # To appease pylint self.log.info("port_number: %s", port_num) self.fail("Invalid port number") cmd = "/usr/bin/ssh {} -oStrictHostKeyChecking=no \ sudo rm -rf /mnt/daos{}/{}/vos-*". \ format(ip_addr, port_val, self.pool.uuid) else: + cmd = None # To appease pylint self.fail("Not supported engine per server configuration") run_command(cmd) diff --git a/src/tests/ftest/util/performance_test_base.py b/src/tests/ftest/util/performance_test_base.py index 4fdb8ae06d4..3196242bed1 100644 --- a/src/tests/ftest/util/performance_test_base.py +++ b/src/tests/ftest/util/performance_test_base.py @@ -342,7 +342,9 @@ def run_performance_ior(self, namespace=None, use_intercept=True): self._run_performance_ior_single(intercept) # Manually stop dfuse after ior write completes - self.stop_dfuse() + if self.dfuse: + self.dfuse.stop() + self.dfuse = None # Wait between write and read self.phase_barrier() @@ -352,7 +354,9 @@ def run_performance_ior(self, namespace=None, use_intercept=True): self._run_performance_ior_single(intercept) # Manually stop dfuse after ior read completes - self.stop_dfuse() + if self.dfuse: + self.dfuse.stop() + self.dfuse = None self._log_daos_metrics() @@ -438,6 +442,8 @@ def run_performance_mdtest(self, namespace=None): self.verify_system_status(self.pool, self.container) # Manually stop dfuse after mdtest completes - self.stop_dfuse() + if self.dfuse: + self.dfuse.stop() + self.dfuse = None self._log_daos_metrics() diff --git a/src/tests/ftest/util/pool_create_all_base.py b/src/tests/ftest/util/pool_create_all_base.py index 0b625e118fe..1c88e2e8c96 100644 --- a/src/tests/ftest/util/pool_create_all_base.py +++ b/src/tests/ftest/util/pool_create_all_base.py @@ -1,5 +1,5 @@ """ -(C) Copyright 2022-2023 Intel Corporation. +(C) Copyright 2022-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -307,8 +307,7 @@ def check_pool_distribution(self, scm_delta_bytes, nvme_delta_bytes=None): result = self.dmg.storage_query_usage() scm_used_bytes = [sys.maxsize, 0] - if nvme_delta_bytes is not None: - nvme_used_bytes = [sys.maxsize, 0] + nvme_used_bytes = [sys.maxsize, 0] for host_storage in result["response"]["HostStorage"].values(): scm_bytes = 0 for scm_devices in host_storage["storage"]["scm_namespaces"]: diff --git a/src/tests/ftest/util/pool_security_test_base.py b/src/tests/ftest/util/pool_security_test_base.py index 296af22a4fc..38536089a39 100644 --- a/src/tests/ftest/util/pool_security_test_base.py +++ b/src/tests/ftest/util/pool_security_test_base.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -163,6 +163,7 @@ def verify_cont_rw_attribute(self, container, action, expect, attribute, value=N with container.no_exception(): result = container.get_attr(attr=attribute) else: + result = None # To appease pylint self.fail( "##In verify_cont_rw_attribute, " "invalid action: {}".format(action)) @@ -188,6 +189,7 @@ def verify_cont_rw_property(self, container, action, expect, cont_property=None, with container.no_exception(): result = container.get_prop() else: + result = None # To appease pylint self.fail("##In verify_cont_rw_property, invalid action: {}".format(action)) self.log.info( " In verify_cont_rw_property %s.\n =daos_cmd.run() result:\n%s", action, result) @@ -224,6 +226,7 @@ def verify_cont_rw_acl(self, container, action, expect, entry=None): elif action.lower() == "read": result = self.get_container_acl_list(container) else: + result = None # To appease pylint self.fail( "##In verify_cont_rw_acl, invalid action: {}".format(action)) self.log.info( @@ -330,6 +333,7 @@ def verify_pool_readwrite(self, pool, action, expect='Pass'): elif action.lower() == "read": result = daos_cmd.pool_query(pool.identifier) else: + result = None # To appease pylint self.fail("##In verify_pool_readwrite, invalid action: {}".format(action)) self.log.info( " In verify_pool_readwrite %s.\n =daos_cmd.run() result:\n%s", diff --git a/src/tests/ftest/util/recovery_test_base.py b/src/tests/ftest/util/recovery_test_base.py index c67fe46d7d3..7673bcb1b4e 100644 --- a/src/tests/ftest/util/recovery_test_base.py +++ b/src/tests/ftest/util/recovery_test_base.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2023 Intel Corporation. + (C) Copyright 2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -46,8 +46,6 @@ def get_vos_file_path(self, pool): self.log.info("vos_file: %s", file) return file - self.fail("vos file wasn't found in {}/{}".format(scm_mount, pool.uuid.lower())) - return None # to appease pylint def wait_for_check_complete(self): diff --git a/src/tests/ftest/util/server_utils.py b/src/tests/ftest/util/server_utils.py index ae6eeaf58b8..ad83ddd97d2 100644 --- a/src/tests/ftest/util/server_utils.py +++ b/src/tests/ftest/util/server_utils.py @@ -132,7 +132,7 @@ def __init__(self, group, bin_dir, # Parameters to set storage prepare and format timeout self.storage_prepare_timeout = BasicParameter(None, 40) - self.storage_format_timeout = BasicParameter(None, 40) + self.storage_format_timeout = BasicParameter(None, 64) self.storage_reset_timeout = BasicParameter(None, 120) self.collect_log_timeout = BasicParameter(None, 120) diff --git a/src/tests/ftest/util/server_utils_base.py b/src/tests/ftest/util/server_utils_base.py index 20cf9410dab..bbda608faad 100644 --- a/src/tests/ftest/util/server_utils_base.py +++ b/src/tests/ftest/util/server_utils_base.py @@ -370,8 +370,6 @@ def __init__(self, default_config=None): # --socket_dir= Location for all daos_server and # daos_engine sockets # --insecure allow for insecure connections - # --recreate-superblocks recreate missing superblocks rather than - # failing self.port = FormattedParameter("--port={}") self.storage = FormattedParameter("--storage={}") self.modules = FormattedParameter("--modules={}") @@ -381,7 +379,6 @@ def __init__(self, default_config=None): self.group = FormattedParameter("--group={}") self.sock_dir = FormattedParameter("--socket_dir={}") self.insecure = FormattedParameter("--insecure", False) - self.recreate = FormattedParameter("--recreate-superblocks", False) self.config = FormattedParameter("--config={}", default_config) class NvmeSubCommand(CommandWithSubCommand): diff --git a/src/tests/ftest/util/soak_test_base.py b/src/tests/ftest/util/soak_test_base.py index 886e667d9ca..d75456cdfd6 100644 --- a/src/tests/ftest/util/soak_test_base.py +++ b/src/tests/ftest/util/soak_test_base.py @@ -68,6 +68,7 @@ def __init__(self, *args, **kwargs): self.initial_resv_file = None self.resv_cont = None self.mpi_module = None + self.mpi_module_use = None self.sudo_cmd = None self.slurm_exclude_servers = True self.control = get_local_host() @@ -559,6 +560,8 @@ def run_soak(self, test_param): self.check_errors = [] self.used = [] self.mpi_module = self.params.get("mpi_module", "/run/*", default="mpi/mpich-x86_64") + self.mpi_module_use = self.params.get( + "mpi_module_use", "/run/*", default="/usr/share/modulefiles") enable_sudo = self.params.get("enable_sudo", "/run/*", default=True) test_to = self.params.get(self.test_id, os.path.join(test_param, "test_timeout", "*")) self.test_name = self.params.get("name", test_param + "*") diff --git a/src/tests/ftest/util/soak_utils.py b/src/tests/ftest/util/soak_utils.py index 99f6547f406..31a77bf6814 100644 --- a/src/tests/ftest/util/soak_utils.py +++ b/src/tests/ftest/util/soak_utils.py @@ -853,7 +853,7 @@ def start_dfuse(self, pool, container, name=None, job_spec=None): self.test_name + "_" + name + "_`hostname -s`_" "" + "${SLURM_JOB_ID}_" + "daos_dfuse.log") dfuse_env = f"export D_LOG_FILE_APPEND_PID=1;export D_LOG_MASK=ERR;export D_LOG_FILE={dfuselog}" - module_load = f"module load {self.mpi_module}" + module_load = f"module use {self.mpi_module_use};module load {self.mpi_module}" dfuse_start_cmds = [ "clush -S -w $SLURM_JOB_NODELIST \"mkdir -p {}\"".format(dfuse.mount_dir.value), @@ -982,7 +982,7 @@ def create_ior_cmdline(self, job_spec, pool, ppn, nodesperjob, oclass_list=None, + "_`hostname -s`_${SLURM_JOB_ID}_daos.log") env = ior_cmd.get_default_env("mpirun", log_file=daos_log) env["D_LOG_FILE_APPEND_PID"] = "1" - sbatch_cmds = [f"module load {self.mpi_module}"] + sbatch_cmds = [f"module use {self.mpi_module_use}", f"module load {self.mpi_module}"] # include dfuse cmdlines if api in ["HDF5-VOL", "POSIX", "POSIX-LIBPIL4DFS", "POSIX-LIBIOIL"]: dfuse, dfuse_start_cmdlist = start_dfuse( @@ -1064,7 +1064,7 @@ def create_macsio_cmdline(self, job_spec, pool, ppn, nodesperjob): env["D_LOG_FILE"] = get_log_file(daos_log or f"{macsio.command}_daos.log") env["D_LOG_FILE_APPEND_PID"] = "1" env["DAOS_UNS_PREFIX"] = format_path(macsio.daos_pool, macsio.daos_cont) - sbatch_cmds = [f"module load {self.mpi_module}"] + sbatch_cmds = [f"module use {self.mpi_module_use}", f"module load {self.mpi_module}"] mpirun_cmd = Mpirun(macsio, mpi_type=self.mpi_module) mpirun_cmd.get_params(self) mpirun_cmd.assign_processes(nodesperjob * ppn) @@ -1153,7 +1153,7 @@ def create_mdtest_cmdline(self, job_spec, pool, ppn, nodesperjob): + "_`hostname -s`_${SLURM_JOB_ID}_daos.log") env = mdtest_cmd.get_default_env("mpirun", log_file=daos_log) env["D_LOG_FILE_APPEND_PID"] = "1" - sbatch_cmds = [f"module load {self.mpi_module}"] + sbatch_cmds = [f"module use {self.mpi_module_use}", f"module load {self.mpi_module}"] # include dfuse cmdlines if api in ["POSIX", "POSIX-LIBPIL4DFS", "POSIX-LIBIOIL"]: dfuse, dfuse_start_cmdlist = start_dfuse( @@ -1327,6 +1327,7 @@ def create_app_cmdline(self, job_spec, pool, ppn, nodesperjob): commands = [] app_params = os.path.join(os.sep, "run", job_spec, "*") mpi_module = self.params.get("module", app_params, self.mpi_module) + mpi_module_use = self.params.get("module_use", app_params, self.mpi_module_use) api_list = self.params.get("api", app_params, default=["DFS"]) apps_dir = os.environ["DAOS_TEST_APP_DIR"] # Update DAOS_TEST_APP_DIR if used in the cmdline param in yaml @@ -1348,7 +1349,7 @@ def create_app_cmdline(self, job_spec, pool, ppn, nodesperjob): oclass_list = self.params.get("oclass", app_params) for file_oclass, dir_oclass in oclass_list: for api in api_list: - sbatch_cmds = [f"module load {mpi_module}"] + sbatch_cmds = [f"module use {mpi_module_use}", f"module load {mpi_module}"] if not self.enable_il and api in ["POSIX-LIBIOIL", "POSIX-LIBPIL4DFS"]: continue add_containers(self, pool, file_oclass, dir_oclass) diff --git a/src/tests/ftest/util/telemetry_utils.py b/src/tests/ftest/util/telemetry_utils.py index 8631037cbe8..aec831b3b8a 100644 --- a/src/tests/ftest/util/telemetry_utils.py +++ b/src/tests/ftest/util/telemetry_utils.py @@ -9,6 +9,7 @@ from logging import getLogger from ClusterShell.NodeSet import NodeSet +from exception_utils import CommandFailure def _gen_stats_metrics(basename): @@ -147,13 +148,23 @@ class TelemetryUtils(): ENGINE_POOL_VOS_WAL_METRICS = [ *_gen_stats_metrics("engine_pool_vos_wal_wal_sz"), *_gen_stats_metrics("engine_pool_vos_wal_wal_qd"), - *_gen_stats_metrics("engine_pool_vos_wal_wal_waiters")] + *_gen_stats_metrics("engine_pool_vos_wal_wal_waiters"), + *_gen_stats_metrics("engine_pool_vos_wal_wal_dur")] ENGINE_POOL_VOS_WAL_REPLAY_METRICS = [ "engine_pool_vos_wal_replay_count", "engine_pool_vos_wal_replay_entries", "engine_pool_vos_wal_replay_size", "engine_pool_vos_wal_replay_time", "engine_pool_vos_wal_replay_transactions"] + ENGINE_POOL_SVC_METRICS = [ + "engine_pool_svc_degraded_ranks", + "engine_pool_svc_disabled_targets", + "engine_pool_svc_draining_targets", + "engine_pool_svc_leader", + "engine_pool_svc_map_version", + "engine_pool_svc_open_pool_handles", + "engine_pool_svc_total_ranks", + "engine_pool_svc_total_targets"] ENGINE_POOL_METRICS = ENGINE_POOL_ACTION_METRICS +\ ENGINE_POOL_BLOCK_ALLOCATOR_METRICS +\ ENGINE_POOL_CHECKPOINT_METRICS +\ @@ -164,7 +175,8 @@ class TelemetryUtils(): ENGINE_POOL_VOS_AGGREGATION_METRICS +\ ENGINE_POOL_VOS_SPACE_METRICS + \ ENGINE_POOL_VOS_WAL_METRICS + \ - ENGINE_POOL_VOS_WAL_REPLAY_METRICS + ENGINE_POOL_VOS_WAL_REPLAY_METRICS +\ + ENGINE_POOL_SVC_METRICS ENGINE_EVENT_METRICS = [ "engine_events_dead_ranks", "engine_events_last_event_ts", @@ -486,8 +498,12 @@ def list_metrics(self, hosts=None): host_list = hosts or self.hosts self.log.info("Listing telemetry metrics from %s", host_list) for host in host_list: - data = self.dmg.telemetry_metrics_list(host=host) info[host] = [] + try: + data = self.dmg.telemetry_metrics_list(host=host) + except CommandFailure as err: + self.log.error("Failed to list metrics on %s: %s", host, err) + continue if "response" in data: if "available_metric_sets" in data["response"]: for entry in data["response"]["available_metric_sets"]: @@ -549,8 +565,12 @@ def get_metrics(self, name, hosts=None): host_list = hosts or self.hosts self.log.info("Querying telemetry metric %s from %s", name, host_list) for host in host_list: - data = self.dmg.telemetry_metrics_query(host=host, metrics=name) info[host] = {} + try: + data = self.dmg.telemetry_metrics_query(host=host, metrics=name) + except CommandFailure as err: + self.log.error("Failed to get metrics for %s: %s", host, err) + continue if "response" in data: if "metric_sets" in data["response"]: for entry in data["response"]["metric_sets"]: diff --git a/src/tests/ftest/util/test_utils_container.py b/src/tests/ftest/util/test_utils_container.py index 77019ea558b..b7cb9588e9f 100644 --- a/src/tests/ftest/util/test_utils_container.py +++ b/src/tests/ftest/util/test_utils_container.py @@ -255,7 +255,6 @@ def read_record(self, container, akey, dkey, data_size, data_array_size=0, "akey": akey, "obj": self.obj, "txn": txn, - "test_hints": test_hints, } try: if data_array_size > 0: @@ -265,6 +264,7 @@ def read_record(self, container, akey, dkey, data_size, data_array_size=0, read_data = container.container.read_an_array(**kwargs) else: kwargs["size"] = data_size + kwargs["test_hints"] = test_hints self._log_method("read_an_obj", kwargs) read_data = container.container.read_an_obj(**kwargs) except DaosApiError as error: diff --git a/src/tests/ftest/util/test_utils_pool.py b/src/tests/ftest/util/test_utils_pool.py index 48e79d61994..fbb6484e292 100644 --- a/src/tests/ftest/util/test_utils_pool.py +++ b/src/tests/ftest/util/test_utils_pool.py @@ -242,6 +242,7 @@ def __init__(self, context, dmg_command, label_generator=None, namespace=POOL_NA self.pool = None self.info = None self.svc_ranks = None + self.svc_leader = None self.connected = False # Flag to allow the non-create operations to use UUID. e.g., if you want # to destroy the pool with UUID, set this to False, then call destroy(). @@ -454,6 +455,7 @@ def create(self): self.svc_ranks = [ int(self.pool.svc.rl_ranks[index]) for index in range(self.pool.svc.rl_nr)] + self.svc_leader = int(data["leader"]) @fail_on(DaosApiError) def connect(self, permission=2): @@ -1044,7 +1046,7 @@ def get_space_per_target(self, ranks, target_idx): rank_result = self.query_targets(rank=rank, target_idx=target_idx) for target, target_info in enumerate(rank_result['response']['Infos']): rank_target_tier_space[rank][target] = {} - for tier in target_info['Space']: + for tier in target_info['space']: rank_target_tier_space[rank][target][tier['media_type']] = { 'total': tier['total'], 'free': tier['free'], diff --git a/src/tests/ftest/util/vol_test_base.py b/src/tests/ftest/util/vol_test_base.py index b45bcf62082..ad9f7b2ca51 100644 --- a/src/tests/ftest/util/vol_test_base.py +++ b/src/tests/ftest/util/vol_test_base.py @@ -1,16 +1,17 @@ """ -(C) Copyright 2020-2023 Intel Corporation. +(C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ +from apricot import TestWithServers from command_utils import ExecutableCommand from command_utils_base import EnvironmentVariables -from dfuse_test_base import DfuseTestBase +from dfuse_utils import get_dfuse, start_dfuse from exception_utils import CommandFailure -class VolTestBase(DfuseTestBase): +class VolTestBase(TestWithServers): """Runs HDF5 vol test-suites. :avocado: recursive @@ -29,14 +30,17 @@ def run_test(self, job_manager, plugin_path, test_repo): client_processes = self.params.get("client_processes") # create pool, container and dfuse mount - self.add_pool(connect=False) - self.add_container(self.pool) + self.log_step('Creating a single pool and container') + pool = self.get_pool(connect=False) + container = self.get_container(pool) # VOL needs to run from a file system that supports xattr. # Currently nfs does not have this attribute so it was recommended # to create a dfuse dir and run vol tests from there. # create dfuse container - self.start_dfuse(self.hostlist_clients, self.pool, self.container) + self.log_step('Starting dfuse so VOL can run from a file system that supports xattr') + dfuse = get_dfuse(self, self.hostlist_clients) + start_dfuse(self, dfuse, pool, container) # Assign the test to run job_manager.job = ExecutableCommand( @@ -44,17 +48,19 @@ def run_test(self, job_manager, plugin_path, test_repo): check_results=["FAILED", "stderr"]) env = EnvironmentVariables() - env["DAOS_POOL"] = "{}".format(self.pool.uuid) - env["DAOS_CONT"] = "{}".format(self.container.uuid) + env["DAOS_POOL"] = "{}".format(pool.uuid) + env["DAOS_CONT"] = "{}".format(container.uuid) env["HDF5_VOL_CONNECTOR"] = "daos" env["HDF5_PLUGIN_PATH"] = "{}".format(plugin_path) job_manager.assign_hosts(self.hostlist_clients) job_manager.assign_processes(client_processes) job_manager.assign_environment(env, True) - job_manager.working_dir.value = self.dfuse.mount_dir.value + job_manager.working_dir.value = dfuse.mount_dir.value # run VOL Command + self.log_step(f'Running {job_manager.job.command}') try: job_manager.run() - except CommandFailure as _error: - self.fail("{} FAILED> \nException occurred: {}".format(job_manager.job, str(_error))) + except CommandFailure as error: + self.log.error(str(error)) + self.fail(f"{job_manager.job.command} failed") diff --git a/src/tests/suite/daos_container.c b/src/tests/suite/daos_container.c index 89e3197b7c2..d75850af720 100644 --- a/src/tests/suite/daos_container.c +++ b/src/tests/suite/daos_container.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -1079,7 +1079,7 @@ co_acl(void **state) rc = daos_cont_open(arg->pool.poh, arg->co_str, DAOS_COO_RW, &arg->coh, NULL, NULL); assert_rc_equal(rc, 0); - rc = daos_cont_set_owner(arg->coh, exp_owner, exp_owner_grp, NULL); + rc = daos_cont_set_owner_no_check(arg->coh, exp_owner, exp_owner_grp, NULL); assert_rc_equal(rc, 0); rc = daos_cont_close(arg->coh, NULL); @@ -1438,7 +1438,7 @@ create_cont_with_user_perms(test_arg_t *arg, uint64_t perms) assert_rc_equal(rc, 0); /* remove current user's ownership, so they don't have owner perms */ - rc = daos_cont_set_owner(coh, "nobody@", NULL, NULL); + rc = daos_cont_set_owner_no_check(coh, "nobody@", NULL, NULL); assert_rc_equal(rc, 0); acl = get_daos_acl_with_user_perms(perms); @@ -2126,8 +2126,8 @@ co_set_owner(void **state) test_arg_t *arg = NULL; d_string_t original_user; d_string_t original_grp; - d_string_t new_user = "newuser@"; - d_string_t new_grp = "newgrp@"; + d_string_t fake_user = "newuser@"; + d_string_t fake_grp = "newgrp@"; int rc; rc = test_setup((void **)&arg, SETUP_CONT_CONNECT, arg0->multi_rank, @@ -2148,39 +2148,50 @@ co_set_owner(void **state) assert_rc_equal(rc, -DER_INVAL); print_message("Set owner with invalid user\n"); - rc = daos_cont_set_owner(arg->coh, "not_a_valid_user", new_grp, - NULL); + rc = daos_cont_set_owner(arg->coh, "notvaliduser!", NULL, NULL); assert_rc_equal(rc, -DER_INVAL); print_message("Set owner with invalid grp\n"); - rc = daos_cont_set_owner(arg->coh, new_user, "not_a_valid_grp", - NULL); + rc = daos_cont_set_owner(arg->coh, NULL, "notvalidgrp!", NULL); assert_rc_equal(rc, -DER_INVAL); - print_message("Set owner user\n"); - rc = daos_cont_set_owner(arg->coh, new_user, NULL, NULL); + print_message("Set owner user to nonexistent\n"); + rc = daos_cont_set_owner(arg->coh, fake_user, NULL, NULL); + assert_rc_equal(rc, -DER_NONEXIST); + + print_message("Set owner user to nonexistent with no_check\n"); + rc = daos_cont_set_owner_no_check(arg->coh, fake_user, NULL, NULL); assert_rc_equal(rc, 0); - expect_ownership(arg, new_user, original_grp); + expect_ownership(arg, fake_user, original_grp); print_message("Change owner user back\n"); rc = daos_cont_set_owner(arg->coh, original_user, NULL, NULL); assert_rc_equal(rc, 0); expect_ownership(arg, original_user, original_grp); - print_message("Set owner group\n"); - rc = daos_cont_set_owner(arg->coh, NULL, new_grp, NULL); + print_message("Set owner group to nonexistent\n"); + rc = daos_cont_set_owner(arg->coh, NULL, fake_grp, NULL); + assert_rc_equal(rc, -DER_NONEXIST); + + print_message("Set owner group to nonexistent with no_check\n"); + rc = daos_cont_set_owner_no_check(arg->coh, NULL, fake_grp, NULL); assert_rc_equal(rc, 0); - expect_ownership(arg, original_user, new_grp); + expect_ownership(arg, original_user, fake_grp); print_message("Change owner group back\n"); rc = daos_cont_set_owner(arg->coh, NULL, original_grp, NULL); assert_rc_equal(rc, 0); expect_ownership(arg, original_user, original_grp); - print_message("Set both owner user and group\n"); - rc = daos_cont_set_owner(arg->coh, new_user, new_grp, NULL); + print_message("Set both owner user and group with no_check\n"); + rc = daos_cont_set_owner_no_check(arg->coh, fake_user, fake_grp, NULL); + assert_rc_equal(rc, 0); + expect_ownership(arg, fake_user, fake_grp); + + print_message("Set both owner user and group back to original\n"); + rc = daos_cont_set_owner(arg->coh, original_user, original_grp, NULL); assert_rc_equal(rc, 0); - expect_ownership(arg, new_user, new_grp); + expect_ownership(arg, original_user, original_grp); } D_FREE(original_user); @@ -2219,32 +2230,39 @@ co_set_owner_access(void **state) test_arg_t *arg0 = *state; test_arg_t *arg = NULL; int rc; - uint64_t no_perm = DAOS_ACL_PERM_CONT_ALL & - ~DAOS_ACL_PERM_SET_OWNER; + uid_t uid = geteuid(); + gid_t gid = getegid(); + char *user = NULL; + char *group = NULL; + uint64_t no_perm = DAOS_ACL_PERM_CONT_ALL & ~DAOS_ACL_PERM_SET_OWNER; + + rc = daos_acl_uid_to_principal(uid, &user); + assert_success(rc); + + rc = daos_acl_gid_to_principal(gid, &group); + assert_success(rc); rc = test_setup((void **)&arg, SETUP_EQ, arg0->multi_rank, SMALL_POOL_SIZE, 0, NULL); assert_success(rc); print_message("Set owner user denied with no set-owner perm\n"); - expect_co_set_owner_access(arg, "user@", NULL, no_perm, - -DER_NO_PERM); + expect_co_set_owner_access(arg, user, NULL, no_perm, -DER_NO_PERM); print_message("Set owner group denied with no set-owner perm\n"); - expect_co_set_owner_access(arg, NULL, "group@", no_perm, - -DER_NO_PERM); + expect_co_set_owner_access(arg, NULL, group, no_perm, -DER_NO_PERM); print_message("Set both owner and grp denied with no set-owner perm\n"); - expect_co_set_owner_access(arg, "user@", "group@", no_perm, - -DER_NO_PERM); + expect_co_set_owner_access(arg, user, group, no_perm, -DER_NO_PERM); - print_message("Set owner allowed with set-owner perm\n"); - expect_co_set_owner_access(arg, "user@", "group@", - DAOS_ACL_PERM_READ | - DAOS_ACL_PERM_SET_OWNER, + print_message("Set owner succeeds with set-owner perm\n"); + expect_co_set_owner_access(arg, user, group, DAOS_ACL_PERM_READ | DAOS_ACL_PERM_SET_OWNER, 0); test_teardown((void **)&arg); + + D_FREE(user); + D_FREE(group); } static void @@ -2295,6 +2313,16 @@ co_owner_implicit_access(void **state) struct daos_acl *acl; daos_prop_t *tmp_prop; daos_prop_t *acl_prop; + uid_t uid = geteuid(); + gid_t gid = getegid(); + char *user = NULL; + char *group = NULL; + + rc = daos_acl_uid_to_principal(uid, &user); + assert_success(rc); + + rc = daos_acl_gid_to_principal(gid, &group); + assert_success(rc); /* * An owner with no permissions still has get/set ACL access @@ -2330,7 +2358,7 @@ co_owner_implicit_access(void **state) daos_prop_free(tmp_prop); print_message("- Verify set-owner denied\n"); - rc = daos_cont_set_owner(arg->coh, "somebody@", "somegroup@", NULL); + rc = daos_cont_set_owner(arg->coh, user, group, NULL); assert_rc_equal(rc, -DER_NO_PERM); print_message("Owner has get-ACL access implicitly\n"); @@ -2363,6 +2391,8 @@ co_owner_implicit_access(void **state) daos_acl_free(acl); daos_prop_free(owner_deny_prop); test_teardown((void **)&arg); + D_FREE(user); + D_FREE(group); } static void diff --git a/src/tests/suite/daos_cr.c b/src/tests/suite/daos_cr.c index 1e1b0c29a13..4607674ea0a 100644 --- a/src/tests/suite/daos_cr.c +++ b/src/tests/suite/daos_cr.c @@ -1851,6 +1851,7 @@ cr_pause(void **state, bool force) uint32_t class = TCC_POOL_BAD_LABEL; uint32_t action = TCA_INTERACT; int rc; + int i; rc = cr_pool_create(state, &pool, false, class); assert_rc_equal(rc, 0); @@ -1878,12 +1879,17 @@ cr_pause(void **state, bool force) rc = cr_system_start(); assert_rc_equal(rc, 0); - /* Sleep for a while after system re-started under check mode. */ - sleep(5); + for (i = 0; i < CR_WAIT_MAX; i += 5) { + /* Sleep for a while after system re-started under check mode. */ + sleep(5); - cr_dci_fini(&dci); - rc = cr_check_query(1, &pool.pool_uuid, &dci); - assert_rc_equal(rc, 0); + cr_dci_fini(&dci); + rc = cr_check_query(1, &pool.pool_uuid, &dci); + if (rc == 0) + break; + + assert_rc_equal(rc, -DER_INVAL); + } rc = cr_ins_verify(&dci, TCIS_PAUSED); assert_rc_equal(rc, 0); diff --git a/src/tests/suite/daos_obj.c b/src/tests/suite/daos_obj.c index 175e4d5cc65..960c5b0a02e 100644 --- a/src/tests/suite/daos_obj.c +++ b/src/tests/suite/daos_obj.c @@ -2431,7 +2431,7 @@ fetch_size(void **state) char *akey[NUM_AKEYS]; const char *akey_fmt = "akey%d"; int i, rc; - daos_size_t size = 131071; + daos_size_t size = 131071, tmp_sz; /** open object */ oid = daos_test_oid_gen(arg->coh, dts_obj_class, 0, 0, arg->myrank); @@ -2480,6 +2480,17 @@ fetch_size(void **state) for (i = 0; i < NUM_AKEYS; i++) assert_int_equal(iod[i].iod_size, size * (i+1)); + print_message("fetch with invalid sgl - NULL sg_iovs with non-zero sg_nr\n"); + sgl->sg_iovs = NULL; + tmp_sz = iod->iod_size; + iod->iod_size = 0; + rc = daos_obj_fetch(oh, DAOS_TX_NONE, 0, &dkey, NUM_AKEYS, iod, sgl, + NULL, NULL); + assert_rc_equal(rc, -DER_INVAL); + + iod->iod_size = tmp_sz; + for (i = 0; i < NUM_AKEYS; i++) + sgl[i].sg_iovs = &sg_iov[i]; print_message("fetch with unknown iod_size and less buffer\n"); for (i = 0; i < NUM_AKEYS; i++) { d_iov_set(&sg_iov[i], buf[i], size * (i+1) - 1); diff --git a/src/tests/suite/dfs_unit_test.c b/src/tests/suite/dfs_unit_test.c index b061b630a2e..e5362c57f22 100644 --- a/src/tests/suite/dfs_unit_test.c +++ b/src/tests/suite/dfs_unit_test.c @@ -2449,10 +2449,10 @@ dfs_test_xattrs(void **state) 0, 0, NULL, &obj); assert_int_equal(rc, 0); + size = 0; rc = dfs_getxattr(dfs_mt, obj, xname1, NULL, &size); assert_int_equal(rc, ENODATA); - size = 0; rc = dfs_setxattr(dfs_mt, obj, xname1, NULL, size, 0); assert_int_equal(rc, 0); @@ -2461,6 +2461,7 @@ dfs_test_xattrs(void **state) assert_int_equal(rc, 0); assert_int_equal(size, 0); + size = 0; rc = dfs_getxattr(dfs_mt, obj, xname2, NULL, &size); assert_int_equal(rc, ENODATA); diff --git a/src/utils/ddb/ddb.c b/src/utils/ddb/ddb.c index 8acb0bd60af..eeeaa293e35 100644 --- a/src/utils/ddb/ddb.c +++ b/src/utils/ddb/ddb.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -772,12 +772,14 @@ ddb_run_cmd(struct ddb_ctx *ctx, const char *cmd_str, bool write_mode) cmd_copy[strlen(cmd_copy) - 1] = '\0'; rc = ddb_str2argv_create(cmd_copy, &parse_args); - if (!SUCCESS(rc)) - D_GOTO(done, rc); + if (!SUCCESS(rc)) { + D_FREE(cmd_copy); + return rc; + } if (parse_args.ap_argc == 0) { D_ERROR("Nothing parsed\n"); - return -DER_INVAL; + D_GOTO(done, rc = -DER_INVAL); } rc = ddb_parse_cmd_args(ctx, parse_args.ap_argc, parse_args.ap_argv, &info); diff --git a/src/utils/ddb/ddb_parse.c b/src/utils/ddb/ddb_parse.c index db4f42ae660..2e55615d016 100644 --- a/src/utils/ddb/ddb_parse.c +++ b/src/utils/ddb/ddb_parse.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2019-2022 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -10,6 +10,14 @@ #include "ddb_common.h" #include "ddb_parse.h" +void +safe_strcat(char *dst, const char *src, size_t dst_size) +{ + size_t remaining_space = dst_size - strlen(dst) - 1; // Subtract 1 for null terminator + + strncat(dst, src, remaining_space); +} + int vos_path_parse(const char *path, struct vos_file_parts *vos_file_parts) { @@ -29,8 +37,8 @@ vos_path_parse(const char *path, struct vos_file_parts *vos_file_parts) while (tok != NULL && rc != 0) { rc = uuid_parse(tok, vos_file_parts->vf_pool_uuid); if (!SUCCESS(rc)) { - strcat(vos_file_parts->vf_db_path, "/"); - strcat(vos_file_parts->vf_db_path, tok); + safe_strcat(vos_file_parts->vf_db_path, "/", DB_PATH_LEN); + safe_strcat(vos_file_parts->vf_db_path, tok, DB_PATH_LEN); } tok = strtok(NULL, "/"); } @@ -83,14 +91,16 @@ ddb_str2argv_create(const char *buf, struct argv_parsed *parse_args) parse_args->ap_argv = we->we_wordv; parse_args->ap_ctx = we; - return rc; + return 0; } void ddb_str2argv_free(struct argv_parsed *parse_args) { - wordfree(parse_args->ap_ctx); - D_FREE(parse_args->ap_ctx); + if (parse_args->ap_ctx != NULL) { + wordfree(parse_args->ap_ctx); + D_FREE(parse_args->ap_ctx); + } } int diff --git a/src/utils/ddb/ddb_parse.h b/src/utils/ddb/ddb_parse.h index 376df83387b..fcd28754445 100644 --- a/src/utils/ddb/ddb_parse.h +++ b/src/utils/ddb/ddb_parse.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2019-2022 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -21,9 +21,9 @@ struct program_args { bool pa_write_mode; bool pa_get_help; }; - +#define DB_PATH_LEN 64 struct vos_file_parts { - char vf_db_path[64]; + char vf_db_path[DB_PATH_LEN]; uuid_t vf_pool_uuid; char vf_vos_file[16]; uint32_t vf_target_idx; diff --git a/src/utils/ddb/ddb_tree_path.c b/src/utils/ddb/ddb_tree_path.c index c564b62bb7e..e57bff9b245 100644 --- a/src/utils/ddb/ddb_tree_path.c +++ b/src/utils/ddb/ddb_tree_path.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2023 Intel Corporation. + * (C) Copyright 2023-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -187,6 +187,8 @@ parse_key(const char *key_str, struct dv_indexed_tree_path *itp, enum path_parts if (strlen(key_str) == 0) return 0; + D_ASSERT(key_part < PATH_PART_END); + /* is an index */ if (key_str[0] == '[') { uint32_t idx; @@ -512,12 +514,14 @@ itp_unset_cont(struct dv_indexed_tree_path *itp) int itp_idx(struct dv_indexed_tree_path *itp, enum path_parts part_key) { + D_ASSERT(part_key < PATH_PART_END); return itp->itp_parts[part_key].itp_part_idx; } bool itp_has_complete(struct dv_indexed_tree_path *itp, enum path_parts part_key) { + D_ASSERT(part_key < PATH_PART_END); return itp->itp_parts[part_key].itp_has_part_value && itp->itp_parts[part_key].itp_has_part_idx; } @@ -525,6 +529,7 @@ itp_has_complete(struct dv_indexed_tree_path *itp, enum path_parts part_key) bool itp_has(struct dv_indexed_tree_path *itp, enum path_parts part_key) { + D_ASSERT(part_key < PATH_PART_END); return itp->itp_parts[part_key].itp_has_part_value || itp->itp_parts[part_key].itp_has_part_idx; } @@ -539,12 +544,14 @@ itp_has_value(struct dv_indexed_tree_path *itp) bool itp_has_idx(struct dv_indexed_tree_path *itp, enum path_parts part_key) { + D_ASSERT(part_key < PATH_PART_END); return itp->itp_parts[part_key].itp_has_part_idx; } bool itp_has_part_value(struct dv_indexed_tree_path *itp, enum path_parts part_key) { + D_ASSERT(part_key < PATH_PART_END); return itp->itp_parts[part_key].itp_has_part_value; } @@ -633,6 +640,7 @@ itp_verify(struct dv_indexed_tree_path *itp) static union itp_part_type * itp_value(struct dv_indexed_tree_path *itp, enum path_parts path_key) { + D_ASSERT(path_key < PATH_PART_END); return &itp->itp_parts[path_key].itp_part_value; } diff --git a/src/utils/ddb/ddb_vos.c b/src/utils/ddb/ddb_vos.c index 520ca4d561f..b331e830fdd 100644 --- a/src/utils/ddb/ddb_vos.c +++ b/src/utils/ddb/ddb_vos.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -379,7 +379,7 @@ vos_iterator_type_to_path_part(vos_iter_type_t type) static enum path_parts vos_enum_to_path_part(vos_iter_type_t t) { - enum path_parts map[VOS_ITER_LARGEST]; + enum path_parts map[VOS_ITER_LARGEST] = {0}; map[VOS_ITER_OBJ] = PATH_PART_OBJ; map[VOS_ITER_DKEY] = PATH_PART_DKEY; @@ -393,7 +393,7 @@ vos_enum_to_path_part(vos_iter_type_t t) static enum path_parts vos_enum_to_parent_path_part(vos_iter_type_t t) { - int map[VOS_ITER_LARGEST]; + int map[VOS_ITER_LARGEST] = {0}; map[VOS_ITER_OBJ] = PATH_PART_CONT; map[VOS_ITER_DKEY] = PATH_PART_OBJ; @@ -1741,11 +1741,9 @@ sync_cb(struct ddbs_sync_info *info, void *cb_args) /* Try to delete the target first */ rc = smd_pool_del_tgt(pool_id, info->dsi_hdr->bbh_vos_id, st); - if (!SUCCESS(rc)) { + if (!SUCCESS(rc)) /* Ignore error for now ... might not exist*/ - D_WARN("delete target failed: "DF_RC"\n", DP_RC(rc)); - rc = 0; - } + D_WARN("delete target failed: " DF_RC "\n", DP_RC(rc)); rc = smd_pool_add_tgt(pool_id, info->dsi_hdr->bbh_vos_id, info->dsi_hdr->bbh_blob_id, st, blob_size); diff --git a/src/utils/ddb/tests/ddb_commands_tests.c b/src/utils/ddb/tests/ddb_commands_tests.c index bf723eeb2e4..f6e5a4a89e7 100644 --- a/src/utils/ddb/tests/ddb_commands_tests.c +++ b/src/utils/ddb/tests/ddb_commands_tests.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -389,8 +389,11 @@ dcv_suit_teardown(void **state) { struct dt_vos_pool_ctx *tctx = *state; - if (tctx == NULL) + if (tctx == NULL) { fail_msg("Test not setup correctly"); + return -DER_UNKNOWN; + } + assert_success(dv_pool_close(tctx->dvt_poh)); ddb_teardown_vos(state); diff --git a/src/utils/ddb/tests/ddb_main_tests.c b/src/utils/ddb/tests/ddb_main_tests.c index cc3830de184..b60b579674d 100644 --- a/src/utils/ddb/tests/ddb_main_tests.c +++ b/src/utils/ddb/tests/ddb_main_tests.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -251,8 +251,11 @@ ddb_main_suit_teardown(void **state) { struct dt_vos_pool_ctx *tctx = *state; - if (tctx == NULL) + if (tctx == NULL) { fail_msg("Test not setup correctly"); + return -DER_UNKNOWN; + } + assert_success(dv_pool_close(tctx->dvt_poh)); ddb_teardown_vos(state); diff --git a/src/utils/ddb/tests/ddb_parse_tests.c b/src/utils/ddb/tests/ddb_parse_tests.c index ea6c0749252..46da5d623a3 100644 --- a/src/utils/ddb/tests/ddb_parse_tests.c +++ b/src/utils/ddb/tests/ddb_parse_tests.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022-2023 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -40,7 +40,6 @@ assert_parsed_fail(const char *str) int rc; rc = ddb_str2argv_create(str, &parse_args); - ddb_str2argv_free(&parse_args); assert_rc_equal(-DER_INVAL, rc); } diff --git a/src/utils/ddb/tests/ddb_test_driver.c b/src/utils/ddb/tests/ddb_test_driver.c index cf127520cfd..e88e045120f 100644 --- a/src/utils/ddb/tests/ddb_test_driver.c +++ b/src/utils/ddb/tests/ddb_test_driver.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2022 Intel Corporation. + * (C) Copyright 2022-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -208,20 +208,20 @@ int ddb_test_pool_setup(struct dt_vos_pool_ctx *tctx) { int rc; - uint64_t size = (1ULL << 30); - struct stat st = {0}; + uint64_t size = (1ULL << 30); char *pool_uuid = "12345678-1234-1234-1234-123456789012"; + int mkdir_result; if (strlen(tctx->dvt_pmem_file) == 0) { char dir[64] = {0}; sprintf(dir, "/mnt/daos/%s", pool_uuid); - if (stat(dir, &st) == -1) { - if (!SUCCESS(mkdir(dir, 0700))) { - rc = daos_errno2der(errno); - return rc; - } + mkdir_result = mkdir(dir, 0700); + if (mkdir_result == -1 && errno != EEXIST) { + rc = daos_errno2der(errno); + return rc; } + snprintf(tctx->dvt_pmem_file, ARRAY_SIZE(tctx->dvt_pmem_file), "%s/ddb_vos_test", dir); } @@ -316,6 +316,11 @@ ddb_teardown_vos(void **state) { struct dt_vos_pool_ctx *tctx = *state; + if (tctx == NULL) { + fail_msg("Test context not setup correctly"); + return -DER_UNKNOWN; + } + vos_self_init("/mnt/daos", false, 0); assert_success(vos_pool_destroy(tctx->dvt_pmem_file, tctx->dvt_pool_uuid)); vos_self_fini(); diff --git a/src/vos/tests/vts_aggregate.c b/src/vos/tests/vts_aggregate.c index 42c6764bb40..1b33fe171e0 100644 --- a/src/vos/tests/vts_aggregate.c +++ b/src/vos/tests/vts_aggregate.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -212,7 +212,7 @@ lookup_object(struct io_test_args *arg, daos_unit_oid_t oid) vos_hdl2cont(arg->ctx.tc_co_hdl), oid, &epr, 0, VOS_OBJ_VISIBLE, DAOS_INTENT_DEFAULT, &obj, 0); if (rc == 0) - vos_obj_release(vos_obj_cache_current(true), obj, false); + vos_obj_release(vos_obj_cache_current(true), obj, 0, false); return rc; } diff --git a/src/vos/tests/vts_io.c b/src/vos/tests/vts_io.c index b3d11d1c99c..4f8536981d5 100644 --- a/src/vos/tests/vts_io.c +++ b/src/vos/tests/vts_io.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -997,11 +997,18 @@ io_obj_cache_test(void **state) &objs[0], 0); assert_rc_equal(rc, 0); - rc = vos_obj_discard_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &obj1); + /** Hold object for discard */ + rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_DISCARD, + DAOS_INTENT_DISCARD, &obj1, 0); assert_rc_equal(rc, 0); - /** Should be prevented because object already held for discard */ - rc = vos_obj_discard_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &obj2); - assert_rc_equal(rc, -DER_UPDATE_AGAIN); + /** Second discard should fail */ + rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_DISCARD, + DAOS_INTENT_DISCARD, &obj2, 0); + assert_rc_equal(rc, -DER_BUSY); + /** Should prevent simultaneous aggregation */ + rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_AGGREGATE, + DAOS_INTENT_PURGE, &obj2, 0); + assert_rc_equal(rc, -DER_BUSY); /** Should prevent simultaneous hold for create as well */ rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_DEFAULT, @@ -1011,17 +1018,43 @@ io_obj_cache_test(void **state) /** Need to be able to hold for read though or iteration won't work */ rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_VISIBLE, DAOS_INTENT_DEFAULT, &obj2, 0); - vos_obj_discard_release(occ, obj2); - vos_obj_discard_release(occ, obj1); + vos_obj_release(occ, obj2, 0, false); + vos_obj_release(occ, obj1, VOS_OBJ_DISCARD, false); + + /** Hold object for aggregation */ + rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_AGGREGATE, + DAOS_INTENT_PURGE, &obj1, 0); + assert_rc_equal(rc, 0); + /** Discard should fail */ + rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_DISCARD, + DAOS_INTENT_DISCARD, &obj2, 0); + assert_rc_equal(rc, -DER_BUSY); + /** Second aggregation should fail */ + rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_AGGREGATE, + DAOS_INTENT_PURGE, &obj2, 0); + assert_rc_equal(rc, -DER_BUSY); + /** Simultaneous create should work */ + rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, + VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_DEFAULT, &obj2, 0); + assert_rc_equal(rc, 0); + vos_obj_release(occ, obj2, 0, false); + + /** Need to be able to hold for read though or iteration won't work */ + rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_VISIBLE, + DAOS_INTENT_DEFAULT, &obj2, 0); + vos_obj_release(occ, obj2, 0, false); + vos_obj_release(occ, obj1, VOS_OBJ_AGGREGATE, false); + /** Now that other one is done, this should work */ - rc = vos_obj_discard_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &obj2); + rc = vos_obj_hold(occ, vos_hdl2cont(ctx->tc_co_hdl), oids[0], &epr, 0, VOS_OBJ_DISCARD, + DAOS_INTENT_DISCARD, &obj2, 0); assert_rc_equal(rc, 0); - vos_obj_discard_release(occ, obj2); + vos_obj_release(occ, obj2, VOS_OBJ_DISCARD, false); rc = umem_tx_end(ummg, 0); assert_rc_equal(rc, 0); - vos_obj_release(occ, objs[0], false); + vos_obj_release(occ, objs[0], 0, false); rc = umem_tx_begin(umml, NULL); assert_rc_equal(rc, 0); @@ -1030,7 +1063,7 @@ io_obj_cache_test(void **state) VOS_OBJ_CREATE | VOS_OBJ_VISIBLE, DAOS_INTENT_DEFAULT, &objs[0], 0); assert_rc_equal(rc, 0); - vos_obj_release(occ, objs[0], false); + vos_obj_release(occ, objs[0], 0, false); rc = umem_tx_end(umml, 0); assert_rc_equal(rc, 0); @@ -1048,20 +1081,20 @@ io_obj_cache_test(void **state) VOS_OBJ_VISIBLE, DAOS_INTENT_DEFAULT, &objs[16], 0); assert_rc_equal(rc, 0); - vos_obj_release(occ, objs[16], false); + vos_obj_release(occ, objs[16], 0, false); for (i = 0; i < 5; i++) - vos_obj_release(occ, objs[i], false); + vos_obj_release(occ, objs[i], 0, false); for (i = 10; i < 15; i++) - vos_obj_release(occ, objs[i], false); + vos_obj_release(occ, objs[i], 0, false); rc = hold_objects(objs, occ, &l_coh, &oids[1], 15, 20, true, 0); assert_int_equal(rc, 0); for (i = 5; i < 10; i++) - vos_obj_release(occ, objs[i], false); + vos_obj_release(occ, objs[i], 0, false); for (i = 15; i < 20; i++) - vos_obj_release(occ, objs[i], false); + vos_obj_release(occ, objs[i], 0, false); rc = vos_cont_close(l_coh); assert_rc_equal(rc, 0); diff --git a/src/vos/vos_aggregate.c b/src/vos/vos_aggregate.c index b7bde4e9b39..80f6bc090b4 100644 --- a/src/vos/vos_aggregate.c +++ b/src/vos/vos_aggregate.c @@ -1,10 +1,10 @@ /** - * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ /** - * Implementation for aggregation and discard + * Implementation for aggregation and discard */ #define D_LOGFAC DD_FAC(vos) @@ -2483,6 +2483,10 @@ aggregate_enter(struct vos_container *cont, int agg_mode, daos_epoch_range_t *ep struct vos_agg_metrics *vam = agg_cont2metrics(cont); int rc; + /** TODO: Now that we have per object mutual exclusion, perhaps we can + * remove the top level mutual exclusion. Keep it for now to avoid too + * much change at once. + */ switch (agg_mode) { default: D_ASSERT(0); @@ -2496,7 +2500,7 @@ aggregate_enter(struct vos_container *cont, int agg_mode, daos_epoch_range_t *ep } if (cont->vc_obj_discard_count != 0) { - D_ERROR(DF_CONT": In object discard epr["DF_U64", "DF_U64"]\n", + D_ERROR(DF_CONT ": In object discard epr[" DF_U64 ", " DF_U64 "]\n", DP_CONT(cont->vc_pool->vp_id, cont->vc_id), cont->vc_epr_discard.epr_lo, cont->vc_epr_discard.epr_hi); return -DER_BUSY; @@ -2523,13 +2527,6 @@ aggregate_enter(struct vos_container *cont, int agg_mode, daos_epoch_range_t *ep return -DER_BUSY; } - if (cont->vc_obj_discard_count != 0) { - D_ERROR(DF_CONT": In object discard epr["DF_U64", "DF_U64"]\n", - DP_CONT(cont->vc_pool->vp_id, cont->vc_id), - cont->vc_epr_discard.epr_lo, cont->vc_epr_discard.epr_hi); - return -DER_BUSY; - } - if (cont->vc_in_discard && cont->vc_epr_discard.epr_lo <= epr->epr_hi) { D_ERROR(DF_CONT": Discard epr["DF_U64", "DF_U64"], " @@ -2549,21 +2546,18 @@ aggregate_enter(struct vos_container *cont, int agg_mode, daos_epoch_range_t *ep break; case AGG_MODE_OBJ_DISCARD: + /** Theoretically, this could overlap with vos_discard as well + * as aggregation but it makes the logic in vos_obj_hold more + * complicated so defer for now and just disallow it. We can + * conflict with aggregation, however without issues. + */ if (cont->vc_in_discard) { - D_ERROR(DF_CONT": In discard epr["DF_U64", "DF_U64"]\n", + D_ERROR(DF_CONT ": Already in discard epr[" DF_U64 ", " DF_U64 "]\n", DP_CONT(cont->vc_pool->vp_id, cont->vc_id), cont->vc_epr_discard.epr_lo, cont->vc_epr_discard.epr_hi); return -DER_BUSY; } - if (cont->vc_in_aggregation) { - D_DEBUG(DB_EPC, DF_CONT": In aggregation epr["DF_U64", "DF_U64"]\n", - DP_CONT(cont->vc_pool->vp_id, cont->vc_id), - cont->vc_epr_aggregation.epr_lo, cont->vc_epr_aggregation.epr_hi); - return -DER_BUSY; - } - - /** Allow discard from multiple objects */ cont->vc_obj_discard_count++; break; } @@ -2642,12 +2636,14 @@ vos_aggregate(daos_handle_t coh, daos_epoch_range_t *epr, int (*yield_func)(void *arg), void *yield_arg, uint32_t flags) { struct vos_container *cont = vos_hdl2cont(coh); + struct vos_agg_metrics *vam = agg_cont2metrics(cont); struct agg_data *ad; uint64_t feats; daos_epoch_t agg_write; bool has_agg_write; int rc; bool run_agg = false; + int blocks = 0; D_DEBUG(DB_TRACE, "epr: %lu -> %lu\n", epr->epr_lo, epr->epr_hi); D_ASSERT(epr != NULL); @@ -2702,11 +2698,25 @@ vos_aggregate(daos_handle_t coh, daos_epoch_range_t *epr, merge_window_init(&ad->ad_agg_param.ap_window); ad->ad_agg_param.ap_flags = flags; - ad->ad_iter_param.ip_flags |= VOS_IT_FOR_PURGE; + ad->ad_iter_param.ip_flags |= VOS_IT_FOR_PURGE | VOS_IT_FOR_AGG; +retry: rc = vos_iterate(&ad->ad_iter_param, VOS_ITER_OBJ, true, &ad->ad_anchors, vos_aggregate_pre_cb, vos_aggregate_post_cb, &ad->ad_agg_param, NULL); - if (rc != 0 || ad->ad_agg_param.ap_nospc_err) { + if (rc == -DER_BUSY) { + /** Hit a conflict with obj_discard. Rather than exiting, let's + * yield and try again. + */ + if (vam && vam->vam_agg_blocked) + d_tm_inc_counter(vam->vam_agg_blocked, 1); + blocks++; + /** Warn once if it goes over 20 times */ + D_CDEBUG(blocks == 20, DLOG_WARN, DB_EPC, + "VOS aggrregation hit conflict (nr=%d), retrying...\n", blocks); + close_merge_window(&ad->ad_agg_param.ap_window, rc); + vos_aggregate_yield(&ad->ad_agg_param); + goto retry; + } else if (rc != 0 || ad->ad_agg_param.ap_nospc_err) { close_merge_window(&ad->ad_agg_param.ap_window, rc); goto exit; } else if (ad->ad_agg_param.ap_csum_err) { @@ -2741,8 +2751,6 @@ vos_aggregate(daos_handle_t coh, daos_epoch_range_t *epr, D_FREE(ad); if (rc < 0) { - struct vos_agg_metrics *vam = agg_cont2metrics(cont); - if (vam && vam->vam_fail_count) d_tm_inc_counter(vam->vam_fail_count, 1); } @@ -2754,12 +2762,13 @@ int vos_discard(daos_handle_t coh, daos_unit_oid_t *oidp, daos_epoch_range_t *epr, int (*yield_func)(void *arg), void *yield_arg) { - struct vos_container *cont = vos_hdl2cont(coh); - struct vos_object *obj; + struct vos_container *cont = vos_hdl2cont(coh); + struct vos_agg_metrics *vam = agg_cont2metrics(cont); struct agg_data *ad; int type = VOS_ITER_OBJ; int rc; int mode = oidp == NULL ? AGG_MODE_DISCARD : AGG_MODE_OBJ_DISCARD; + int blocks = 0; D_ASSERT(epr != NULL); D_ASSERTF(epr->epr_lo <= epr->epr_hi, @@ -2770,22 +2779,9 @@ vos_discard(daos_handle_t coh, daos_unit_oid_t *oidp, daos_epoch_range_t *epr, if (ad == NULL) return -DER_NOMEM; - if (oidp != NULL) { - rc = vos_obj_discard_hold(vos_obj_cache_current(cont->vc_pool->vp_sysdb), - cont, *oidp, &obj); - if (rc != 0) { - if (rc == -DER_NONEXIST) - rc = 0; - goto free_agg_data; - } - - D_ASSERT(obj != NULL); - D_ASSERT(obj->obj_discard); - } - rc = aggregate_enter(cont, mode, epr); if (rc != 0) - goto release_obj; + goto exit; if (oidp != NULL) { D_DEBUG(DB_EPC, "Discard "DF_UOID" epr "DF_X64"-"DF_X64"\n", DP_UOID(*oidp), @@ -2820,17 +2816,26 @@ vos_discard(daos_handle_t coh, daos_unit_oid_t *oidp, daos_epoch_range_t *epr, ad->ad_agg_param.ap_yield_arg = yield_arg; ad->ad_iter_param.ip_flags |= VOS_IT_FOR_DISCARD; - rc = vos_iterate(&ad->ad_iter_param, type, true, &ad->ad_anchors, - vos_aggregate_pre_cb, vos_aggregate_post_cb, - &ad->ad_agg_param, NULL); +retry: + rc = vos_iterate(&ad->ad_iter_param, type, true, &ad->ad_anchors, vos_aggregate_pre_cb, + vos_aggregate_post_cb, &ad->ad_agg_param, NULL); + if (rc == -DER_BUSY) { + /** Hit an object conflict with EC aggregation. Rather than exiting, let's + * yield and try again. + */ + blocks++; + /** Warn once if it goes over 20 times */ + D_CDEBUG(blocks == 20, DLOG_WARN, DB_EPC, + "VOS discard hit conflict (nr=%d), retrying...\n", blocks); + if (vam && vam->vam_discard_blocked) + d_tm_inc_counter(vam->vam_discard_blocked, 1); + vos_aggregate_yield(&ad->ad_agg_param); + goto retry; + } aggregate_exit(cont, mode); -release_obj: - if (oidp != NULL) - vos_obj_discard_release(vos_obj_cache_current(cont->vc_pool->vp_sysdb), obj); - -free_agg_data: +exit: D_FREE(ad); return rc; diff --git a/src/vos/vos_common.c b/src/vos/vos_common.c index 4f3ce42e08a..28e2ac86757 100644 --- a/src/vos/vos_common.c +++ b/src/vos/vos_common.c @@ -822,6 +822,19 @@ vos_metrics_alloc(const char *path, int tgt_id) if (rc) D_WARN("Failed to create 'merged_size' telemetry : "DF_RC"\n", DP_RC(rc)); + /* VOS aggregation conflicts with discard */ + rc = d_tm_add_metric(&vam->vam_agg_blocked, D_TM_COUNTER, "aggregation blocked by discard", + NULL, "%s/%s/agg_blocked/tgt_%u", path, VOS_AGG_DIR, tgt_id); + if (rc) + D_WARN("Failed to create 'agg_blocked' telemetry : " DF_RC "\n", DP_RC(rc)); + + /* VOS discard conflicts with aggregation */ + rc = d_tm_add_metric(&vam->vam_discard_blocked, D_TM_COUNTER, + "discard blocked by aggregation", NULL, "%s/%s/discard_blocked/tgt_%u", + path, VOS_AGG_DIR, tgt_id); + if (rc) + D_WARN("Failed to create 'discard_blocked' telemetry : " DF_RC "\n", DP_RC(rc)); + /* VOS aggregation failed */ rc = d_tm_add_metric(&vam->vam_fail_count, D_TM_COUNTER, "aggregation failures", NULL, "%s/%s/fail_count/tgt_%u", path, VOS_AGG_DIR, tgt_id); diff --git a/src/vos/vos_dtx.c b/src/vos/vos_dtx.c index a982c460f99..1851331dd6a 100644 --- a/src/vos/vos_dtx.c +++ b/src/vos/vos_dtx.c @@ -877,7 +877,8 @@ vos_dtx_commit_one(struct vos_container *cont, struct dtx_id *dti, daos_epoch_t if (rc != 0) D_FREE(dce); - if (rm_cos != NULL && (rc == 0 || rc == -DER_NONEXIST)) + if (rm_cos != NULL && + (rc == 0 || rc == -DER_NONEXIST || (rc == -DER_ALREADY && dae == NULL))) *rm_cos = true; return rc; @@ -2717,7 +2718,7 @@ vos_dtx_mark_sync(daos_handle_t coh, daos_unit_oid_t oid, daos_epoch_t epoch) sizeof(obj->obj_df->vo_sync), UMEM_COMMIT_IMMEDIATE); } - vos_obj_release(occ, obj, false); + vos_obj_release(occ, obj, 0, false); return 0; } diff --git a/src/vos/vos_internal.h b/src/vos/vos_internal.h index a813bc697de..c761543bd8d 100644 --- a/src/vos/vos_internal.h +++ b/src/vos/vos_internal.h @@ -28,40 +28,42 @@ #define VOS_MINOR_EPC_MAX EVT_MINOR_EPC_MAX -#define VOS_TX_LOG_FAIL(rc, ...) \ - do { \ - bool __is_err = true; \ - \ - if (rc >= 0) \ - break; \ - switch (rc) { \ - case -DER_TX_RESTART: \ - case -DER_INPROGRESS: \ - case -DER_EXIST: \ - case -DER_NONEXIST: \ - __is_err = false; \ - break; \ - } \ - D_CDEBUG(__is_err, DLOG_ERR, DB_IO, \ - __VA_ARGS__); \ +#define VOS_TX_LOG_FAIL(rc, ...) \ + do { \ + bool __is_err = true; \ + \ + if (rc >= 0) \ + break; \ + switch (rc) { \ + case -DER_TX_RESTART: \ + case -DER_INPROGRESS: \ + case -DER_UPDATE_AGAIN: \ + case -DER_BUSY: \ + case -DER_EXIST: \ + case -DER_NONEXIST: \ + __is_err = false; \ + break; \ + } \ + D_CDEBUG(__is_err, DLOG_ERR, DB_IO, __VA_ARGS__); \ } while (0) -#define VOS_TX_TRACE_FAIL(rc, ...) \ - do { \ - bool __is_err = true; \ - \ - if (rc >= 0) \ - break; \ - switch (rc) { \ - case -DER_TX_RESTART: \ - case -DER_INPROGRESS: \ - case -DER_EXIST: \ - case -DER_NONEXIST: \ - __is_err = false; \ - break; \ - } \ - D_CDEBUG(__is_err, DLOG_ERR, DB_TRACE, \ - __VA_ARGS__); \ +#define VOS_TX_TRACE_FAIL(rc, ...) \ + do { \ + bool __is_err = true; \ + \ + if (rc >= 0) \ + break; \ + switch (rc) { \ + case -DER_TX_RESTART: \ + case -DER_INPROGRESS: \ + case -DER_UPDATE_AGAIN: \ + case -DER_BUSY: \ + case -DER_EXIST: \ + case -DER_NONEXIST: \ + __is_err = false; \ + break; \ + } \ + D_CDEBUG(__is_err, DLOG_ERR, DB_TRACE, __VA_ARGS__); \ } while (0) #define VOS_CONT_ORDER 20 /* Order of container tree */ @@ -185,6 +187,8 @@ struct vos_agg_metrics { struct d_tm_node_t *vam_merge_recs; /* Total merged EV records */ struct d_tm_node_t *vam_merge_size; /* Total merged size */ struct d_tm_node_t *vam_fail_count; /* Aggregation failed */ + struct d_tm_node_t *vam_agg_blocked; /* Aggregation waiting for discard */ + struct d_tm_node_t *vam_discard_blocked; /* Discard waiting for aggregation */ }; struct vos_gc_metrics { @@ -224,14 +228,15 @@ struct vos_space_metrics { /* VOS Pool metrics for WAL */ struct vos_wal_metrics { - struct d_tm_node_t *vwm_wal_sz; /* WAL size for single tx */ - struct d_tm_node_t *vwm_wal_qd; /* WAL transaction queue depth */ - struct d_tm_node_t *vwm_wal_waiters; /* Waiters for WAL reclaiming */ - struct d_tm_node_t *vwm_replay_size; /* WAL replay size in bytes */ - struct d_tm_node_t *vwm_replay_time; /* WAL replay time in us */ - struct d_tm_node_t *vwm_replay_count; /* Total replay count */ - struct d_tm_node_t *vwm_replay_tx; /* Total replayed TX count */ - struct d_tm_node_t *vwm_replay_ent; /* Total replayed entry count */ + struct d_tm_node_t *vwm_wal_sz; /* WAL size for single tx */ + struct d_tm_node_t *vwm_wal_qd; /* WAL transaction queue depth */ + struct d_tm_node_t *vwm_wal_waiters; /* Waiters for WAL reclaiming */ + struct d_tm_node_t *vwm_wal_dur; /* WAL commit duration */ + struct d_tm_node_t *vwm_replay_size; /* WAL replay size in bytes */ + struct d_tm_node_t *vwm_replay_time; /* WAL replay time in us */ + struct d_tm_node_t *vwm_replay_count; /* Total replay count */ + struct d_tm_node_t *vwm_replay_tx; /* Total replayed TX count */ + struct d_tm_node_t *vwm_replay_ent; /* Total replayed entry count */ }; void vos_wal_metrics_init(struct vos_wal_metrics *vw_metrics, const char *path, int tgt_id); @@ -1064,8 +1069,11 @@ struct vos_iterator { vos_iter_type_t it_type; enum vos_iter_state it_state; uint32_t it_ref_cnt; + /** Note: it_for_agg is only set at object level as it's only used for + * mutual exclusion between aggregation and object discard. + */ uint32_t it_from_parent : 1, it_for_purge : 1, it_for_discard : 1, it_for_migration : 1, - it_show_uncommitted : 1, it_ignore_uncommitted : 1, it_for_sysdb : 1; + it_show_uncommitted : 1, it_ignore_uncommitted : 1, it_for_sysdb : 1, it_for_agg : 1; }; /* Auxiliary structure for passing information between parent and nested diff --git a/src/vos/vos_io.c b/src/vos/vos_io.c index 3347f7e5701..31b6aa323d6 100644 --- a/src/vos/vos_io.c +++ b/src/vos/vos_io.c @@ -571,8 +571,8 @@ vos_ioc_destroy(struct vos_io_context *ioc, bool evict) dcs_csum_info_list_fini(&ioc->ic_csum_list); if (ioc->ic_obj) - vos_obj_release(vos_obj_cache_current(ioc->ic_cont->vc_pool->vp_sysdb), - ioc->ic_obj, evict); + vos_obj_release(vos_obj_cache_current(ioc->ic_cont->vc_pool->vp_sysdb), ioc->ic_obj, + 0, evict); vos_ioc_reserve_fini(ioc); vos_ilog_fetch_finish(&ioc->ic_dkey_info); diff --git a/src/vos/vos_iterator.c b/src/vos/vos_iterator.c index 1b30175c5d0..360664dee38 100644 --- a/src/vos/vos_iterator.c +++ b/src/vos/vos_iterator.c @@ -764,7 +764,7 @@ vos_iter_cb(vos_iter_cb_t iter_cb, daos_handle_t ih, vos_iter_entry_t *iter_ent, *acts |= VOS_ITER_CB_YIELD; if (rc == 0 && iter->it_parent != NULL && (param->ip_flags & - (VOS_IT_RECX_VISIBLE | VOS_IT_FOR_PURGE | VOS_IT_FOR_DISCARD)) == 0) { + (VOS_IT_RECX_VISIBLE | VOS_IT_FOR_AGG | VOS_IT_FOR_DISCARD)) == 0) { /** If scanning the whole tree, we need to revalidate the parent * chain and possibly jump back to another level to continue */ rc = vos_iter_validate_internal(iter->it_parent); diff --git a/src/vos/vos_obj.c b/src/vos/vos_obj.c index bd2a1f51e2b..dedda2ad805 100644 --- a/src/vos/vos_obj.c +++ b/src/vos/vos_obj.c @@ -541,8 +541,8 @@ vos_obj_punch(daos_handle_t coh, daos_unit_oid_t oid, daos_epoch_t epoch, rc = vos_mark_agg(cont, &obj->obj_df->vo_tree, &cont->vc_cont_df->cd_obj_root, epoch); - vos_obj_release(vos_obj_cache_current(cont->vc_pool->vp_sysdb), - obj, rc != 0); + vos_obj_release(vos_obj_cache_current(cont->vc_pool->vp_sysdb), obj, 0, + rc != 0); } } @@ -677,7 +677,7 @@ vos_obj_key2anchor(daos_handle_t coh, daos_unit_oid_t oid, daos_key_t *dkey, dao key_tree_release(toh, (krec->kr_bmap & KREC_BF_EVT) != 0); out: - vos_obj_release(occ, obj, false); + vos_obj_release(occ, obj, 0, false); return rc; } @@ -713,7 +713,7 @@ vos_obj_delete_internal(daos_handle_t coh, daos_unit_oid_t oid, bool only_delete rc = umem_tx_end(umm, rc); out: - vos_obj_release(occ, obj, true); + vos_obj_release(occ, obj, 0, true); return rc; } @@ -794,7 +794,7 @@ vos_obj_del_key(daos_handle_t coh, daos_unit_oid_t oid, daos_key_t *dkey, out_tx: rc = umem_tx_end(umm, rc); out: - vos_obj_release(occ, obj, true); + vos_obj_release(occ, obj, 0, true); return rc; } @@ -1716,6 +1716,8 @@ vos_obj_iter_prep(vos_iter_type_t type, vos_iter_param_t *param, oiter->it_iter.it_for_discard = 1; if (param->ip_flags & VOS_IT_FOR_MIGRATION) oiter->it_iter.it_for_migration = 1; + if (param->ip_flags & VOS_IT_FOR_AGG) + oiter->it_iter.it_for_agg = 1; if (is_sysdb) oiter->it_iter.it_for_sysdb = 1; if (param->ip_flags == VOS_IT_KEY_TREE) { @@ -1880,16 +1882,22 @@ dkey_nested_iter_init(struct vos_obj_iter *oiter, struct vos_iter_info *info) { int rc; struct vos_container *cont = vos_hdl2cont(info->ii_hdl); + uint64_t flags = 0; + + if ((oiter->it_flags & VOS_IT_PUNCHED) == 0) + flags |= VOS_OBJ_VISIBLE; + if (oiter->it_iter.it_for_agg) + flags |= VOS_OBJ_AGGREGATE; + if (oiter->it_iter.it_for_discard) + flags |= VOS_OBJ_DISCARD; /* XXX the condition epoch ranges could cover multiple versions of * the object/key if it's punched more than once. However, rebuild * system should guarantee this will never happen. */ - rc = vos_obj_hold(vos_obj_cache_current(cont->vc_pool->vp_sysdb), cont, - info->ii_oid, &info->ii_epr, oiter->it_iter.it_bound, - (oiter->it_flags & VOS_IT_PUNCHED) ? 0 : - VOS_OBJ_VISIBLE, vos_iter_intent(&oiter->it_iter), - &oiter->it_obj, NULL); + rc = vos_obj_hold(vos_obj_cache_current(cont->vc_pool->vp_sysdb), cont, info->ii_oid, + &info->ii_epr, oiter->it_iter.it_bound, flags, + vos_iter_intent(&oiter->it_iter), &oiter->it_obj, NULL); D_ASSERTF(rc != -DER_NONEXIST, "Nested iterator called without setting probe"); @@ -1916,7 +1924,8 @@ dkey_nested_iter_init(struct vos_obj_iter *oiter, struct vos_iter_info *info) return 0; failed: - vos_obj_release(vos_obj_cache_current(cont->vc_pool->vp_sysdb), oiter->it_obj, false); + vos_obj_release(vos_obj_cache_current(cont->vc_pool->vp_sysdb), oiter->it_obj, flags, + false); return rc; } @@ -2180,6 +2189,7 @@ vos_obj_iter_fini(struct vos_iterator *iter) struct vos_obj_iter *oiter = vos_iter2oiter(iter); int rc; struct vos_object *object; + uint64_t flags = 0; if (daos_handle_is_inval(oiter->it_hdl)) D_GOTO(out, rc = -DER_NO_HDL); @@ -2213,9 +2223,16 @@ vos_obj_iter_fini(struct vos_iterator *iter) */ object = oiter->it_obj; if (oiter->it_flags != VOS_IT_KEY_TREE && object != NULL && - (iter->it_type == VOS_ITER_DKEY || !iter->it_from_parent)) - vos_obj_release(vos_obj_cache_current(object->obj_cont->vc_pool->vp_sysdb), - object, false); + (iter->it_type == VOS_ITER_DKEY || !iter->it_from_parent)) { + if (iter->it_type == VOS_ITER_DKEY) { + if (iter->it_for_discard) + flags = VOS_OBJ_DISCARD; + else if (iter->it_for_agg) + flags = VOS_OBJ_AGGREGATE; + } + vos_obj_release(vos_obj_cache_current(object->obj_cont->vc_pool->vp_sysdb), object, + flags, false); + } vos_ilog_fetch_finish(&oiter->it_ilog_info); D_FREE(oiter); @@ -2635,12 +2652,15 @@ vos_obj_iter_empty(struct vos_iterator *iter) D_ASSERT(0); return -DER_INVAL; case VOS_ITER_DKEY: + /* fall through */ case VOS_ITER_AKEY: if (oiter->it_flags & VOS_IT_DKEY_EV) evt = true; + /* fall through */ case VOS_ITER_SINGLE: if (!evt) return dbtree_iter_empty(oiter->it_hdl); + /* fall through */ case VOS_ITER_RECX: return evt_iter_empty(oiter->it_hdl); } diff --git a/src/vos/vos_obj.h b/src/vos/vos_obj.h index e4a8c11fd7a..c687cd77f9c 100644 --- a/src/vos/vos_obj.h +++ b/src/vos/vos_obj.h @@ -49,19 +49,25 @@ struct vos_object { struct vos_container *obj_cont; /** nobody should access this object */ bool obj_zombie; - /** Object is in discard */ - bool obj_discard; + /** Object is held for discard */ + uint32_t obj_discard : 1, + /** If non-zero, object is held for aggregation */ + obj_aggregate : 1; }; enum { /** Only return the object if it's visible */ - VOS_OBJ_VISIBLE = (1 << 0), + VOS_OBJ_VISIBLE = (1 << 0), /** Create the object if it doesn't exist */ - VOS_OBJ_CREATE = (1 << 1), - /** Hold for object specific discard */ - VOS_OBJ_DISCARD = (1 << 2), + VOS_OBJ_CREATE = (1 << 1), + /** Hold for discard */ + VOS_OBJ_DISCARD = (1 << 2), + /** Hold for VOS or EC aggregation */ + VOS_OBJ_AGGREGATE = (1 << 3), /** Hold the object for delete dkey */ - VOS_OBJ_KILL_DKEY = (1 << 3), + VOS_OBJ_KILL_DKEY = (1 << 4), + /** Don't actually complete the hold, just check for conflicts */ + VOS_OBJ_NO_HOLD = (1 << 5), }; /** @@ -101,7 +107,7 @@ vos_obj_hold(struct daos_lru_cache *occ, struct vos_container *cont, * \param obj [IN] Reference to be released. */ void -vos_obj_release(struct daos_lru_cache *occ, struct vos_object *obj, bool evict); +vos_obj_release(struct daos_lru_cache *occ, struct vos_object *obj, uint64_t flags, bool evict); /** Evict an object reference from the cache */ void vos_obj_evict(struct daos_lru_cache *occ, struct vos_object *obj); diff --git a/src/vos/vos_obj_cache.c b/src/vos/vos_obj_cache.c index a4577799053..083ca2216a5 100644 --- a/src/vos/vos_obj_cache.c +++ b/src/vos/vos_obj_cache.c @@ -206,7 +206,7 @@ vos_obj_cache_current(bool standalone) static __thread struct vos_object obj_local = {0}; void -vos_obj_release(struct daos_lru_cache *occ, struct vos_object *obj, bool evict) +vos_obj_release(struct daos_lru_cache *occ, struct vos_object *obj, uint64_t flags, bool evict) { if (obj == &obj_local) { @@ -216,6 +216,10 @@ vos_obj_release(struct daos_lru_cache *occ, struct vos_object *obj, bool evict) } D_ASSERT((occ != NULL) && (obj != NULL)); + if (flags & VOS_OBJ_AGGREGATE) + obj->obj_aggregate = 0; + else if (flags & VOS_OBJ_DISCARD) + obj->obj_discard = 0; if (evict) daos_lru_ref_evict(occ, &obj->obj_llink); @@ -223,35 +227,6 @@ vos_obj_release(struct daos_lru_cache *occ, struct vos_object *obj, bool evict) daos_lru_ref_release(occ, &obj->obj_llink); } -int -vos_obj_discard_hold(struct daos_lru_cache *occ, struct vos_container *cont, daos_unit_oid_t oid, - struct vos_object **objp) -{ - struct vos_object *obj = NULL; - daos_epoch_range_t epr = {0, DAOS_EPOCH_MAX}; - int rc; - - rc = vos_obj_hold(occ, cont, oid, &epr, 0, VOS_OBJ_DISCARD, - DAOS_INTENT_DISCARD, &obj, NULL); - if (rc != 0) - return rc; - - D_ASSERTF(!obj->obj_discard, "vos_obj_hold should return an error if already in discard\n"); - - obj->obj_discard = true; - *objp = obj; - - return 0; -} - -void -vos_obj_discard_release(struct daos_lru_cache *occ, struct vos_object *obj) -{ - obj->obj_discard = false; - - vos_obj_release(occ, obj, false); -} - /** Move local object to the lru cache */ static inline int cache_object(struct daos_lru_cache *occ, struct vos_object **objp) @@ -296,6 +271,37 @@ cache_object(struct daos_lru_cache *occ, struct vos_object **objp) return 0; } +static bool +vos_obj_op_conflict(struct vos_object *obj, uint64_t flags, uint32_t intent, bool create) +{ + bool discard = flags & VOS_OBJ_DISCARD; + bool agg = flags & VOS_OBJ_AGGREGATE; + + /* VOS aggregation is mutually exclusive with VOS discard. + * Object discard is mutually exclusive with VOS discard. + * EC aggregation is not mutually exclusive with anything. + * For simplicity, we do make all of them mutually exclusive on the same + * object. + */ + + if (obj->obj_discard) { + /** Mutually exclusive with create, discard and aggregation */ + if (create || discard || agg) { + D_DEBUG(DB_EPC, "Conflict detected, discard already running on object\n"); + return true; + } + } else if (obj->obj_aggregate) { + /** Mutually exclusive with discard */ + if (discard || agg) { + D_DEBUG(DB_EPC, + "Conflict detected, aggregation already running on object\n"); + return true; + } + } + + return false; +} + int vos_obj_hold(struct daos_lru_cache *occ, struct vos_container *cont, daos_unit_oid_t oid, daos_epoch_range_t *epr, daos_epoch_t bound, @@ -310,12 +316,13 @@ vos_obj_hold(struct daos_lru_cache *occ, struct vos_container *cont, uint32_t cond_mask = 0; bool create; void *create_flag = NULL; - bool visible_only; + bool visible_only; D_ASSERT(cont != NULL); D_ASSERT(cont->vc_pool); - *obj_p = NULL; + if (obj_p != NULL) + *obj_p = NULL; if (cont->vc_pool->vp_dying) return -DER_SHUTDOWN; @@ -340,6 +347,10 @@ vos_obj_hold(struct daos_lru_cache *occ, struct vos_container *cont, rc = daos_lru_ref_hold(occ, &lkey, sizeof(lkey), create_flag, &lret); if (rc == -DER_NONEXIST) { D_ASSERT(obj_local.obj_cont == NULL); + if (flags & VOS_OBJ_NO_HOLD) { + /** Object is not cached, so there can be no other holders */ + return 0; + } obj = &obj_local; init_object(obj, oid, cont); } else if (rc != 0) { @@ -406,14 +417,25 @@ vos_obj_hold(struct daos_lru_cache *occ, struct vos_container *cont, } check_object: - if (obj->obj_discard && (create || (flags & VOS_OBJ_DISCARD) != 0)) { - /** Cleanup before assert so unit test that triggers doesn't corrupt the state */ - vos_obj_release(occ, obj, false); + if (vos_obj_op_conflict(obj, flags, intent, create)) { + /** Cleanup so unit test that triggers doesn't corrupt the state */ + vos_obj_release(occ, obj, 0, false); /* Update request will retry with this error */ - rc = -DER_UPDATE_AGAIN; + if (create) + rc = -DER_UPDATE_AGAIN; + else + rc = -DER_BUSY; goto failed_2; } + if (flags & VOS_OBJ_NO_HOLD) { + /** Just checking for conflicts, so we are done */ + vos_obj_release(occ, obj, 0, false); + return 0; + } + + D_ASSERT(obj_p != NULL); + if ((flags & VOS_OBJ_DISCARD) || intent == DAOS_INTENT_KILL || intent == DAOS_INTENT_PUNCH) goto out; @@ -497,13 +519,18 @@ vos_obj_hold(struct daos_lru_cache *occ, struct vos_container *cont, goto failed_2; } + if (flags & VOS_OBJ_AGGREGATE) + obj->obj_aggregate = 1; + else if (flags & VOS_OBJ_DISCARD) + obj->obj_discard = 1; *obj_p = obj; return 0; failed: - vos_obj_release(occ, obj, true); + vos_obj_release(occ, obj, 0, true); failed_2: - VOS_TX_LOG_FAIL(rc, "failed to hold object, rc="DF_RC"\n", DP_RC(rc)); + VOS_TX_LOG_FAIL(rc, "failed to hold object " DF_UOID ", rc=" DF_RC "\n", DP_UOID(oid), + DP_RC(rc)); return rc; } diff --git a/src/vos/vos_obj_index.c b/src/vos/vos_obj_index.c index d1b2a713d03..ea47cf4454c 100644 --- a/src/vos/vos_obj_index.c +++ b/src/vos/vos_obj_index.c @@ -847,11 +847,13 @@ oi_iter_aggregate(daos_handle_t ih, bool range_discard) { struct vos_iterator *iter = vos_hdl2iter(ih); struct vos_oi_iter *oiter = iter2oiter(iter); + struct vos_container *cont = oiter->oit_cont; struct vos_obj_df *obj; daos_unit_oid_t oid; d_iov_t rec_iov; bool delete = false, invisible = false; int rc; + uint64_t base_flag = range_discard ? VOS_OBJ_DISCARD : VOS_OBJ_AGGREGATE; D_ASSERT(iter->it_type == VOS_ITER_OBJ); @@ -865,6 +867,18 @@ oi_iter_aggregate(daos_handle_t ih, bool range_discard) obj = (struct vos_obj_df *)rec_iov.iov_buf; oid = obj->vo_id; + rc = vos_obj_hold(vos_obj_cache_current(cont->vc_pool->vp_sysdb), cont, oid, + &oiter->oit_epr, iter->it_bound, base_flag | VOS_OBJ_NO_HOLD, + DAOS_INTENT_PURGE, NULL, NULL); + if (rc != 0) { + /** -DER_BUSY means the object is in-use already. We will after a yield in this + * case. + */ + D_CDEBUG(rc == -DER_BUSY, DB_EPC, DLOG_ERR, "Hold check failed for " DF_UOID "\n", + DP_UOID(oid)); + return rc; + } + rc = umem_tx_begin(vos_cont2umm(oiter->oit_cont), NULL); if (rc != 0) goto exit; diff --git a/src/vos/vos_pool.c b/src/vos/vos_pool.c index 542f760e06d..10b9d6ce763 100644 --- a/src/vos/vos_pool.c +++ b/src/vos/vos_pool.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -386,6 +386,11 @@ vos_wal_metrics_init(struct vos_wal_metrics *vw_metrics, const char *path, int t if (rc) D_WARN("Failed to create WAL waiters telemetry: "DF_RC"\n", DP_RC(rc)); + rc = d_tm_add_metric(&vw_metrics->vwm_wal_dur, D_TM_DURATION, "WAL commit duration", NULL, + "%s/%s/wal_dur/tgt_%d", path, VOS_WAL_DIR, tgt_id); + if (rc) + D_WARN("Failed to create WAL commit duration telemetry: " DF_RC "\n", DP_RC(rc)); + /* Initialize metrics for WAL replay */ rc = d_tm_add_metric(&vw_metrics->vwm_replay_count, D_TM_COUNTER, "Number of WAL replays", NULL, "%s/%s/replay_count/tgt_%u", path, VOS_WAL_DIR, tgt_id); @@ -451,15 +456,19 @@ vos_wal_reserve(struct umem_store *store, uint64_t *tx_id) static inline int vos_wal_commit(struct umem_store *store, struct umem_wal_tx *wal_tx, void *data_iod) { - struct bio_wal_info wal_info; - struct vos_pool *pool; - struct bio_wal_stats ws = { 0 }; - struct vos_wal_metrics *vwm; - int rc; + struct bio_wal_info wal_info; + struct vos_pool *pool; + struct bio_wal_stats ws = {0}; + struct vos_wal_metrics *vwm; + int rc; D_ASSERT(store && store->stor_priv != NULL); vwm = (struct vos_wal_metrics *)store->stor_stats; + if (vwm != NULL) + d_tm_mark_duration_start(vwm->vwm_wal_dur, D_TM_CLOCK_REALTIME); rc = bio_wal_commit(store->stor_priv, wal_tx, data_iod, (vwm != NULL) ? &ws : NULL); + if (vwm != NULL) + d_tm_mark_duration_end(vwm->vwm_wal_dur); if (rc) { DL_ERROR(rc, "WAL commit failed."); /* diff --git a/src/vos/vos_query.c b/src/vos/vos_query.c index d05ffba40bd..3c06bb69de1 100644 --- a/src/vos/vos_query.c +++ b/src/vos/vos_query.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -794,7 +794,7 @@ vos_obj_query_key(daos_handle_t coh, daos_unit_oid_t oid, uint32_t flags, *max_write = obj->obj_df->vo_max_write; if (obj != NULL) - vos_obj_release(vos_obj_cache_current(is_sysdb), obj, false); + vos_obj_release(vos_obj_cache_current(is_sysdb), obj, 0, false); if (rc == 0 || rc == -DER_NONEXIST) { if (vos_ts_wcheck(query->qt_ts_set, obj_epr.epr_hi, diff --git a/utils/config/examples/daos_server_local.yml b/utils/config/examples/daos_server_local.yml index f143f3fc223..814ac659824 100644 --- a/utils/config/examples/daos_server_local.yml +++ b/utils/config/examples/daos_server_local.yml @@ -25,13 +25,13 @@ engines: # Storage definitions (one per tier) storage: - - # When scm_class is set to ram, tmpfs will be used to emulate SCM. + # When class is set to ram, tmpfs will be used to emulate SCM. # The size of ram is specified by scm_size in GB units. class: ram scm_size: 8 scm_mount: /mnt/daos - - # When bdev_class is set to file, Linux AIO will be used to emulate NVMe. + # When class is set to file, Linux AIO will be used to emulate NVMe. # The size of file that will be created is specified by bdev_size in GB units. # The location of the files that will be created is specified in bdev_list. class: file diff --git a/utils/config/examples/daos_server_unittests.yml b/utils/config/examples/daos_server_unittests.yml deleted file mode 100644 index 3c7d3353328..00000000000 --- a/utils/config/examples/daos_server_unittests.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Example configuration file using loopback and emulated storage - -name: daos_server # sys group daos_server -access_points: ['example'] # management service leader (bootstrap) -# port: 10001 # control listen port, default 10001 -provider: ofi+tcp -control_log_mask: DEBUG -control_log_file: /tmp/daos_server.log -telemetry_port: 9191 - -## Transport Credentials Specifying certificates to secure communications -## -#transport_config: -# # Specify to bypass loading certificates and use insecure communications channels -# allow_insecure: false -# # Location where daos_server will look for Client certificates -# client_cert_dir: .daos/clients -# client_cert_dir: /etc/daos/certs/clients -# # Custom CA Root certificate for generated certs -# ca_cert: /etc/daos/certs/daosCA.crt -# # Server certificate for use in TLS handshakes -# cert: /etc/daos/certs/server.crt -# # Key portion of Server Certificate -# key: /etc/daos/certs/server.key - -engines: - - - pinned_numa_node: 0 - targets: 1 # number of I/O service threads per-engine - nr_xs_helpers: 0 # count of I/O offload threads per engine - fabric_iface: lo # map to OFI_INTERFACE=lo - fabric_iface_port: 31316 # map to OFI_PORT=31316 - log_mask: DEBUG,RPC=ERR,MEM=ERR - log_file: /tmp/daos_engine.0.log # map to D_LOG_FILE=/tmp/daos_engine.0.log - - # Environment variable values should be supplied without encapsulating quotes. - env_vars: # influence DAOS I/O Engine behavior by setting env variables - - DAOS_MD_CAP=1024 - - CRT_TIMEOUT=30 - - FI_SOCKETS_MAX_CONN_RETRY=1 - - FI_SOCKETS_CONN_TIMEOUT=2000 - - DD_SUBSYS=all - # uncomment to enable scalable endpoint - # - CRT_CREDIT_EP_CTX=0 - # - CRT_CTX_NUM=8 - - # Legacy style storage definitions (support for this style will be removed in future) - - # When scm_class is set to ram, tmpfs will be used to emulate SCM. - # The size of ram is specified by scm_size in GB units. - scm_class: ram - scm_size: 8 - scm_mount: /mnt/daos - - # If no NVMe devices are specified in config file, no NVMe will be used. - bdev_class: nvme - bdev_list: [] - bdev_busid_range: "" diff --git a/utils/cq/daos_pylint.py b/utils/cq/daos_pylint.py index 13780d594d4..6f924f950cb 100755 --- a/utils/cq/daos_pylint.py +++ b/utils/cq/daos_pylint.py @@ -88,8 +88,8 @@ def _read_files(self, infile, outfile): new_lineno = 1 scons_header = False - def _remap_count(): - for iline in range(new_lineno, new_lineno + added): + def _remap_count(_added): + for iline in range(new_lineno, new_lineno + _added): self.line_map[iline] = old_lineno - 1 for line in infile.readlines(): @@ -108,7 +108,7 @@ def _remap_count(): newvar = var.strip("\", '") variables.append(newvar) added = self.write_variables(outfile, match.group(1), variables) - _remap_count() + _remap_count(added) new_lineno += added match = re.search(r'^(\s*)Export\(.(.*).\)', line) @@ -123,7 +123,7 @@ def _remap_count(): newvar = var.strip("\", '") variables.append(newvar) added = self.read_variables(outfile, match.group(1), variables) - _remap_count() + _remap_count(added) new_lineno += added if not scons_header: @@ -134,7 +134,7 @@ def _remap_count(): # not universally correct it should be correct for all flake clean code. if line.strip() == '': added = self.write_header(outfile) - _remap_count() + _remap_count(added) new_lineno += added scons_header = True diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index d1c64f24918..abbd3061528 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -4,7 +4,7 @@ pyenchant ## https://github.com/pycqa/flake8/issues/1389 https://github.com/PyCQA/flake8/pull/1720 flake8<6.0.0 isort==5.13.2 -pylint==3.1.0 +pylint==3.2.2 yamllint==1.35.1 codespell==2.2.6 # Used by ci/jira_query.py which pip installs it standalone. diff --git a/utils/githooks/hook_base.sh b/utils/githooks/hook_base.sh index f672586426b..3def155df68 100755 --- a/utils/githooks/hook_base.sh +++ b/utils/githooks/hook_base.sh @@ -1,9 +1,33 @@ #!/bin/bash +# +# Copyright 2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# set -eu . utils/githooks/find_base.sh export TARGET +# Common function to keep headers aligned +function _print_githook_header() { + printf "%-17s " "${1}:" +} +export -f _print_githook_header + +# Get list of staged files, excluding deleted +# shellcheck disable=SC2086 +function _git_diff_cached_files() { + IFS=' ' read -r -a _filter <<< "${1:-}" + local args="${2:-}" + if [ ${#_filter[@]} -eq 0 ]; then + git diff "$TARGET" --cached --name-only --diff-filter=d $args + else + git diff "$TARGET" --cached --name-only --diff-filter=d $args -- "${_filter[@]}" + fi +} +export -f _git_diff_cached_files + hook=${0##*/} rm -f ".${hook}" diff --git a/utils/githooks/pre-commit.d/10-update-copyright b/utils/githooks/pre-commit.d/10-update-copyright index 090c50f9cb4..e2641848cd7 100755 --- a/utils/githooks/pre-commit.d/10-update-copyright +++ b/utils/githooks/pre-commit.d/10-update-copyright @@ -1,13 +1,19 @@ #!/bin/bash - +# +# Copyright 2022-2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# # A git hook to validate and correct the copyright date in source files. -echo "Copyright update:" +_print_githook_header "Copyright" if [ -e .git/MERGE_HEAD ]; then - echo " Merge commit" + echo "Merge commit. Skipping" exit 0 fi +echo "Updating copyright headers" + regex='(^[[:blank:]]*[\*/]*.*)((Copyright[[:blank:]]*)([0-9]{4})(-([0-9]{4}))?)([[:blank:]]*(Intel.*$))' year=$(date +%Y) errors=0 @@ -39,20 +45,16 @@ targets=( '.env' ) +if [ -z "$files" ]; then + files=$(git diff "$TARGET" --cached --diff-filter=AM --name-only -- "${targets[@]}") +else + echo " Checking against custom files" +fi + os=$(uname -s) . utils/githooks/git-version.sh -if [ -z "$files" ]; then - if [ "$TARGET" = "HEAD" ]; then - echo " Checking against HEAD" - files=$(git diff HEAD --diff-filter=AM --name-only -- "${targets[@]}") - else - echo " Checking against branch ${TARGET}" - files=$(git diff "$TARGET"... --diff-filter=AM --name-only -- "${targets[@]}") - fi -fi - for file in $files; do if [[ "$file" == *vendor* ]] || [[ "$file" == *pb.go ]] || [[ "$file" == *_string.go ]] || [[ "$file" == *pb-c* ]] || diff --git a/utils/githooks/pre-commit.d/20-codespell.sh b/utils/githooks/pre-commit.d/20-codespell.sh index bf6f85e9b04..570a85c1d6e 100755 --- a/utils/githooks/pre-commit.d/20-codespell.sh +++ b/utils/githooks/pre-commit.d/20-codespell.sh @@ -9,20 +9,15 @@ set -ue -echo "CodeSpell:" +_print_githook_header "CodeSpell" # shellcheck disable=SC1091 if ! command -v codespell > /dev/null 2>&1 then - echo " codespell not installed. Install codespell command to improve pre-commit checks" + echo "codespell not installed. Install codespell command to improve pre-commit checks:" echo " python3 -m pip install -r ./utils/cq/requirements.txt" exit 0 fi -if [ "$TARGET" = "HEAD" ]; then - echo " Checking against HEAD" - git diff HEAD --name-only | xargs codespell -else - echo " Checking against branch ${TARGET}" - git diff "$TARGET"... --name-only | xargs codespell -fi +echo "Checking for spelling mistakes" +_git_diff_cached_files | xargs codespell diff --git a/utils/githooks/pre-commit.d/30-Jenkinsfile b/utils/githooks/pre-commit.d/30-Jenkinsfile index 8c9c8ee1bbd..6ee2efed4aa 100755 --- a/utils/githooks/pre-commit.d/30-Jenkinsfile +++ b/utils/githooks/pre-commit.d/30-Jenkinsfile @@ -1,5 +1,9 @@ #!/bin/sh - +# +# Copyright 2023-2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# # Checks the Jenkinsfile for syntax errors # # Will only check if Jenkinsfile is modified @@ -7,14 +11,14 @@ set -ue -echo "Jenkinsfile:" +_print_githook_header "Jenkinsfile" -if ! git diff --cached --stat 2>/dev/null | grep -q Jenkinsfile; then - echo " No changes, skipping checking" +if [ -z "$(_git_diff_cached_files "Jenkinsfile")" ] ; then + echo "No Jenkinsfile changes. Skipping" exit 0 fi -echo " Checking Jenkinsfile" +echo "Checking syntax" HOST="${HOST:-build.hpdd.intel.com}" CURL_VERBOSE=${CURL_VERBOSE:-""} @@ -22,7 +26,7 @@ CURL_PROXY="${CURL_PROXY:+-x }${CURL_PROXY:-}" CURL_OPTS="$CURL_PROXY $CURL_VERBOSE -s" URL="https://$HOST/pipeline-model-converter/validate" if ! output=$(curl $CURL_OPTS -s -X POST -F "jenkinsfile=<${1:-Jenkinsfile}" "$URL"); then - echo " Failed to access $URL. Skipping Jenkinsfile check" + echo " Failed to access $URL. Skipping" exit 0 fi diff --git a/utils/githooks/pre-commit.d/40-yamllint.sh b/utils/githooks/pre-commit.d/40-yamllint.sh index d395dc25564..60b62fe86a8 100755 --- a/utils/githooks/pre-commit.d/40-yamllint.sh +++ b/utils/githooks/pre-commit.d/40-yamllint.sh @@ -1,5 +1,9 @@ #!/bin/bash - +# +# Copyright 2022-2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# # Runs yamllint for the DAOS project. # # Picks up yamllint config settings from .yamllint.yaml @@ -7,18 +11,16 @@ # can create one locally to overrule these settings for local commit hooks. set -ue +_print_githook_header "Yaml Lint" + if ! command -v yamllint > /dev/null 2>&1 then - echo "No yaml checking, install yamllint command to improve pre-commit checks" - echo "python3 -m pip install -r ./utils/cq/requirements.txt" + echo "yamllint not installed. Install yamllint command to improve pre-commit checks:" + echo " python3 -m pip install -r ./utils/cq/requirements.txt" exit 0 fi echo "Checking yaml formatting" -targets=( - '*.yml' - '*.yaml' -) -git diff --diff-filter=ACMRTUXB --name-only --cached -z -- "${targets[@]}" |\ - xargs -r0 yamllint --strict +_git_diff_cached_files '*.yml *.yaml' '-z' | xargs -r0 yamllint --strict + diff --git a/utils/githooks/pre-commit.d/50-clang-format b/utils/githooks/pre-commit.d/50-clang-format index 36d5d6adb78..82b725d2624 100755 --- a/utils/githooks/pre-commit.d/50-clang-format +++ b/utils/githooks/pre-commit.d/50-clang-format @@ -1,22 +1,28 @@ #!/bin/bash - +# +# Copyright 2022-2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# set -ue -echo "Clang-format:" +_print_githook_header "Clang-format" if [ -e .git/MERGE_HEAD ]; then - echo " Merge commit" + echo "Merge commit. Skipping" exit 0 fi if ! command -v git-clang-format > /dev/null 2>&1; then - echo " No git-clang-format checking" + echo "git-clang-format not installed. Skipping" exit 0 fi if ! command -v clang-format > /dev/null 2>&1; then - echo " No clang-format checking" + echo "clang-format not installed. Skipping" exit 0 fi +echo "Formatting C files" + # Check version of clang-format, and print a helpful message if it's too old. If the right version # is not found then exit. ./site_scons/site_tools/extra/extra.py || exit 0 diff --git a/utils/githooks/pre-commit.d/60-gofmt.sh b/utils/githooks/pre-commit.d/60-gofmt.sh index 022c2466427..0a702948786 100755 --- a/utils/githooks/pre-commit.d/60-gofmt.sh +++ b/utils/githooks/pre-commit.d/60-gofmt.sh @@ -1,24 +1,22 @@ #!/bin/bash - +# +# Copyright 2022-2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# # Runs gofmt for the DAOS project as a commit hook. # # To get the most out of this hook the 'gh' command should be installed and working. set -ue -echo "Gofmt:" +_print_githook_header "Gofmt" # shellcheck disable=SC1091 -go_files= -if [ "$TARGET" = "HEAD" ]; then - echo " Checking against HEAD" - go_files=$(git diff HEAD --name-only --diff-filter=d | grep -e '.go$' || exit 0) -else - echo " Checking against branch ${TARGET}" - go_files=$(git diff "$TARGET"... --name-only --diff-filter=d | grep -e '.go$' || exit 0) -fi +go_files=$(_git_diff_cached_files '*.go') if [ -z "$go_files" ]; then + echo "No GO changes. Skipping" exit 0 fi @@ -29,11 +27,11 @@ fi output=$(echo "$go_files" | xargs gofmt -d) if [ -n "$output" ]; then - echo " ERROR: Your code hasn't been run through gofmt!" - echo " Please configure your editor to run gofmt on save." - echo " Alternatively, at a minimum, run the following command:" - echo -n " find src/control -name '*.go' -and -not -path '*vendor*'" - echo " | xargs gofmt -w" - echo -e "\n gofmt check found the following:\n\n$output\n" - exit 1 + echo " ERROR: Your code hasn't been run through gofmt!" + echo " Please configure your editor to run gofmt on save." + echo " Alternatively, at a minimum, run the following command:" + echo -n " find src/control -name '*.go' -and -not -path '*vendor*'" + echo " | xargs gofmt -w" + echo -e "\n gofmt check found the following:\n\n$output\n" + exit 1 fi diff --git a/utils/githooks/pre-commit.d/70-isort.sh b/utils/githooks/pre-commit.d/70-isort.sh index 65e3178e8a3..9b3d9fc445a 100755 --- a/utils/githooks/pre-commit.d/70-isort.sh +++ b/utils/githooks/pre-commit.d/70-isort.sh @@ -1,25 +1,36 @@ #!/bin/bash - +# +# Copyright 2023-2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# # Runs isort for the DAOS project. set -ue +_print_githook_header "isort" + +py_files=$(_git_diff_cached_files "*.py SConstruct */SConscript") + +if [ -z "$py_files" ]; then + echo "No python changes. Skipping" + exit 0 +fi + if ! command -v isort > /dev/null 2>&1; then - echo "No isort checking, install isort command to improve pre-commit checks" + echo "isort not installed. Install isort command to improve pre-commit checks:" + echo " python3 -m pip install -r ./utils/cq/requirements.txt" . /etc/os-release if [ "$ID" = "fedora" ]; then - echo "dnf install python3-isort" - echo "or" + echo " or" + echo " dnf install python3-isort" fi - echo "python3 -m pip install -r ./utils/cq/requirements.txt" exit 0 fi -echo "isort:" +echo "Checking if python imports are sorted" -echo " Running isort for python imports." -isort_args=(--jobs 8 .) -if ! isort --check-only "${isort_args[@]}"; then - echo " isort check failed, run 'isort ${isort_args[*]}' to fix." +if ! echo "$py_files" | xargs -r isort --check-only --jobs 8; then + echo " isort check failed, run 'isort --jobs 8 .' to fix." exit 1 fi diff --git a/utils/githooks/pre-commit.d/71-flake.sh b/utils/githooks/pre-commit.d/71-flake.sh index 619457210b3..70eb9d9c08f 100755 --- a/utils/githooks/pre-commit.d/71-flake.sh +++ b/utils/githooks/pre-commit.d/71-flake.sh @@ -1,5 +1,9 @@ #!/bin/sh - +# +# Copyright 2022-2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# # Runs flake8 for the DAOS project. # # Will first check uncommitted code, then either the entire tree or against an entire @@ -15,25 +19,24 @@ set -ue -echo "Flake8:" +_print_githook_header "Flake8" if ! command -v flake8 > /dev/null 2>&1; then - echo " No flake checking, install flake8 command to improve pre-commit checks" - echo "python3 -m pip install -r ./utils/cq/requirements.txt" + echo "flake8 not installed. Install flake8 command to improve pre-commit checks:" + echo " python3 -m pip install -r ./utils/cq/requirements.txt" exit 0 fi if flake8 --version | grep ^6\\.; then - echo " Flake8 >= 6.x does not have the --diff option, skipping." + echo "flake8 >= 6.x does not have the --diff option. Skipping." exit 0 fi if [ ! -f .flake8 ]; then - echo " No .flake8, skipping flake checks" + echo "No .flake8 config. Skipping" exit 0 fi -echo " Checking uncommitted code with flake." -git diff -u | flake8 --diff +echo "Linting python" if ! BRANCH=origin/$(git rev-parse --abbrev-ref HEAD 2>/dev/null); then echo " Failed to determine branch with git rev-parse" @@ -47,18 +50,9 @@ else # shellcheck disable=SC1091 - if [ "$TARGET" = "HEAD" ]; then - echo " Checking against branch HEAD" - git diff HEAD -U10 | flake8 --config .flake8 --diff - - echo " Checking scons code against branch ${TARGET}" - git diff HEAD -U10 | flake8 --config .flake8-scons --diff - else - - echo " Checking against branch ${TARGET}" - git diff "$TARGET"... -U10 | flake8 --config .flake8 --diff + # non-scons + git diff "$TARGET" -U10 | flake8 --config .flake8 --diff - echo " Checking scons code against branch ${TARGET}" - git diff "$TARGET"... -U10 | flake8 --config .flake8-scons --diff - fi + # scons + git diff "$TARGET" -U10 | flake8 --config .flake8-scons --diff fi diff --git a/utils/githooks/pre-commit.d/72-pylint.sh b/utils/githooks/pre-commit.d/72-pylint.sh index f17d6b943e8..645e2d872fc 100755 --- a/utils/githooks/pre-commit.d/72-pylint.sh +++ b/utils/githooks/pre-commit.d/72-pylint.sh @@ -1,21 +1,18 @@ #!/bin/sh - +# +# Copyright 2022-2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# # Runs pylint for the DAOS project as a commit hook. # # To get the most out of this hook the 'gh' command should be installed and working. set -ue -echo "Pylint:" +_print_githook_header "Pylint" # shellcheck disable=SC1091 +echo "Linting python" -if [ -f utils/cq/daos_pylint.py ]; then - if [ "$TARGET" = "HEAD" ]; then - echo " Checking against HEAD" - git diff HEAD --name-only | ./utils/cq/daos_pylint.py --files-from-stdin - else - echo " Checking against branch ${TARGET}" - git diff "$TARGET" --name-only | ./utils/cq/daos_pylint.py --files-from-stdin - fi -fi +_git_diff_cached_files | ./utils/cq/daos_pylint.py --files-from-stdin diff --git a/utils/githooks/pre-commit.d/73-ftest.sh b/utils/githooks/pre-commit.d/73-ftest.sh new file mode 100755 index 00000000000..6822cd8dcdb --- /dev/null +++ b/utils/githooks/pre-commit.d/73-ftest.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# +# Copyright 2024 Intel Corporation. +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +# Runs "tags.py lint" for ftest changes. +# + +set -ue + +_print_githook_header "Ftest" + +if ! python3 -c 'import yaml' > /dev/null 2>&1; then + echo "python3 requirements not installed. Install requirements to improve pre-commit checks:" + echo " python3 -m pip install -r ./utils/cq/requirements.txt" + exit 0 +fi + +echo "Linting modified files" + +_git_diff_cached_files '*/ftest/*.py' | xargs -r python3 src/tests/ftest/tags.py lint diff --git a/utils/node_local_test.py b/utils/node_local_test.py index 095a2b7c58c..29d2f483e05 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -761,9 +761,6 @@ def _start(self): cmd = [daos_server, 'start', f'--config={self._yaml_file.name}', '--insecure'] - if self.conf.args.no_root: - cmd.append('--recreate-superblocks') - # pylint: disable=consider-using-with self._sp = subprocess.Popen(cmd, env=plain_env) diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index fba56d39041..a629d6c5407 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -14,8 +14,8 @@ %endif Name: daos -Version: 2.5.101 -Release: 5%{?relval}%{?dist} +Version: 2.7.100 +Release: 1%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -589,6 +589,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Mon May 20 2024 Phillip Henderson <phillip.henderson@intel.com> 2.7.100-1 +- Bump version to 2.7.100 + * Fri May 03 2024 Lei Huang <lei.huang@intel.com> 2.5.101-5 - Add libaio as a dependent package diff --git a/utils/rpms/packaging/get_release_branch b/utils/rpms/packaging/get_release_branch index 9fd0934f6b3..334c4f0e2d9 100755 --- a/utils/rpms/packaging/get_release_branch +++ b/utils/rpms/packaging/get_release_branch @@ -1,12 +1,14 @@ #!/bin/bash # find the base branch of the current branch +# base branches can be master, release/2.4+, release/3+ +# or optionally branches passed into $1 set -eu -o pipefail IFS=' ' read -r -a add_bases <<< "${1:-}" origin=origin mapfile -t all_bases < <(echo "master" - git branch -r | sed -ne "/^ $origin\\/release\\/[0-9]/s/^ $origin\\///p") + git branch -r | sed -ne "/^ $origin\\/release\\/\(2.[4-9]\|[3-9]\)/s/^ $origin\\///p") all_bases+=("${add_bases[@]}") TARGET="master" min_diff=-1