From 7e2dfcefe1a9851fe633abfdea21b94b0668a1d0 Mon Sep 17 00:00:00 2001 From: wangdi Date: Mon, 4 Dec 2023 06:24:19 -0800 Subject: [PATCH 01/23] DAOS-14352 object: right epoch to fetch the data (#13049) (#13111) * DAOS-14352 object: right epoch to fetch the data (#13049) If the epoch is higher than EC aggregate boundary, then it should use stable epoch to fetch the data, since the data could be aggregated independently on parity and data shard, so using stable epoch could make sure the consistency view during rebuild. And also EC aggregation should already aggregate the parity, so there should not be any partial update on the parity as well. Otherwise there might be partial update on this rebuilding shard, so let's use the epoch from the parity shard to fetch the data here, which will make sure partial update will not be fetched here. And also EC aggregation is being disabled at the moment, so there should not be any vos aggregation impact this process as well. * DAOS-14352 object: fetch ec boundary (#13155) Fetch EC boundary epoch before object migration actively, since during reintegration, EC boundary may not be synced to the reintegrating target before object migration. * DAOS-14352 object: checking fetch data epoch during parity rebuild. (#13237) During parity rebuild, it should compare with stable epoch to make sure using the right epoch to fetch from data shards. Signed-off-by: Di Wang --- src/container/container_iv.c | 33 ++++++++++- src/container/srv_container.c | 8 +++ src/include/daos_srv/container.h | 2 + src/object/srv_obj_migrate.c | 97 +++++++++++++++++++++++++------ src/tests/suite/daos_rebuild_ec.c | 74 +++++++++++++++++++++++ 5 files changed, 195 insertions(+), 19 deletions(-) diff --git a/src/container/container_iv.c b/src/container/container_iv.c index cccf8f7d6f2..bac555ac158 100644 --- a/src/container/container_iv.c +++ b/src/container/container_iv.c @@ -191,8 +191,12 @@ cont_iv_ent_copy(struct ds_iv_entry *entry, struct cont_iv_key *key, cip_acl.dal_ace[src->iv_prop.cip_acl.dal_len]); memcpy(&dst->iv_prop, &src->iv_prop, size); break; + case IV_CONT_AGG_EPOCH_BOUNDRY: + dst->iv_agg_eph.eph = src->iv_agg_eph.eph; + break; default: - D_ERROR("bad iv_class_id %d.\n", entry->iv_class->iv_class_id); + D_ERROR("bad iv_class_id %d: "DF_RC"\n", entry->iv_class->iv_class_id, + DP_RC(-DER_INVAL)); return -DER_INVAL; }; @@ -1071,6 +1075,33 @@ cont_iv_ec_agg_eph_refresh(void *ns, uuid_t cont_uuid, daos_epoch_t eph) IV_CONT_AGG_EPOCH_BOUNDRY); } +int +ds_cont_fetch_ec_agg_boundary(void *ns, uuid_t cont_uuid) +{ + + struct cont_iv_entry iv_entry = { 0 }; + int rc; + + /* Only happens on xstream 0 */ + D_ASSERT(dss_get_module_info()->dmi_xs_id == 0); + uuid_copy(iv_entry.cont_uuid, cont_uuid); + rc = crt_group_rank(NULL, &iv_entry.iv_agg_eph.rank); + if (rc) { + D_ERROR(DF_UUID" crt_group_rank failed "DF_RC"\n", + DP_UUID(cont_uuid), DP_RC(rc)); + return rc; + } + + rc = cont_iv_fetch(ns, IV_CONT_AGG_EPOCH_BOUNDRY, cont_uuid, &iv_entry, + sizeof(struct cont_iv_entry), sizeof(struct cont_iv_entry), + true); + if (rc) + D_ERROR(DF_UUID" cont_iv_fetch failed "DF_RC"\n", + DP_UUID(cont_uuid), DP_RC(rc)); + return rc; + +} + int cont_iv_capability_update(void *ns, uuid_t cont_hdl_uuid, uuid_t cont_uuid, uint64_t flags, uint64_t sec_capas, uint32_t pm_ver) diff --git a/src/container/srv_container.c b/src/container/srv_container.c index 282761ec2bf..35da023ff24 100644 --- a/src/container/srv_container.c +++ b/src/container/srv_container.c @@ -1799,6 +1799,12 @@ cont_agg_eph_leader_ult(void *arg) while (!dss_ult_exiting(svc->cs_ec_leader_ephs_req)) { d_rank_list_t fail_ranks = { 0 }; + if (pool->sp_rebuilding) { + D_DEBUG(DB_MD, DF_UUID "skip during rebuilding.\n", + DP_UUID(pool->sp_uuid)); + goto yield; + } + rc = map_ranks_init(pool->sp_map, PO_COMP_ST_DOWNOUT | PO_COMP_ST_DOWN, &fail_ranks); if (rc) { @@ -1870,6 +1876,8 @@ cont_agg_eph_leader_ult(void *arg) continue; } ec_agg->ea_current_eph = min_eph; + if (pool->sp_rebuilding) + break; } map_ranks_fini(&fail_ranks); diff --git a/src/include/daos_srv/container.h b/src/include/daos_srv/container.h index 7f0a5d3390d..3442f4b7de1 100644 --- a/src/include/daos_srv/container.h +++ b/src/include/daos_srv/container.h @@ -256,4 +256,6 @@ void ds_cont_ec_timestamp_update(struct ds_cont_child *cont); typedef int(*cont_rdb_iter_cb_t)(uuid_t pool_uuid, uuid_t cont_uuid, struct rdb_tx *tx, void *arg); int ds_cont_rdb_iterate(struct cont_svc *svc, cont_rdb_iter_cb_t iter_cb, void *cb_arg); int ds_cont_rf_check(uuid_t pool_uuid, uuid_t cont_uuid, struct rdb_tx *tx); + +int ds_cont_fetch_ec_agg_boundary(void *ns, uuid_t cont_uuid); #endif /* ___DAOS_SRV_CONTAINER_H_ */ diff --git a/src/object/srv_obj_migrate.c b/src/object/srv_obj_migrate.c index 5ff44520bc1..4d22c1bb228 100644 --- a/src/object/srv_obj_migrate.c +++ b/src/object/srv_obj_migrate.c @@ -928,7 +928,8 @@ migrate_update_parity(struct migrate_one *mrone, daos_epoch_t parity_eph, static int __migrate_fetch_update_parity(struct migrate_one *mrone, daos_handle_t oh, - daos_iod_t *iods, daos_epoch_t **ephs, uint32_t iods_num, + daos_iod_t *iods, daos_epoch_t fetch_eph, + daos_epoch_t **ephs, uint32_t iods_num, struct ds_cont_child *ds_cont, bool encode) { d_sg_list_t sgls[OBJ_ENUM_UNPACK_MAX_IODS]; @@ -959,7 +960,7 @@ __migrate_fetch_update_parity(struct migrate_one *mrone, daos_handle_t oh, D_DEBUG(DB_REBUILD, DF_UOID" mrone %p dkey "DF_KEY" nr %d eph "DF_U64"\n", DP_UOID(mrone->mo_oid), mrone, DP_KEY(&mrone->mo_dkey), iods_num, mrone->mo_epoch); - rc = mrone_obj_fetch(mrone, oh, sgls, iods, iods_num, mrone->mo_epoch, DIOF_FOR_MIGRATION, + rc = mrone_obj_fetch(mrone, oh, sgls, iods, iods_num, fetch_eph, DIOF_FOR_MIGRATION, NULL); if (rc) { D_ERROR("migrate dkey "DF_KEY" failed: "DF_RC"\n", @@ -1022,22 +1023,54 @@ static int migrate_fetch_update_parity(struct migrate_one *mrone, daos_handle_t oh, struct ds_cont_child *ds_cont) { + int i; + int j; int rc = 0; /* If it is parity recxs from another replica, then let's encode it anyway */ - if (mrone->mo_iods_num_from_parity > 0) { - rc = __migrate_fetch_update_parity(mrone, oh, mrone->mo_iods_from_parity, - mrone->mo_iods_update_ephs_from_parity, - mrone->mo_iods_num_from_parity, ds_cont, - true); + for (i = 0; i < mrone->mo_iods_num_from_parity; i++) { + for (j = 0; j < mrone->mo_iods_from_parity[i].iod_nr; j++) { + daos_iod_t iod = mrone->mo_iods_from_parity[i]; + daos_epoch_t fetch_eph; + daos_epoch_t update_eph; + daos_epoch_t *update_eph_p; - if (rc) - return rc; + iod.iod_nr = 1; + iod.iod_recxs = &mrone->mo_iods_from_parity[i].iod_recxs[j]; + /* If the epoch is higher than EC aggregate boundary, then + * it should use stable epoch to fetch the data, since + * the data could be aggregated independently on parity + * and data shard, so using stable epoch could make sure + * the consistency view during rebuild. And also EC aggregation + * should already aggregate the parity, so there should not + * be any partial update on the parity as well. + * + * Otherwise there might be partial update on this rebuilding + * shard, so let's use the epoch from the parity shard to fetch + * the data here, which will make sure partial update will not + * be fetched here. And also EC aggregation is being disabled + * at the moment, so there should not be any vos aggregation + * impact this process as well. + */ + if (ds_cont->sc_ec_agg_eph_boundary > + mrone->mo_iods_update_ephs_from_parity[i][j]) + fetch_eph = mrone->mo_epoch; + else + fetch_eph = mrone->mo_iods_update_ephs_from_parity[i][j]; + + update_eph = mrone->mo_iods_update_ephs_from_parity[i][j]; + update_eph_p = &update_eph; + rc = __migrate_fetch_update_parity(mrone, oh, &iod, fetch_eph, &update_eph_p, + mrone->mo_iods_num_from_parity, ds_cont, + true); + if (rc) + return rc; + } } /* Otherwise, keep it as replicate recx */ if (mrone->mo_iod_num > 0) { - rc = __migrate_fetch_update_parity(mrone, oh, mrone->mo_iods, + rc = __migrate_fetch_update_parity(mrone, oh, mrone->mo_iods, mrone->mo_epoch, mrone->mo_iods_update_ephs, mrone->mo_iod_num, ds_cont, false); } @@ -1350,21 +1383,39 @@ migrate_fetch_update_bulk(struct migrate_one *mrone, daos_handle_t oh, /* For EC object, if the migration include both extent from parity rebuild * and extent from replicate rebuild, let rebuild the extent with parity first, * then extent from replication. - * - * Since the parity shard epoch should be higher or equal to the data shard epoch, - * so let's use the minimum epochs of all parity shards as the update epoch of - * this data shard. */ - for (i = 0; i < mrone->mo_iods_num_from_parity; i++) { for (j = 0; j < mrone->mo_iods_from_parity[i].iod_nr; j++) { daos_iod_t iod = mrone->mo_iods_from_parity[i]; + daos_epoch_t fetch_eph; + daos_epoch_t update_eph; iod.iod_nr = 1; iod.iod_recxs = &mrone->mo_iods_from_parity[i].iod_recxs[j]; - rc = __migrate_fetch_update_bulk(mrone, oh, &iod, 1, - mrone->mo_iods_update_ephs_from_parity[i][j], - mrone->mo_iods_update_ephs_from_parity[i][j], + + /* If the epoch is higher than EC aggregate boundary, then + * it should use stable epoch to fetch the data, since + * the data could be aggregated independently on parity + * and data shard, so using stable epoch could make sure + * the consistency view during rebuild. And also EC aggregation + * should already aggregate the parity, so there should not + * be any partial update on the parity as well. + * + * Otherwise there might be partial update on this rebuilding + * shard, so let's use the epoch from the parity shard to fetch + * the data here, which will make sure partial update will not + * be fetched here. And also EC aggregation is being disabled + * at the moment, so there should not be any vos aggregation + * impact this process as well. + */ + if (ds_cont->sc_ec_agg_eph_boundary > + mrone->mo_iods_update_ephs_from_parity[i][j]) + fetch_eph = mrone->mo_epoch; + else + fetch_eph = mrone->mo_iods_update_ephs_from_parity[i][j]; + + update_eph = mrone->mo_iods_update_ephs_from_parity[i][j]; + rc = __migrate_fetch_update_bulk(mrone, oh, &iod, 1, fetch_eph, update_eph, DIOF_EC_RECOV_FROM_PARITY, ds_cont); if (rc != 0) D_GOTO(out, rc); @@ -3165,6 +3216,16 @@ migrate_cont_iter_cb(daos_handle_t ih, d_iov_t *key_iov, D_GOTO(out_put, rc); } + rc = ds_cont_fetch_ec_agg_boundary(dp->sp_iv_ns, cont_uuid); + if (rc) { + /* Sometime it may too early to fetch the EC boundary, + * since EC boundary does not start yet, which is forbidden + * during rebuild anyway, so let's continue. + */ + D_DEBUG(DB_REBUILD, DF_UUID" fetch agg_boundary failed: "DF_RC"\n", + DP_UUID(cont_uuid), DP_RC(rc)); + } + arg.yield_freq = DEFAULT_YIELD_FREQ; arg.cont_root = root; arg.snaps = snapshots; diff --git a/src/tests/suite/daos_rebuild_ec.c b/src/tests/suite/daos_rebuild_ec.c index 6669d32490e..072bb9a4692 100644 --- a/src/tests/suite/daos_rebuild_ec.c +++ b/src/tests/suite/daos_rebuild_ec.c @@ -1253,6 +1253,77 @@ rebuild_ec_multiple_failure_tgts(void **state) ioreq_fini(&req); } +static void +rebuild_ec_parity_overwrite_fail_parity(void **state) +{ + test_arg_t *arg = *state; + daos_obj_id_t oid; + struct ioreq req; + char *data; + char *verify_data; + daos_recx_t recx; + d_rank_t data_ranks[2]; + d_rank_t parity_rank; + uint64_t dkey_hash; + int shard_idx; + int stripe_size = 4 * CELL_SIZE; + + if (!test_runable(arg, 8)) + return; + + oid = daos_test_oid_gen(arg->coh, OC_EC_4P2G1, 0, 0, arg->myrank); + ioreq_init(&req, arg->coh, oid, DAOS_IOD_ARRAY, arg); + data = (char *)malloc(stripe_size); + verify_data = (char *)malloc(stripe_size); + make_buffer(data, 'a', stripe_size); + + recx.rx_idx = 0; /* full stripe */ + recx.rx_nr = stripe_size; + insert_recxs("d_key", "a_key", 1, DAOS_TX_NONE, &recx, 1, + data, stripe_size, &req); + + make_buffer(data, 'b', 1000); + memcpy(verify_data, data, stripe_size); + recx.rx_idx = 0; + recx.rx_nr = 1000; + insert_recxs("d_key", "a_key", 1, DAOS_TX_NONE, &recx, 1, + data, 1000, &req); + + dkey_hash = d_hash_murmur64((const unsigned char *)"d_key", strlen("d_key"), 5731); + + /* fail parity epoch */ + shard_idx = (dkey_hash % 6 + 5) % 6; + parity_rank = get_rank_by_oid_shard(arg, oid, shard_idx); + rebuild_single_pool_rank(arg, parity_rank, true); + + /* fail data shard */ + shard_idx = (dkey_hash % 6 + 0) % 6; + data_ranks[0] = get_rank_by_oid_shard(arg, oid, shard_idx); + shard_idx = (dkey_hash % 6 + 1) % 6; + data_ranks[1] = get_rank_by_oid_shard(arg, oid, shard_idx); + + rebuild_pools_ranks(&arg, 1, data_ranks, 2, true); + recx.rx_idx = 0; /* full stripe */ + recx.rx_nr = stripe_size; + lookup_recxs("d_key", "a_key", 1, DAOS_TX_NONE, &recx, 1, + data, stripe_size, &req); + + assert_memory_equal(data, verify_data, stripe_size); + + reintegrate_pools_ranks(&arg, 1, data_ranks, 2, true); + reintegrate_pools_ranks(&arg, 1, &parity_rank, 1, true); + + lookup_recxs("d_key", "a_key", 1, DAOS_TX_NONE, &recx, 1, + data, stripe_size, &req); + + assert_memory_equal(data, verify_data, stripe_size); + + + ioreq_fini(&req); + free(data); + free(verify_data); +} + /** create a new pool/container for each test */ static const struct CMUnitTest rebuild_tests[] = { {"REBUILD0: rebuild partial update with data tgt fail", @@ -1385,6 +1456,9 @@ static const struct CMUnitTest rebuild_tests[] = { {"REBUILD45: multiple shards fail tgts", rebuild_ec_multiple_failure_tgts, rebuild_ec_8nodes_setup, test_teardown}, + {"REBUILD46: fail parity shard and data shards after overwrite", + rebuild_ec_parity_overwrite_fail_parity, rebuild_ec_8nodes_setup, + test_teardown}, }; int From 5d918d300bfdde5db3757e73b66c55370d3a22c9 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Tue, 5 Dec 2023 09:36:41 -0600 Subject: [PATCH 02/23] DAOS-13883 cart: control request_post_init/incr through env vars (#12565) (#12566) Add D_POST_INIT / D_POST_INCR env vars This enables tuning of multi-recv buffer sizes and number of server preposted recvs when multi-recv is not enabled. Signed-off-by: Jerome Soumagne --- src/cart/crt_hg.c | 3 ++ src/cart/crt_hg.h | 4 +++ src/cart/crt_init.c | 56 +++++++++++++++++++++++++---------- src/cart/crt_internal_types.h | 4 +++ 4 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/cart/crt_hg.c b/src/cart/crt_hg.c index 299088524eb..2b74f882bc1 100644 --- a/src/cart/crt_hg.c +++ b/src/cart/crt_hg.c @@ -874,6 +874,9 @@ crt_hg_class_init(int provider, int idx, bool primary, hg_class_t **ret_hg_class if (prov_data->cpg_max_unexp_size > 0) init_info.na_init_info.max_unexpected_size = prov_data->cpg_max_unexp_size; + init_info.request_post_init = crt_gdata.cg_post_init; + init_info.request_post_incr = crt_gdata.cg_post_incr; + hg_class = HG_Init_opt(info_string, crt_is_service(), &init_info); if (hg_class == NULL) { D_ERROR("Could not initialize HG class.\n"); diff --git a/src/cart/crt_hg.h b/src/cart/crt_hg.h index 48ea4c74f41..dacb9fed577 100644 --- a/src/cart/crt_hg.h +++ b/src/cart/crt_hg.h @@ -27,6 +27,10 @@ /** number of prepost HG handles when enable pool */ #define CRT_HG_POOL_PREPOST_NUM (16) +/** default values for init / incr to prepost handles */ +#define CRT_HG_POST_INIT (512) +#define CRT_HG_POST_INCR (512) + struct crt_rpc_priv; struct crt_common_hdr; struct crt_corpc_hdr; diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c index 988e35db701..fcc62c207e5 100644 --- a/src/cart/crt_init.c +++ b/src/cart/crt_init.c @@ -63,16 +63,39 @@ dump_envariables(void) { int i; char *val; - char *envars[] = {"D_PROVIDER", "D_INTERFACE", "D_DOMAIN", "D_PORT", - "CRT_PHY_ADDR_STR", "D_LOG_STDERR_IN_LOG", "D_LOG_SIZE", - "D_LOG_FILE", "D_LOG_FILE_APPEND_PID", "D_LOG_MASK", "DD_MASK", - "DD_STDERR", "DD_SUBSYS", "CRT_TIMEOUT", "CRT_ATTACH_INFO_PATH", - "OFI_PORT", "OFI_INTERFACE", "OFI_DOMAIN", "CRT_CREDIT_EP_CTX", - "CRT_CTX_SHARE_ADDR", "CRT_CTX_NUM", "D_FI_CONFIG", - "FI_UNIVERSE_SIZE", "CRT_ENABLE_MEM_PIN", - "FI_OFI_RXM_USE_SRX", "D_LOG_FLUSH", "CRT_MRC_ENABLE", - "CRT_SECONDARY_PROVIDER", "D_PROVIDER_AUTH_KEY", "D_PORT_AUTO_ADJUST", - "D_POLL_TIMEOUT"}; + char *envars[] = {"D_PROVIDER", + "D_INTERFACE", + "D_DOMAIN", + "D_PORT", + "CRT_PHY_ADDR_STR", + "D_LOG_STDERR_IN_LOG", + "D_LOG_SIZE", + "D_LOG_FILE", + "D_LOG_FILE_APPEND_PID", + "D_LOG_MASK", + "DD_MASK", + "DD_STDERR", + "DD_SUBSYS", + "CRT_TIMEOUT", + "CRT_ATTACH_INFO_PATH", + "OFI_PORT", + "OFI_INTERFACE", + "OFI_DOMAIN", + "CRT_CREDIT_EP_CTX", + "CRT_CTX_SHARE_ADDR", + "CRT_CTX_NUM", + "D_FI_CONFIG", + "FI_UNIVERSE_SIZE", + "CRT_ENABLE_MEM_PIN", + "FI_OFI_RXM_USE_SRX", + "D_LOG_FLUSH", + "CRT_MRC_ENABLE", + "CRT_SECONDARY_PROVIDER", + "D_PROVIDER_AUTH_KEY", + "D_PORT_AUTO_ADJUST", + "D_POLL_TIMEOUT", + "D_POST_INIT", + "D_POST_INCR"}; D_INFO("-- ENVARS: --\n"); for (i = 0; i < ARRAY_SIZE(envars); i++) { @@ -237,12 +260,13 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, /* first step init - for initializing crt_gdata */ static int data_init(int server, crt_init_options_t *opt) { - uint32_t timeout; + uint32_t timeout = 0; uint32_t credits; uint32_t fi_univ_size = 0; uint32_t mem_pin_enable = 0; uint32_t is_secondary; char ucx_ib_fork_init = 0; + uint32_t post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR; int rc = 0; D_DEBUG(DB_ALL, "initializing crt_gdata...\n"); @@ -252,6 +276,12 @@ static int data_init(int server, crt_init_options_t *opt) D_DEBUG(DB_ALL, "Starting RPCID %#lx. Num cores: %ld\n", crt_gdata.cg_rpcid, crt_gdata.cg_num_cores); + /* Set context post init / post incr to tune number of pre-posted recvs */ + d_getenv_int("D_POST_INIT", &post_init); + crt_gdata.cg_post_init = post_init; + d_getenv_int("D_POST_INCR", &post_incr); + crt_gdata.cg_post_incr = post_incr; + is_secondary = 0; /* Apply CART-890 workaround for server side only */ if (server) { @@ -264,13 +294,9 @@ static int data_init(int server, crt_init_options_t *opt) * is running using a secondary provider */ d_getenv_int("CRT_SECONDARY_PROVIDER", &is_secondary); - } - crt_gdata.cg_provider_is_primary = (is_secondary) ? 0 : 1; - timeout = 0; - if (opt && opt->cio_crt_timeout != 0) timeout = opt->cio_crt_timeout; else diff --git a/src/cart/crt_internal_types.h b/src/cart/crt_internal_types.h index 18148ef1bd6..1041f62bae0 100644 --- a/src/cart/crt_internal_types.h +++ b/src/cart/crt_internal_types.h @@ -91,6 +91,10 @@ struct crt_gdata { /** */ struct crt_prov_gdata *cg_prov_gdata_secondary; + /** Hints to mercury for request post init (ignored for clients) */ + uint32_t cg_post_init; + uint32_t cg_post_incr; + /** global timeout value (second) for all RPCs */ uint32_t cg_timeout; From c02ffb7103c8c20217a5222b91f997244417128f Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Tue, 5 Dec 2023 10:22:32 -0600 Subject: [PATCH 03/23] DAOS-14711 dfs: fix coverity 1565890 (#13439) Signed-off-by: Mohamad Chaarawi --- src/client/dfs/dfs.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/client/dfs/dfs.c b/src/client/dfs/dfs.c index 5368f605a89..7e40b618dac 100644 --- a/src/client/dfs/dfs.c +++ b/src/client/dfs/dfs.c @@ -4658,9 +4658,11 @@ dfs_read_int(dfs_t *dfs, dfs_obj_t *obj, daos_off_t off, dfs_iod_t *iod, if (rc) D_GOTO(err_params, rc); - rc = dc_task_schedule(task, true); - if (rc) - D_GOTO(err_task, rc); + /* + * dc_task_schedule() calls tse_task_complete() even on error (which also calls the + * completion cb that frees params in this case, so we can just ignore the rc here. + */ + dc_task_schedule(task, true); return 0; err_params: From 38a7e7a629dff95b05dbe7f2b0cca573ec96ea5c Mon Sep 17 00:00:00 2001 From: Ken Cain Date: Tue, 5 Dec 2023 11:26:23 -0500 Subject: [PATCH 04/23] DAOS-14686 test: ior/crash.py fix race in ci_nhandles check (#13405) (#13419) Cherry pick of master commit 5daa960 to release/2.4 branch. Before this change, the final container query ci_nhandle verification (done while ior is running without crash/interruption) would pass only if timing was fortunate. The code expected to see ci_nhandles=1 but the correct expectation is in fact ci_nhandles=2 (one for the test container handle open to perform the query, and another for the ior just launched). For a long time, the test would be lucky and always observe ci_nhandles=1. More recently, ci_nhandles=2 observations are occurring, presumably due to timing differences introduced in daos. With this change, the test is updated to expect ci_nhandles=2, and is also modified to perform multiple (5) attempts with 2 second delays between attempts, to give time for ior to start up and open the container. Signed-off-by: Kenneth Cain --- src/tests/ftest/ior/crash.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tests/ftest/ior/crash.py b/src/tests/ftest/ior/crash.py index a999e97d5d8..43f430e4ed1 100644 --- a/src/tests/ftest/ior/crash.py +++ b/src/tests/ftest/ior/crash.py @@ -26,6 +26,7 @@ def cont_nhandles_match(self, exp_nhandles=1, attempts=5, delay_sec=2): for _ in range(attempts): if self.container.check_container_info(ci_nhandles=exp_nhandles): return True + self.log.info("check_container_info does not match yet, sleep %d sec", delay_sec) time.sleep(delay_sec) return False @@ -97,5 +98,7 @@ def test_ior_crash(self): self.fail("One or more engines crashed") # Verify container handle opened by ior is closed (by ior before its graceful exit) - self.assertTrue(self.cont_nhandles_match(attempts=1, delay_sec=0), + # Give ior some time to get started and open the container! + # And, expect 2 open handles, one for this container open/query, and another for ior itself + self.assertTrue(self.cont_nhandles_match(exp_nhandles=2, attempts=5, delay_sec=2), "Error confirming container info nhandles") From db85d0f85f4bed7ea3b645ae08de23ad1d49facc Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 5 Dec 2023 16:49:21 +0000 Subject: [PATCH 05/23] DAOS-14624 gurt: Fix d_rank_list_del and friends (#13317) (#13387) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function int d_rank_list_del(d_rank_list_t *rank_list, d_rank_t rank) calls d_rank_list_t *d_rank_list_realloc(d_rank_list_t *ptr, uint32_t size) which may free (or reallocate) the object ptr points and return NULL (or the address of a new object). Due to d_rank_list_del’s signature, this can’t be conveyed to the caller, who may continue using the freed object. A similar pitfall exists in d_rank_list_append. This patch renames d_rank_list_realloc to d_rank_list_resize and changes its behavior so that it never frees the d_rank_list_t object. Features: dfuse Signed-off-by: Li Wei Signed-off-by: Ashley Pittman Co-authored-by: Li Wei --- src/cart/crt_group.c | 15 +++++++------ src/gurt/misc.c | 47 +++++++++++++++++++-------------------- src/include/gurt/common.h | 2 +- src/rdb/rdb_raft.c | 12 ++-------- 4 files changed, 34 insertions(+), 42 deletions(-) diff --git a/src/cart/crt_group.c b/src/cart/crt_group.c index cb34c06160c..f02ebdc96b8 100644 --- a/src/cart/crt_group.c +++ b/src/cart/crt_group.c @@ -2404,10 +2404,11 @@ grp_regen_linear_list(struct crt_grp_priv *grp_priv) /* If group size changed - reallocate the list */ if (!linear_list->rl_ranks || linear_list->rl_nr != grp_priv->gp_size) { - linear_list = d_rank_list_realloc(linear_list, - grp_priv->gp_size); - if (linear_list == NULL) - return -DER_NOMEM; + int rc; + + rc = d_rank_list_resize(linear_list, grp_priv->gp_size); + if (rc != 0) + return rc; } index = 0; @@ -2460,9 +2461,9 @@ grp_add_to_membs_list(struct crt_grp_priv *grp_priv, d_rank_t rank, uint64_t inc first = membs->rl_nr; new_amount = first + RANK_LIST_REALLOC_SIZE; - membs = d_rank_list_realloc(membs, new_amount); - if (membs == NULL) - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_resize(membs, new_amount); + if (rc != 0) + D_GOTO(out, rc); for (i = first; i < first + RANK_LIST_REALLOC_SIZE; i++) { membs->rl_ranks[i] = CRT_NO_RANK; diff --git a/src/gurt/misc.c b/src/gurt/misc.c index 003fa68a5ec..4c49b4dd6e6 100644 --- a/src/gurt/misc.c +++ b/src/gurt/misc.c @@ -388,26 +388,27 @@ d_rank_list_alloc(uint32_t size) return rank_list; } -d_rank_list_t * -d_rank_list_realloc(d_rank_list_t *ptr, uint32_t size) +int +d_rank_list_resize(d_rank_list_t *ptr, uint32_t size) { d_rank_t *new_rl_ranks; if (ptr == NULL) - return d_rank_list_alloc(size); + return -DER_INVAL; if (size == 0) { - d_rank_list_free(ptr); - return NULL; + D_FREE(ptr->rl_ranks); + ptr->rl_nr = 0; + return 0; } D_REALLOC_ARRAY(new_rl_ranks, ptr->rl_ranks, ptr->rl_nr, size); if (new_rl_ranks != NULL) { ptr->rl_ranks = new_rl_ranks; ptr->rl_nr = size; } else { - ptr = NULL; + return -DER_NOMEM; } - return ptr; + return 0; } void @@ -430,12 +431,11 @@ d_rank_list_copy(d_rank_list_t *dst, d_rank_list_t *src) } if (dst->rl_nr != src->rl_nr) { - dst = d_rank_list_realloc(dst, src->rl_nr); - if (dst == NULL) { - D_ERROR("d_rank_list_realloc() failed.\n"); - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_resize(dst, src->rl_nr); + if (rc != 0) { + D_ERROR("d_rank_list_resize() failed.\n"); + D_GOTO(out, rc); } - dst->rl_nr = src->rl_nr; } memcpy(dst->rl_ranks, src->rl_ranks, dst->rl_nr * sizeof(d_rank_t)); @@ -533,10 +533,10 @@ d_rank_list_del(d_rank_list_t *rank_list, d_rank_t rank) D_ASSERT(idx <= new_num); num_bytes = (new_num - idx) * sizeof(d_rank_t); memmove(dest, src, num_bytes); - rank_list = d_rank_list_realloc(rank_list, new_num); - if (rank_list == NULL) { - D_ERROR("d_rank_list_realloc() failed.\n"); - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_resize(rank_list, new_num); + if (rc != 0) { + D_ERROR("d_rank_list_resize() failed.\n"); + D_GOTO(out, rc); } out: return rc; @@ -545,16 +545,15 @@ d_rank_list_del(d_rank_list_t *rank_list, d_rank_t rank) int d_rank_list_append(d_rank_list_t *rank_list, d_rank_t rank) { - uint32_t old_num = rank_list->rl_nr; - d_rank_list_t *new_rank_list; - int rc = 0; + uint32_t old_num = rank_list->rl_nr; + int rc = 0; - new_rank_list = d_rank_list_realloc(rank_list, old_num + 1); - if (new_rank_list == NULL) { - D_ERROR("d_rank_list_realloc() failed.\n"); - D_GOTO(out, rc = -DER_NOMEM); + rc = d_rank_list_resize(rank_list, old_num + 1); + if (rc != 0) { + D_ERROR("d_rank_list_resize() failed.\n"); + D_GOTO(out, rc); } - new_rank_list->rl_ranks[old_num] = rank; + rank_list->rl_ranks[old_num] = rank; out: return rc; diff --git a/src/include/gurt/common.h b/src/include/gurt/common.h index 923be756e02..b55e4c53cd6 100644 --- a/src/include/gurt/common.h +++ b/src/include/gurt/common.h @@ -406,7 +406,7 @@ void d_rank_list_filter(d_rank_list_t *src_set, d_rank_list_t *dst_set, bool exclude); int d_rank_list_merge(d_rank_list_t *src_set, d_rank_list_t *merge_set); d_rank_list_t *d_rank_list_alloc(uint32_t size); -d_rank_list_t *d_rank_list_realloc(d_rank_list_t *ptr, uint32_t size); +int d_rank_list_resize(d_rank_list_t *ptr, uint32_t size); void d_rank_list_free(d_rank_list_t *rank_list); int d_rank_list_copy(d_rank_list_t *dst, d_rank_list_t *src); void d_rank_list_shuffle(d_rank_list_t *rank_list); diff --git a/src/rdb/rdb_raft.c b/src/rdb/rdb_raft.c index cfc292c0c6a..01af06519b9 100644 --- a/src/rdb/rdb_raft.c +++ b/src/rdb/rdb_raft.c @@ -1093,18 +1093,10 @@ rdb_raft_update_node(struct rdb *db, uint64_t index, raft_entry_t *entry) goto out_replicas; } - if (entry->type == RAFT_LOGTYPE_ADD_NODE) { + if (entry->type == RAFT_LOGTYPE_ADD_NODE) rc = d_rank_list_append(replicas, rank); - } else if (entry->type == RAFT_LOGTYPE_REMOVE_NODE) { - /* never expect 1->0 in practice, right? But protect against double-free. */ - bool replicas_freed = (replicas->rl_nr == 1); - + else if (entry->type == RAFT_LOGTYPE_REMOVE_NODE) rc = d_rank_list_del(replicas, rank); - if (replicas_freed) { - D_ASSERT(rc == -DER_NOMEM); - replicas = NULL; - } - } if (rc != 0) goto out_replicas; From 7c7cc4dee0d5374ea190c4ffbfabc882e09ae319 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Tue, 5 Dec 2023 14:23:43 -0800 Subject: [PATCH 06/23] DAOS-14559 ci: use isort to organize python imports (#13243) (#13258) (#13322) Combination of #13243 and #13258. Create a new generic linting job and add isort checking. Pin versions of python checking tools. Add job to master that checks versions are up-to-date. Fix isort options for line-length and blacklisting. Resolve isort and codespell issues. Overall this increases coverage with new checks, gives us a common linting job which should be easier to manage that multiple ones and allows us to run PRs with pinned versions to prevent upstream changes blocking development but still fails master checking when updates are available. Signed-off-by: Ashley Pittman Signed-off-by: Yinglin Sun Signed-off-by: Dalton Bohning --- .github/workflows/linting.yml | 42 ++ .github/workflows/spelling.yml | 2 + .github/workflows/version-checks.yml | 26 + .github/workflows/yaml.yml | 3 + SConstruct | 10 +- ci/functional/junit_list_unsuccessful | 5 +- ci/gha_helper.py | 4 +- ci/jira_query.py | 7 +- pyproject.toml | 5 + requirements.txt | 4 +- site_scons/components/__init__.py | 3 +- site_scons/env_modules.py | 7 +- site_scons/prereq_tools/base.py | 27 +- site_scons/site_tools/compiler_setup.py | 4 +- site_scons/site_tools/daos_builder.py | 8 +- site_scons/site_tools/doneapi.py | 4 +- site_scons/site_tools/extra/__init__.py | 2 +- site_scons/site_tools/extra/extra.py | 4 +- site_scons/site_tools/go_builder.py | 6 +- site_scons/site_tools/protoc/__init__.py | 2 + site_scons/site_tools/stack_analyzer.py | 3 +- src/cart/SConscript | 1 + src/client/dfuse/dfuse_core.c | 2 +- src/client/pydaos/__init__.py | 1 + src/client/pydaos/pydaos_core.py | 5 +- src/client/pydaos/raw/__init__.py | 11 +- src/client/pydaos/raw/daos_api.py | 11 +- src/client/setup.py | 5 +- src/common/dav/out.h | 3 +- src/control/SConscript | 4 +- .../hashicorp/go-msgpack/codec/test.py | 9 +- src/rdb/raft_tests/raft_tests.py | 4 +- src/tests/ftest/__init__.py | 1 + src/tests/ftest/aggregation/basic.py | 1 + .../ftest/aggregation/dfuse_space_check.py | 3 +- src/tests/ftest/aggregation/io_small.py | 3 +- .../ftest/aggregation/multiple_pool_cont.py | 4 +- src/tests/ftest/aggregation/punching.py | 1 + .../ftest/aggregation/shutdown_restart.py | 2 +- src/tests/ftest/aggregation/throttling.py | 1 + src/tests/ftest/avocado_tests.py | 1 + src/tests/ftest/cart/iv/iv_one_node.py | 8 +- src/tests/ftest/cart/iv/iv_two_node.py | 8 +- src/tests/ftest/cart/util/cart_logparse.py | 2 +- src/tests/ftest/cart/util/cart_logtest.py | 5 +- src/tests/ftest/cart/util/cart_logusage.py | 199 ++++++++ src/tests/ftest/cart/util/cart_utils.py | 15 +- src/tests/ftest/checksum/csum_basic.py | 4 +- .../ftest/checksum/csum_error_logging.py | 1 - src/tests/ftest/config_file_gen.py | 11 +- .../ftest/container/api_basic_attribute.py | 3 +- .../ftest/container/auto_oc_selection.py | 3 +- src/tests/ftest/container/basic_snapshot.py | 3 +- src/tests/ftest/container/boundary.py | 5 +- src/tests/ftest/container/destroy.py | 3 +- .../ftest/container/fill_destroy_loop.py | 3 +- .../container/full_pool_container_create.py | 3 +- src/tests/ftest/container/label.py | 5 +- src/tests/ftest/container/multiple_delete.py | 2 +- src/tests/ftest/container/open.py | 3 +- src/tests/ftest/container/query_attribute.py | 2 +- src/tests/ftest/container/query_properties.py | 3 +- src/tests/ftest/container/snapshot.py | 3 +- .../ftest/control/config_generate_output.py | 4 +- .../ftest/control/config_generate_run.py | 1 - src/tests/ftest/control/daos_agent_config.py | 2 +- src/tests/ftest/control/daos_server_helper.py | 2 +- src/tests/ftest/control/dmg_nvme_scan_test.py | 3 +- .../ftest/control/dmg_pool_query_ranks.py | 2 +- .../ftest/control/dmg_pool_query_test.py | 2 +- src/tests/ftest/control/dmg_storage_query.py | 5 +- .../ftest/control/dmg_storage_scan_scm.py | 2 +- src/tests/ftest/control/dmg_system_cleanup.py | 3 +- .../ftest/control/dmg_system_reformat.py | 3 +- .../ftest/control/dmg_telemetry_io_latency.py | 4 +- src/tests/ftest/control/dmg_telemetry_nvme.py | 2 +- src/tests/ftest/control/log_entry.py | 3 +- src/tests/ftest/control/ms_failover.py | 3 +- src/tests/ftest/control/ms_resilience.py | 3 +- src/tests/ftest/control/ssd_socket.py | 3 +- .../ftest/control/super_block_versioning.py | 2 +- src/tests/ftest/control/version.py | 4 +- src/tests/ftest/daos_racer/parallel.py | 2 +- src/tests/ftest/daos_test/dfs.py | 1 + src/tests/ftest/daos_vol/bigio.py | 2 +- src/tests/ftest/daos_vol/h5_suite.py | 2 +- src/tests/ftest/datamover/copy_procs.py | 1 + src/tests/ftest/datamover/dst_create.py | 3 +- src/tests/ftest/datamover/large_dir.py | 1 + src/tests/ftest/datamover/large_file.py | 1 + src/tests/ftest/datamover/negative.py | 2 +- src/tests/ftest/datamover/negative_space.py | 1 + src/tests/ftest/datamover/obj_small.py | 3 +- src/tests/ftest/datamover/posix_meta_entry.py | 1 + .../ftest/datamover/posix_preserve_props.py | 3 +- src/tests/ftest/datamover/posix_subsets.py | 3 +- src/tests/ftest/datamover/posix_symlinks.py | 1 + src/tests/ftest/datamover/serial_small.py | 3 +- src/tests/ftest/dbench/dbench.py | 2 +- src/tests/ftest/deployment/agent_failure.py | 7 +- src/tests/ftest/deployment/basic_checkout.py | 2 +- .../ftest/deployment/critical_integration.py | 12 +- src/tests/ftest/deployment/disk_failure.py | 5 +- src/tests/ftest/deployment/io_sys_admin.py | 4 +- src/tests/ftest/deployment/ior_per_rank.py | 3 +- src/tests/ftest/deployment/network_failure.py | 8 +- .../ftest/deployment/server_rank_failure.py | 7 +- src/tests/ftest/deployment/target_failure.py | 6 +- src/tests/ftest/dfuse/bash.py | 2 +- src/tests/ftest/dfuse/caching_check.py | 2 +- src/tests/ftest/dfuse/container_type.py | 1 - src/tests/ftest/dfuse/enospace.py | 2 +- src/tests/ftest/dfuse/find.py | 6 +- src/tests/ftest/dfuse/mu_mount.py | 3 +- src/tests/ftest/dfuse/mu_perms.py | 9 +- src/tests/ftest/dfuse/posix_stat.py | 2 +- src/tests/ftest/dfuse/simul.py | 3 +- src/tests/ftest/dfuse/sparse_file.py | 4 +- src/tests/ftest/dtx/basic.py | 3 +- .../offline_rebuild_aggregation.py | 1 + .../ftest/erasurecode/rebuild_disabled.py | 1 + .../erasurecode/rebuild_disabled_single.py | 1 + src/tests/ftest/erasurecode/space_usage.py | 4 +- src/tests/ftest/erasurecode/truncate.py | 2 +- src/tests/ftest/fault_domain/fault_domain.py | 3 +- src/tests/ftest/fault_injection/ec.py | 2 +- src/tests/ftest/fault_injection/pool.py | 1 + src/tests/ftest/harness/advanced.py | 5 +- src/tests/ftest/harness/basic.py | 2 +- src/tests/ftest/harness/config.py | 4 +- src/tests/ftest/harness/core_files.py | 5 +- src/tests/ftest/harness/launch_setup.py | 3 +- src/tests/ftest/harness/skip_list.py | 3 +- src/tests/ftest/harness/slurm.py | 1 - src/tests/ftest/harness/unit.py | 8 +- .../upgrade_downgrade_base.py | 8 +- src/tests/ftest/io/macsio_test.py | 2 +- src/tests/ftest/io/parallel_io.py | 5 +- src/tests/ftest/ior/crash.py | 2 +- src/tests/ftest/ior/intercept_messages.py | 3 +- .../ftest/ior/intercept_messages_pil4dfs.py | 2 +- src/tests/ftest/ior/small.py | 1 - src/tests/ftest/launch.py | 2 + src/tests/ftest/network/cart_self_test.py | 3 +- src/tests/ftest/nvme/fault.py | 2 +- src/tests/ftest/nvme/fragmentation.py | 8 +- src/tests/ftest/nvme/health.py | 5 +- src/tests/ftest/nvme/io_verification.py | 5 +- src/tests/ftest/nvme/object.py | 6 +- src/tests/ftest/nvme/pool_capacity.py | 8 +- src/tests/ftest/nvme/pool_exclude.py | 8 +- src/tests/ftest/nvme/pool_extend.py | 4 +- src/tests/ftest/object/array.py | 5 +- src/tests/ftest/object/create_many_dkeys.py | 7 +- src/tests/ftest/object/fetch_bad_param.py | 3 +- src/tests/ftest/object/integrity.py | 4 +- src/tests/ftest/object/open_bad_param.py | 3 +- src/tests/ftest/object/punch_test.py | 3 +- .../ftest/object/same_key_different_value.py | 3 +- src/tests/ftest/object/update_bad_param.py | 3 +- src/tests/ftest/osa/offline_drain.py | 2 +- src/tests/ftest/osa/offline_extend.py | 2 +- src/tests/ftest/osa/offline_parallel_test.py | 9 +- src/tests/ftest/osa/offline_reintegration.py | 2 +- src/tests/ftest/osa/online_drain.py | 4 +- src/tests/ftest/osa/online_extend.py | 6 +- src/tests/ftest/osa/online_parallel_test.py | 14 +- src/tests/ftest/osa/online_reintegration.py | 8 +- src/tests/ftest/performance/mdtest_hard.py | 1 + src/tests/ftest/pool/api_attribute.py | 3 +- src/tests/ftest/pool/bad_connect.py | 5 +- src/tests/ftest/pool/bad_query.py | 3 +- src/tests/ftest/pool/create.py | 2 +- src/tests/ftest/pool/destroy.py | 5 +- src/tests/ftest/pool/dynamic_server_pool.py | 3 +- src/tests/ftest/pool/evict.py | 3 +- src/tests/ftest/pool/label.py | 5 +- src/tests/ftest/pool/list_pools.py | 3 +- src/tests/ftest/pool/list_verbose.py | 2 +- src/tests/ftest/pool/management_race.py | 7 +- .../ftest/pool/multi_server_create_delete.py | 4 +- src/tests/ftest/pool/permission.py | 5 +- src/tests/ftest/pool/query_attribute.py | 1 + src/tests/ftest/pool/svc.py | 3 +- src/tests/ftest/pool/verify_space.py | 1 - src/tests/ftest/rebuild/cascading_failures.py | 2 +- .../ftest/rebuild/container_create_race.py | 1 - src/tests/ftest/rebuild/no_cap.py | 3 +- src/tests/ftest/rebuild/read_array.py | 2 +- src/tests/ftest/rebuild/widely_striped.py | 1 + src/tests/ftest/scrubber/csum_fault.py | 1 + src/tests/ftest/security/cont_acl.py | 1 + src/tests/ftest/security/cont_delete_acl.py | 1 - src/tests/ftest/security/cont_get_acl.py | 3 +- .../ftest/security/cont_overwrite_acl.py | 3 +- src/tests/ftest/security/cont_update_acl.py | 3 +- src/tests/ftest/security/pool_acl.py | 4 +- src/tests/ftest/security/pool_connect_init.py | 3 +- src/tests/ftest/security/pool_groups.py | 4 +- src/tests/ftest/server/daos_server_restart.py | 2 +- src/tests/ftest/server/metadata.py | 7 +- .../ftest/server/multiengine_persocket.py | 9 +- src/tests/ftest/server/replay.py | 1 - src/tests/ftest/server/storage_tiers.py | 2 +- .../ftest/telemetry/dkey_akey_enum_punch.py | 5 +- src/tests/ftest/util/agent_utils.py | 14 +- src/tests/ftest/util/agent_utils_params.py | 2 +- .../ftest/util/apricot/apricot/__init__.py | 3 +- src/tests/ftest/util/apricot/apricot/test.py | 34 +- src/tests/ftest/util/cart_ctl_utils.py | 3 +- src/tests/ftest/util/cmocka_utils.py | 2 +- src/tests/ftest/util/command_utils.py | 20 +- src/tests/ftest/util/command_utils_base.py | 3 +- src/tests/ftest/util/configuration_utils.py | 3 +- .../ftest/util/cont_security_test_base.py | 7 +- .../ftest/util/container_rf_test_base.py | 4 +- src/tests/ftest/util/control_test_base.py | 3 +- src/tests/ftest/util/daos_core_base.py | 4 +- src/tests/ftest/util/daos_io_conf.py | 2 +- src/tests/ftest/util/daos_perf_utils.py | 2 +- src/tests/ftest/util/daos_racer_utils.py | 7 +- src/tests/ftest/util/daos_utils.py | 2 +- src/tests/ftest/util/daos_utils_base.py | 2 +- src/tests/ftest/util/data_mover_test_base.py | 20 +- src/tests/ftest/util/data_mover_utils.py | 3 +- src/tests/ftest/util/dbench_utils.py | 3 +- src/tests/ftest/util/dfuse_test_base.py | 3 +- src/tests/ftest/util/dfuse_utils.py | 10 +- src/tests/ftest/util/distro_utils.py | 1 + src/tests/ftest/util/dmg_utils.py | 10 +- src/tests/ftest/util/dmg_utils_base.py | 3 +- src/tests/ftest/util/dmg_utils_params.py | 3 +- src/tests/ftest/util/ec_utils.py | 13 +- src/tests/ftest/util/exception_utils.py | 3 +- src/tests/ftest/util/fault_config_utils.py | 3 +- src/tests/ftest/util/file_count_test_base.py | 1 - src/tests/ftest/util/fio_utils.py | 7 +- src/tests/ftest/util/general_utils.py | 15 +- src/tests/ftest/util/io_utilities.py | 5 +- .../ftest/util/ior_intercept_test_base.py | 3 +- src/tests/ftest/util/ior_test_base.py | 5 +- src/tests/ftest/util/ior_utils.py | 6 +- src/tests/ftest/util/job_manager_utils.py | 13 +- src/tests/ftest/util/macsio_test_base.py | 2 +- src/tests/ftest/util/macsio_util.py | 2 +- src/tests/ftest/util/mdtest_test_base.py | 3 +- src/tests/ftest/util/mdtest_utils.py | 2 +- src/tests/ftest/util/mpiio_test_base.py | 5 +- src/tests/ftest/util/mpiio_utils.py | 3 +- src/tests/ftest/util/nvme_utils.py | 5 +- src/tests/ftest/util/osa_utils.py | 11 +- src/tests/ftest/util/performance_test_base.py | 9 +- src/tests/ftest/util/pool_create_all_base.py | 3 +- .../ftest/util/pool_security_test_base.py | 4 +- src/tests/ftest/util/rebuild_test_base.py | 2 +- src/tests/ftest/util/run_utils.py | 5 +- src/tests/ftest/util/scrubber_test_base.py | 2 +- src/tests/ftest/util/security_test_base.py | 1 + src/tests/ftest/util/server_utils.py | 15 +- src/tests/ftest/util/server_utils_base.py | 7 +- src/tests/ftest/util/server_utils_params.py | 3 +- src/tests/ftest/util/slurm_utils.py | 4 +- src/tests/ftest/util/soak_test_base.py | 31 +- src/tests/ftest/util/soak_utils.py | 28 +- src/tests/ftest/util/support_test_base.py | 139 ++++++ src/tests/ftest/util/telemetry_utils.py | 3 +- src/tests/ftest/util/test_utils_base.py | 7 +- src/tests/ftest/util/test_utils_container.py | 8 +- src/tests/ftest/util/test_utils_pool.py | 15 +- src/tests/ftest/util/thread_manager.py | 3 +- src/tests/ftest/util/vol_test_base.py | 4 +- src/tests/ftest/util/write_host_file.py | 2 +- src/tests/ftest/vmd/fault_reintegration.py | 7 +- src/tests/ftest/vmd/led.py | 1 - src/tests/suite/SConscript | 2 +- src/utils/daos_hdlr.c | 6 +- src/vos/storage_estimator/common/dfs_sb.py | 4 +- src/vos/storage_estimator/common/explorer.py | 6 +- src/vos/storage_estimator/common/parse_csv.py | 2 +- .../common/tests/storage_estimator_test.py | 13 +- .../storage_estimator/common/tests/util.py | 2 +- src/vos/storage_estimator/common/util.py | 2 +- src/vos/storage_estimator/common/vos_size.py | 2 +- .../daos_storage_estimator.py | 4 +- src/vos/tests/evt_stress.py | 4 +- utils/ansible/ftest/library/daos_hugepages.py | 2 +- utils/cq/d_logging_check.py | 454 ++++++++++++++++++ utils/cq/daos_pylint.py | 15 +- utils/cq/requirements.txt | 10 +- utils/githooks/pre-commit.d/20-flake.sh | 1 + utils/githooks/pre-commit.d/20-isort.sh | 17 + utils/githooks/pre-commit.d/30-yamllint.sh | 2 +- utils/node_local_test.py | 31 +- utils/run_utest.py | 11 +- 294 files changed, 1509 insertions(+), 697 deletions(-) create mode 100644 .github/workflows/linting.yml create mode 100644 .github/workflows/version-checks.yml create mode 100644 pyproject.toml create mode 100644 src/tests/ftest/cart/util/cart_logusage.py create mode 100644 src/tests/ftest/util/support_test_base.py create mode 100755 utils/cq/d_logging_check.py create mode 100755 utils/githooks/pre-commit.d/20-isort.sh diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 00000000000..500d6abeea8 --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,42 @@ +name: Linting + +# Always run on Pull Requests as then these checks can be marked as required. +on: + push: + branches: + - master + - 'feature/*' + - 'release/*' + pull_request: + +jobs: + # Run isort on the tree. + # This checks .py files only so misses SConstruct and SConscript files are not checked, rather + # for these files check them afterwards. The output-filter will not be installed for this part + # so regressions will be detected but not annotated. + isort: + name: Python isort + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/setup-python@v3 + - uses: isort/isort-action@master + with: + requirementsFiles: "requirements.txt utils/cq/requirements.txt" + - name: Run on SConstruct file. + run: isort --check-only SConstruct + - name: Run on build files. + run: find . -name SConscript | xargs isort --check-only + + log-check: + name: Logging macro checking + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Check DAOS logging macro use. + run: ./utils/cq/d_logging_check.py --github src diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index d02b39405fc..47d6d069c98 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -10,6 +10,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + - name: Install extra python packages + run: pip install --requirement utils/cq/requirements.txt - name: Run check uses: codespell-project/actions-codespell@master with: diff --git a/.github/workflows/version-checks.yml b/.github/workflows/version-checks.yml new file mode 100644 index 00000000000..11d8abf98f8 --- /dev/null +++ b/.github/workflows/version-checks.yml @@ -0,0 +1,26 @@ +name: Version checking + +on: + push: + branches: + - master + +jobs: + upgrade-check: + name: Check for updates + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + package: [pylint, yamllint, isort, codespell] + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Install extra python packages + run: python3 -m pip install --requirement utils/cq/requirements.txt + - name: Check ${{ matrix.package }} version + run: python -m ${{ matrix.package }} --version | tee -a version-pre + - name: Upgrade + run: pip install --upgrade ${{ matrix.package }} + - name: Check ${{ matrix.package }} for version + run: python -m ${{ matrix.package }} --version | diff version-pre - diff --git a/.github/workflows/yaml.yml b/.github/workflows/yaml.yml index 5f52a7bace4..470db020ae3 100644 --- a/.github/workflows/yaml.yml +++ b/.github/workflows/yaml.yml @@ -11,6 +11,7 @@ on: paths: - '**/*.yaml' - '**/*.yml' + - utils/cq/requirements.txt jobs: yaml-lint: @@ -25,5 +26,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3' + - name: Install extra python packages + run: pip install --requirement utils/cq/requirements.txt - name: Run check run: yamllint --format github . diff --git a/SConstruct b/SConstruct index c018e4b223a..9dbcfaf2255 100644 --- a/SConstruct +++ b/SConstruct @@ -1,12 +1,12 @@ """Build DAOS""" +import errno import os -import sys import subprocess # nosec +import sys import time -import errno + import SCons.Warnings -from prereq_tools import PreReqComponent -# pylint: disable=reimported +from prereq_tools import PreReqComponent # pylint: disable=reimported if sys.version_info.major < 3: print(""""Python 2.7 is no longer supported in the DAOS build. @@ -177,8 +177,8 @@ def check_for_release_target(): # pylint: disable=too-many-locals # pylint: disable=consider-using-f-string try: # pylint: disable=import-outside-toplevel - import pygit2 import github + import pygit2 import yaml except ImportError: print("You need yaml, pygit2 and pygithub python modules to create releases") diff --git a/ci/functional/junit_list_unsuccessful b/ci/functional/junit_list_unsuccessful index d7e1fb57894..c1c37602d16 100755 --- a/ci/functional/junit_list_unsuccessful +++ b/ci/functional/junit_list_unsuccessful @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """ list JUnit failures """ -from glob import glob import sys -from junitparser import JUnitXml, Error, Failure +from glob import glob + +from junitparser import Error, Failure, JUnitXml for file in glob(sys.argv[1]): for case in JUnitXml.fromfile(file): diff --git a/ci/gha_helper.py b/ci/gha_helper.py index 13b2fca4d46..ac4e2965b74 100755 --- a/ci/gha_helper.py +++ b/ci/gha_helper.py @@ -3,11 +3,11 @@ """Helper module to choose build/cache keys to use for GitHub actions""" import os -import sys -from os.path import join import random import string import subprocess # nosec +import sys +from os.path import join BUILD_FILES = ['site_scons/prereq_tools', 'site_scons/components', diff --git a/ci/jira_query.py b/ci/jira_query.py index b70c01c448a..8826090b64c 100755 --- a/ci/jira_query.py +++ b/ci/jira_query.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 """Interface between CI and bug-tracking tools""" +import json import os +import random +import string import sys -import json import time import urllib -import random -import string + import jira # Script to improve interaction with Jenkins, GitHub and Jira. This is intended to work in several diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..15ea695e048 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[tool.isort] +supported_extensions = ["py"] +skip = [".git/", "src/rdb/raft", "build", "install", "venv", "src/control/vendor/"] +line_length = 99 +skip_gitignore = true diff --git a/requirements.txt b/requirements.txt index 769762707f0..93ea995a9e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -# Requirements which are not versioned. +# Packages required to build/test DAOS. defusedxml distro jira @@ -8,6 +8,6 @@ ninja pyelftools pyxattr pyyaml -scons # Works around a bug in scons on EL 8. +scons tabulate wheel diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py index 1e764464ec3..76363ed7af0 100644 --- a/site_scons/components/__init__.py +++ b/site_scons/components/__init__.py @@ -21,9 +21,10 @@ """Defines common components used by HPDD projects""" import platform + import distro -from SCons.Script import GetOption from prereq_tools import GitRepoRetriever +from SCons.Script import GetOption # Check if this is an ARM platform PROCESSOR = platform.machine() diff --git a/site_scons/env_modules.py b/site_scons/env_modules.py index 97dba45feb2..ed2b6b295c0 100644 --- a/site_scons/env_modules.py +++ b/site_scons/env_modules.py @@ -19,12 +19,13 @@ # SOFTWARE. """Wrapper for Modules so we can load an MPI before builds or tests""" -import os -import sys import errno -import subprocess # nosec +import os import shutil +import subprocess # nosec +import sys from subprocess import PIPE, Popen # nosec + import distro diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py index 8ba3833f686..e1d86d54899 100644 --- a/site_scons/prereq_tools/base.py +++ b/site_scons/prereq_tools/base.py @@ -20,28 +20,21 @@ # -*- coding: utf-8 -*- """Classes for building external prerequisite components""" -# pylint: disable=too-many-lines -import os -from copy import deepcopy -import sys -import json +import configparser import datetime -import traceback import errno +import json +# pylint: disable=too-many-lines +import os import shutil import subprocess # nosec -import configparser -from SCons.Variables import BoolVariable -from SCons.Variables import EnumVariable -from SCons.Variables import ListVariable -from SCons.Variables import PathVariable -from SCons.Script import Dir -from SCons.Script import Exit -from SCons.Script import GetOption -from SCons.Script import SetOption -from SCons.Script import WhereIs -from SCons.Script import BUILD_TARGETS +import sys +import traceback +from copy import deepcopy + from SCons.Errors import InternalError +from SCons.Script import BUILD_TARGETS, Dir, Exit, GetOption, SetOption, WhereIs +from SCons.Variables import BoolVariable, EnumVariable, ListVariable, PathVariable class DownloadFailure(Exception): diff --git a/site_scons/site_tools/compiler_setup.py b/site_scons/site_tools/compiler_setup.py index 163acf2c406..584eb4e997f 100644 --- a/site_scons/site_tools/compiler_setup.py +++ b/site_scons/site_tools/compiler_setup.py @@ -1,8 +1,6 @@ """Common DAOS library for setting up the compiler""" -from SCons.Script import GetOption, Exit -from SCons.Script import Configure - +from SCons.Script import Configure, Exit, GetOption DESIRED_FLAGS = ['-Wno-gnu-designator', '-Wno-missing-braces', diff --git a/site_scons/site_tools/daos_builder.py b/site_scons/site_tools/daos_builder.py index 36676952ffd..4f06c9c93b2 100644 --- a/site_scons/site_tools/daos_builder.py +++ b/site_scons/site_tools/daos_builder.py @@ -1,13 +1,9 @@ """Common DAOS build functions""" import os -from SCons.Subst import Literal -from SCons.Script import Dir -from SCons.Script import GetOption -from SCons.Script import WhereIs -from SCons.Script import Depends -from SCons.Script import Exit from env_modules import load_mpi +from SCons.Script import Depends, Dir, Exit, GetOption, WhereIs +from SCons.Subst import Literal libraries = {} missing = set() diff --git a/site_scons/site_tools/doneapi.py b/site_scons/site_tools/doneapi.py index b57c569d461..b9938b5a197 100644 --- a/site_scons/site_tools/doneapi.py +++ b/site_scons/site_tools/doneapi.py @@ -3,13 +3,13 @@ Hack to support oneapi version of Intel compilers """ -import sys import os +import sys +import SCons.Errors import SCons.Tool.gcc import SCons.Util import SCons.Warnings -import SCons.Errors # pylint: disable=too-few-public-methods diff --git a/site_scons/site_tools/extra/__init__.py b/site_scons/site_tools/extra/__init__.py index 2003b75110e..f44a9530de1 100644 --- a/site_scons/site_tools/extra/__init__.py +++ b/site_scons/site_tools/extra/__init__.py @@ -7,4 +7,4 @@ # pylint: disable=unused-import -from .extra import generate, exists # noqa: F401 +from .extra import exists, generate # noqa: F401 diff --git a/site_scons/site_tools/extra/extra.py b/site_scons/site_tools/extra/extra.py index b3054b0fc1f..e7d6c38bfee 100755 --- a/site_scons/site_tools/extra/extra.py +++ b/site_scons/site_tools/extra/extra.py @@ -9,9 +9,9 @@ This is used by scons to reformat automatically generated header files to be readable, but also outside of scons by the clang-format commit hook to check the version. """ -import subprocess # nosec -import re import os +import re +import subprocess # nosec import sys from SCons.Builder import Builder diff --git a/site_scons/site_tools/go_builder.py b/site_scons/site_tools/go_builder.py index 6829f688f06..b3706a8976d 100644 --- a/site_scons/site_tools/go_builder.py +++ b/site_scons/site_tools/go_builder.py @@ -1,11 +1,11 @@ """DAOS functions for building go""" -import subprocess # nosec B404 +import json import os import re -import json +import subprocess # nosec B404 -from SCons.Script import Configure, GetOption, Scanner, Glob, Exit, File +from SCons.Script import Configure, Exit, File, GetOption, Glob, Scanner GO_COMPILER = 'go' MIN_GO_VERSION = '1.18.0' diff --git a/site_scons/site_tools/protoc/__init__.py b/site_scons/site_tools/protoc/__init__.py index dea641af3d3..239ab73422c 100644 --- a/site_scons/site_tools/protoc/__init__.py +++ b/site_scons/site_tools/protoc/__init__.py @@ -20,7 +20,9 @@ # SOFTWARE. import os + import SCons.Builder + # pylint: disable=too-few-public-methods,missing-class-docstring diff --git a/site_scons/site_tools/stack_analyzer.py b/site_scons/site_tools/stack_analyzer.py index 46830086f5a..11fe8af1734 100644 --- a/site_scons/site_tools/stack_analyzer.py +++ b/site_scons/site_tools/stack_analyzer.py @@ -5,9 +5,10 @@ Analyze stack usage output """ -import os import argparse import atexit +import os + from SCons.Script import Exit diff --git a/src/cart/SConscript b/src/cart/SConscript index e44751a06b0..9b5b2c8ecef 100644 --- a/src/cart/SConscript +++ b/src/cart/SConscript @@ -4,6 +4,7 @@ # """Build CaRT components""" from datetime import date + import SCons.Action SRC = ['crt_bulk.c', 'crt_context.c', 'crt_corpc.c', diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c index ab0f6b56a89..8c1889d4ac9 100644 --- a/src/client/dfuse/dfuse_core.c +++ b/src/client/dfuse/dfuse_core.c @@ -703,7 +703,7 @@ dfuse_cont_open_by_label(struct dfuse_projection_info *fs_handle, struct dfuse_p rc = daos_cont_open(dfp->dfp_poh, label, DAOS_COO_RO, &dfc->dfs_coh, &c_info, NULL); } if (rc == -DER_NONEXIST) { - DFUSE_TRA_INFO(dfc, "daos_cont_open() failed: " DF_RC, DP_RC(rc)); + DFUSE_TRA_INFO(dfc, "daos_cont_open() failed: " DF_RC, DP_RC(rc)); D_GOTO(err_free, rc = daos_der2errno(rc)); } else if (rc != -DER_SUCCESS) { DFUSE_TRA_ERROR(dfc, "daos_cont_open() failed: " DF_RC, DP_RC(rc)); diff --git a/src/client/pydaos/__init__.py b/src/client/pydaos/__init__.py index c8e4e1c3359..2f13d83dd8d 100644 --- a/src/client/pydaos/__init__.py +++ b/src/client/pydaos/__init__.py @@ -8,6 +8,7 @@ """ import atexit + from . import pydaos_shim # pylint: disable=relative-beyond-top-level,import-self DAOS_MAGIC = 0x7A8A diff --git a/src/client/pydaos/pydaos_core.py b/src/client/pydaos/pydaos_core.py index ddd845b57d9..401b77a0982 100644 --- a/src/client/pydaos/pydaos_core.py +++ b/src/client/pydaos/pydaos_core.py @@ -11,10 +11,7 @@ import enum # pylint: disable-next=relative-beyond-top-level -from . import pydaos_shim -from . import DAOS_MAGIC -from . import PyDError -from . import DaosClient +from . import DAOS_MAGIC, DaosClient, PyDError, pydaos_shim # Import Object class as an enumeration ObjClassID = enum.Enum( diff --git a/src/client/pydaos/raw/__init__.py b/src/client/pydaos/raw/__init__.py index 5bc0b5872cf..991b711d202 100644 --- a/src/client/pydaos/raw/__init__.py +++ b/src/client/pydaos/raw/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2019-2021 Intel Corporation. +# (C) Copyright 2019-2023 Intel Corporation. # # SPDX-License-Identifier: BSD-2-Clause-Patent # @@ -7,9 +7,10 @@ """ # pylint: disable=wildcard-import -from .conversion import * -from .daos_cref import * -from .daos_api import * +from .conversion import * # noqa: F403 +from .daos_api import * # noqa: F403 +from .daos_cref import * # noqa: F403 + # pylint: enable=wildcard-import -__all__ = ["daos_api", "conversion", "daos_cref"] +__all__ = ["daos_api", "conversion", "daos_cref"] # noqa: F405 diff --git a/src/client/pydaos/raw/daos_api.py b/src/client/pydaos/raw/daos_api.py index 9e4c4f93260..d12d53eb355 100644 --- a/src/client/pydaos/raw/daos_api.py +++ b/src/client/pydaos/raw/daos_api.py @@ -8,19 +8,16 @@ # pylint: disable=consider-using-f-string import ctypes -import threading -import os +import enum import inspect +import os import sys +import threading import time -import enum from .. import pydaos_shim # pylint: disable=relative-beyond-top-level - -from . import daos_cref -from . import conversion from .. import DaosClient - +from . import conversion, daos_cref DaosObjClass = enum.Enum( "DaosObjClass", diff --git a/src/client/setup.py b/src/client/setup.py index 03f8f8ae550..668e7871d47 100644 --- a/src/client/setup.py +++ b/src/client/setup.py @@ -8,9 +8,10 @@ If run from within a compiled DAOS source tree this it will detect the install path automatically, otherwise it'll use the defaults. """ -import os import json -from setuptools import setup, find_packages, Extension +import os + +from setuptools import Extension, find_packages, setup def load_conf(): diff --git a/src/common/dav/out.h b/src/common/dav/out.h index 28bb05b7f73..ebe12044db4 100644 --- a/src/common/dav/out.h +++ b/src/common/dav/out.h @@ -40,8 +40,7 @@ /* produce debug/trace output */ #if defined(DAV_EXTRA_DEBUG) -#define DAV_DBG(fmt, ...) - D_DEBUG(DAV_LOG_FAC, fmt "\n", ## __VA_ARGS__) +#define DAV_DBG(fmt, ...) D_DEBUG(DAV_LOG_FAC, fmt "\n", ##__VA_ARGS__) #else #define DAV_DBG(fmt, ...) SUPPRESS_UNUSED(__VA_ARGS__) #endif diff --git a/src/control/SConscript b/src/control/SConscript index a3fe2c80268..933b090b442 100644 --- a/src/control/SConscript +++ b/src/control/SConscript @@ -1,9 +1,9 @@ """Build DAOS Control Plane""" # pylint: disable=too-many-locals import os -from os.path import join -from os import urandom from binascii import b2a_hex +from os import urandom +from os.path import join def is_firmware_mgmt_build(benv): diff --git a/src/control/vendor/github.com/hashicorp/go-msgpack/codec/test.py b/src/control/vendor/github.com/hashicorp/go-msgpack/codec/test.py index 800376f6841..f2c57b06f28 100644 --- a/src/control/vendor/github.com/hashicorp/go-msgpack/codec/test.py +++ b/src/control/vendor/github.com/hashicorp/go-msgpack/codec/test.py @@ -11,7 +11,14 @@ # Ensure all "string" keys are utf strings (else encoded as bytes) -import cbor, msgpack, msgpackrpc, sys, os, threading +import os +import sys +import threading + +import cbor +import msgpack +import msgpackrpc + def get_test_data_list(): # get list with all primitive types, and a combo type diff --git a/src/rdb/raft_tests/raft_tests.py b/src/rdb/raft_tests/raft_tests.py index f7ca5acd4be..0e701f4fd7c 100755 --- a/src/rdb/raft_tests/raft_tests.py +++ b/src/rdb/raft_tests/raft_tests.py @@ -8,10 +8,10 @@ Makefile. Check the output for the number of "not ok" occurrences and return this number as the return code. ''' +import json +import os import subprocess # nosec import sys -import os -import json TEST_NOT_RUN = -1 DIR = os.path.join(os.path.dirname(os.path.relpath(os.path.dirname(__file__))), 'raft') diff --git a/src/tests/ftest/__init__.py b/src/tests/ftest/__init__.py index bbb94ab2c77..b9ad76b7b8f 100644 --- a/src/tests/ftest/__init__.py +++ b/src/tests/ftest/__init__.py @@ -1,3 +1,4 @@ """ftest __init__.py.""" import pkg_resources + pkg_resources.declare_namespace(__name__) diff --git a/src/tests/ftest/aggregation/basic.py b/src/tests/ftest/aggregation/basic.py index 3ead6abfbbf..61b389faaad 100644 --- a/src/tests/ftest/aggregation/basic.py +++ b/src/tests/ftest/aggregation/basic.py @@ -5,6 +5,7 @@ """ import time + from ior_test_base import IorTestBase diff --git a/src/tests/ftest/aggregation/dfuse_space_check.py b/src/tests/ftest/aggregation/dfuse_space_check.py index 824c42af002..f95025328f3 100644 --- a/src/tests/ftest/aggregation/dfuse_space_check.py +++ b/src/tests/ftest/aggregation/dfuse_space_check.py @@ -4,8 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import os +import time + from ior_test_base import IorTestBase diff --git a/src/tests/ftest/aggregation/io_small.py b/src/tests/ftest/aggregation/io_small.py index ebb13d98352..6dd84cf3dbc 100644 --- a/src/tests/ftest/aggregation/io_small.py +++ b/src/tests/ftest/aggregation/io_small.py @@ -5,8 +5,9 @@ """ import time -from ior_test_base import IorTestBase + from general_utils import human_to_bytes +from ior_test_base import IorTestBase class DaosAggregationIOSmall(IorTestBase): diff --git a/src/tests/ftest/aggregation/multiple_pool_cont.py b/src/tests/ftest/aggregation/multiple_pool_cont.py index e6b6b2f7968..41f1a19329c 100644 --- a/src/tests/ftest/aggregation/multiple_pool_cont.py +++ b/src/tests/ftest/aggregation/multiple_pool_cont.py @@ -5,9 +5,9 @@ """ import time -from ior_utils import run_ior -from ior_test_base import IorTestBase from exception_utils import CommandFailure +from ior_test_base import IorTestBase +from ior_utils import run_ior from job_manager_utils import get_job_manager diff --git a/src/tests/ftest/aggregation/punching.py b/src/tests/ftest/aggregation/punching.py index c347f0ee284..b95484a778b 100644 --- a/src/tests/ftest/aggregation/punching.py +++ b/src/tests/ftest/aggregation/punching.py @@ -5,6 +5,7 @@ """ import time + from mdtest_test_base import MdtestBase diff --git a/src/tests/ftest/aggregation/shutdown_restart.py b/src/tests/ftest/aggregation/shutdown_restart.py index b421b0efab1..bea3c2e9eae 100644 --- a/src/tests/ftest/aggregation/shutdown_restart.py +++ b/src/tests/ftest/aggregation/shutdown_restart.py @@ -5,8 +5,8 @@ """ import time -from ior_test_base import IorTestBase from dmg_utils import check_system_query_status +from ior_test_base import IorTestBase class IoAggregation(IorTestBase): diff --git a/src/tests/ftest/aggregation/throttling.py b/src/tests/ftest/aggregation/throttling.py index e343df82912..f40de713773 100644 --- a/src/tests/ftest/aggregation/throttling.py +++ b/src/tests/ftest/aggregation/throttling.py @@ -5,6 +5,7 @@ """ import time + from ior_test_base import IorTestBase from ior_utils import IorCommand, IorMetrics diff --git a/src/tests/ftest/avocado_tests.py b/src/tests/ftest/avocado_tests.py index 0a2c12a8859..42092e3588c 100644 --- a/src/tests/ftest/avocado_tests.py +++ b/src/tests/ftest/avocado_tests.py @@ -5,6 +5,7 @@ """ import time + from apricot import Test diff --git a/src/tests/ftest/cart/iv/iv_one_node.py b/src/tests/ftest/cart/iv/iv_one_node.py index e566d5d36ff..42b2b8380de 100644 --- a/src/tests/ftest/cart/iv/iv_one_node.py +++ b/src/tests/ftest/cart/iv/iv_one_node.py @@ -3,14 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import time -import tempfile +import codecs import json import os +import shlex import struct -import codecs import subprocess # nosec -import shlex +import tempfile +import time import traceback from cart_utils import CartTest diff --git a/src/tests/ftest/cart/iv/iv_two_node.py b/src/tests/ftest/cart/iv/iv_two_node.py index 4c49e9305a9..0ca61fe10b1 100644 --- a/src/tests/ftest/cart/iv/iv_two_node.py +++ b/src/tests/ftest/cart/iv/iv_two_node.py @@ -3,14 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import time -import tempfile +import codecs import json import os +import shlex import struct -import codecs import subprocess # nosec -import shlex +import tempfile +import time from cart_utils import CartTest diff --git a/src/tests/ftest/cart/util/cart_logparse.py b/src/tests/ftest/cart/util/cart_logparse.py index 63cb9f368f6..cfe74ce8b4e 100644 --- a/src/tests/ftest/cart/util/cart_logparse.py +++ b/src/tests/ftest/cart/util/cart_logparse.py @@ -10,10 +10,10 @@ This provides a way of querying CaRT logfiles for processing. """ -from collections import OrderedDict import bz2 import os import re +from collections import OrderedDict class InvalidPid(Exception): diff --git a/src/tests/ftest/cart/util/cart_logtest.py b/src/tests/ftest/cart/util/cart_logtest.py index 09a3a6fc9c6..a76b30653b8 100755 --- a/src/tests/ftest/cart/util/cart_logtest.py +++ b/src/tests/ftest/cart/util/cart_logtest.py @@ -6,13 +6,14 @@ """This provides consistency checking for CaRT log files.""" +import argparse import re import sys import time -import argparse -from collections import OrderedDict, Counter +from collections import Counter, OrderedDict import cart_logparse + HAVE_TABULATE = True try: import tabulate diff --git a/src/tests/ftest/cart/util/cart_logusage.py b/src/tests/ftest/cart/util/cart_logusage.py new file mode 100644 index 00000000000..358718141de --- /dev/null +++ b/src/tests/ftest/cart/util/cart_logusage.py @@ -0,0 +1,199 @@ +# (C) Copyright 2023 Intel Corporation +# +# SPDX-License-Identifier: BSD-2-Clause-Patent + +"""Mark code usage from daos logs + +Track what lines of code has caused logging and use this to generate xml 'coverage' data which +can be rendered in Jenkins and used to identify areas which are not exercised. + +Registered as a callback for all log tracing but saves results across the entire run. +""" + +import json +import os + + +class CodeLoc(): + """Logging data for single code location""" + + def __init__(self, line=None): + self.lineno = 0 + self.count = 0 + self.fault_injected = False + self.no_fault = False + self.allocation = False + if line: + self.lineno = line.lineno + self.allocation = (line.is_calloc() or line.is_realloc()) + self.add(line) + + def add(self, line): + """Record an extra logging instance for this location""" + self.count += 1 + if line.is_fi_site(): + self.fault_injected = True + else: + self.no_fault = True + + def is_fi_location(self): + """Return true if a possible fault injection site""" + return (self.allocation or self.fault_injected) + + def counts(self): + """Return a tuple of (count, possible-count)""" + if not self.is_fi_location(): + return (1, 1) + count = 0 + if self.no_fault: + count += 1 + if self.fault_injected: + count += 1 + return (count, 2) + + def xml_str(self): + """Return a xml string for this line""" + if not self.is_fi_location(): + return f' \n' + + condc = 'condition-coverage="50% (1/2)"' + + taken = '0%' + not_taken = '0%' + if self.no_fault: + taken = '100%' + if self.fault_injected: + not_taken = '100%' + if self.no_fault: + condc = 'condition-coverage="100% (2/2)"' + + return f""" + + + + + +""" + + +class UsageTracer(): + """Save what lines are executed""" + + def __init__(self): + self._files = {} + + def report(self): + """Report per log""" + return + + def load(self, fname): + """Load intermediate data from file""" + with open(fname, 'r') as fd: + idata = json.load(fd) + + data = {} + for (key, value) in idata.items(): + data[key] = {} + # Iterate over files. + for (key2, value2) in value.items(): + data[key][key2] = {} + # Iterate over line numbers. + for (key3, value3) in value2.items(): + new_obj = CodeLoc() + new_obj.lineno = key3 + new_obj.count = value3[0] + new_obj.allocation = value3[1] + new_obj.fault_injected = value3[2] + new_obj.no_fault = value3[3] + new_obj.lineno = key3 + data[key][key2][key3] = new_obj + self._files = data + + def save(self, fname): + """Save intermediate data to file""" + data = {} + # Iterate over directories. + for (key, value) in self._files.items(): + data[key] = {} + # Iterate over files. + for (key2, value2) in value.items(): + data[key][key2] = {} + # Iterate over line numbers. + for (key3, value3) in value2.items(): + data[key][key2][key3] = [value3.count, value3.allocation, + value3.fault_injected, value3.no_fault] + + with open(fname, 'w') as fd: + json.dump(data, fd) + + def report_all(self, fname): + """Report on everything""" + if not self._files: + return + with open(fname, 'w') as fd: + self._save(fd) + + def _save(self, fd): + + fd.write("\n") + fd.write('\n") + fd.write('\n') + fd.write(""" +. + +\n""") + + for (dname, bname) in self._files.items(): + # Patch up two areas of logging in the go code that mis-report source code names. + if not dname.startswith('src'): + if dname == '': + dname = 'src/control/cmd/daos' + else: + parts = dname.split('/') + while parts[0] != 'src': + parts.pop(0) + dname = '/'.join(parts) + + fd.write(f'\n') + fd.write('\n') + + for data in bname: + taken = 0 + possible = 0 + xml = '' + for loc in bname[data].values(): + (ttt, ptt) = loc.counts() + taken += ttt + possible += ptt + xml += loc.xml_str() + rate = taken / possible + fd.write( + f' \n') + fd.write(' \n') + fd.write(' \n') + fd.write(xml) + fd.write(' \n') + fd.write(' \n') + fd.write('\n') + fd.write('\n') + fd.write('\n') + + def add_line(self, line): + """Register a line""" + try: + fname = line.filename + except AttributeError: + return + + dname = os.path.dirname(fname) + bname = os.path.basename(fname) + if dname not in self._files: + self._files[dname] = {} + if bname not in self._files[dname]: + self._files[dname][bname] = {} + lineno = line.lineno + if lineno in self._files[dname][bname]: + self._files[dname][bname][lineno].add(line) + else: + self._files[dname][bname][lineno] = CodeLoc(line=line) diff --git a/src/tests/ftest/cart/util/cart_utils.py b/src/tests/ftest/cart/util/cart_utils.py index cb6712fb55a..5bed4b31617 100644 --- a/src/tests/ftest/cart/util/cart_utils.py +++ b/src/tests/ftest/cart/util/cart_utils.py @@ -3,23 +3,22 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import time +import glob +import logging import os +import re import shlex import subprocess # nosec -import logging -import re -import glob - -from ClusterShell.NodeSet import NodeSet +import time import cart_logparse import cart_logtest from apricot import TestWithoutServers -from run_utils import stop_processes +from ClusterShell.NodeSet import NodeSet from host_utils import get_local_host -from write_host_file import write_host_file from job_manager_utils import Orterun +from run_utils import stop_processes +from write_host_file import write_host_file class CartTest(TestWithoutServers): diff --git a/src/tests/ftest/checksum/csum_basic.py b/src/tests/ftest/checksum/csum_basic.py index bbb396b9f3b..7b4cee25bcd 100644 --- a/src/tests/ftest/checksum/csum_basic.py +++ b/src/tests/ftest/checksum/csum_basic.py @@ -5,10 +5,10 @@ """ import ctypes -from pydaos.raw import (DaosContainer, IORequest, - DaosObj) + from apricot import TestWithServers from general_utils import create_string_buffer +from pydaos.raw import DaosContainer, DaosObj, IORequest class CsumContainerValidation(TestWithServers): diff --git a/src/tests/ftest/checksum/csum_error_logging.py b/src/tests/ftest/checksum/csum_error_logging.py index d3de374da56..9569f2e31ea 100644 --- a/src/tests/ftest/checksum/csum_error_logging.py +++ b/src/tests/ftest/checksum/csum_error_logging.py @@ -5,7 +5,6 @@ """ from avocado import fail_on - from daos_core_base import DaosCoreBase from dmg_utils import get_dmg_smd_info from exception_utils import CommandFailure diff --git a/src/tests/ftest/config_file_gen.py b/src/tests/ftest/config_file_gen.py index c0031d0e4bd..d51e029d48f 100755 --- a/src/tests/ftest/config_file_gen.py +++ b/src/tests/ftest/config_file_gen.py @@ -10,16 +10,13 @@ import logging import sys - from argparse import ArgumentParser, RawDescriptionHelpFormatter + +from util.agent_utils_params import DaosAgentTransportCredentials, DaosAgentYamlParameters from util.command_utils_base import CommonConfig +from util.dmg_utils_params import DmgTransportCredentials, DmgYamlParameters from util.exception_utils import CommandFailure -from util.agent_utils_params import \ - DaosAgentYamlParameters, DaosAgentTransportCredentials -from util.server_utils_params import \ - DaosServerYamlParameters, DaosServerTransportCredentials -from util.dmg_utils_params import \ - DmgYamlParameters, DmgTransportCredentials +from util.server_utils_params import DaosServerTransportCredentials, DaosServerYamlParameters def generate_agent_config(args): diff --git a/src/tests/ftest/container/api_basic_attribute.py b/src/tests/ftest/container/api_basic_attribute.py index 3766114150a..fb6640cf7b9 100644 --- a/src/tests/ftest/container/api_basic_attribute.py +++ b/src/tests/ftest/container/api_basic_attribute.py @@ -5,10 +5,9 @@ ''' import traceback -from pydaos.raw import DaosApiError - from apricot import TestWithServers from general_utils import DaosTestError +from pydaos.raw import DaosApiError from test_utils_base import CallbackHandler diff --git a/src/tests/ftest/container/auto_oc_selection.py b/src/tests/ftest/container/auto_oc_selection.py index bf5e7a16380..cd2188e059e 100644 --- a/src/tests/ftest/container/auto_oc_selection.py +++ b/src/tests/ftest/container/auto_oc_selection.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail class AutoOCSelectionTest(TestWithServers): diff --git a/src/tests/ftest/container/basic_snapshot.py b/src/tests/ftest/container/basic_snapshot.py index cc21c55ef57..f4204735583 100644 --- a/src/tests/ftest/container/basic_snapshot.py +++ b/src/tests/ftest/container/basic_snapshot.py @@ -3,10 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from pydaos.raw import DaosContainer, DaosSnapshot, DaosApiError - from apricot import TestWithServers from general_utils import get_random_bytes +from pydaos.raw import DaosApiError, DaosContainer, DaosSnapshot class BasicSnapshot(TestWithServers): diff --git a/src/tests/ftest/container/boundary.py b/src/tests/ftest/container/boundary.py index ad73b7d2bc6..670cab02593 100644 --- a/src/tests/ftest/container/boundary.py +++ b/src/tests/ftest/container/boundary.py @@ -4,13 +4,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import itertools import random - -from avocado.core.exceptions import TestFail +import time from apricot import TestWithServers +from avocado.core.exceptions import TestFail from general_utils import DaosTestError from thread_manager import ThreadManager diff --git a/src/tests/ftest/container/destroy.py b/src/tests/ftest/container/destroy.py index f5040835ed7..e93c4eac114 100644 --- a/src/tests/ftest/container/destroy.py +++ b/src/tests/ftest/container/destroy.py @@ -6,9 +6,8 @@ import traceback import uuid -from pydaos.raw import DaosApiError - from apricot import TestWithServers +from pydaos.raw import DaosApiError class ContainerDestroyTest(TestWithServers): diff --git a/src/tests/ftest/container/fill_destroy_loop.py b/src/tests/ftest/container/fill_destroy_loop.py index b71f88cc72e..ada553772b6 100644 --- a/src/tests/ftest/container/fill_destroy_loop.py +++ b/src/tests/ftest/container/fill_destroy_loop.py @@ -5,9 +5,8 @@ """ import os -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail from general_utils import bytes_to_human, human_to_bytes from run_utils import run_remote diff --git a/src/tests/ftest/container/full_pool_container_create.py b/src/tests/ftest/container/full_pool_container_create.py index e543ecae176..1874e78acde 100644 --- a/src/tests/ftest/container/full_pool_container_create.py +++ b/src/tests/ftest/container/full_pool_container_create.py @@ -4,8 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import time + from apricot import TestWithServers -from general_utils import get_random_bytes, DaosTestError +from general_utils import DaosTestError, get_random_bytes from test_utils_container import TestContainerData diff --git a/src/tests/ftest/container/label.py b/src/tests/ftest/container/label.py index 27894b18bcf..d9d8ddb6499 100644 --- a/src/tests/ftest/container/label.py +++ b/src/tests/ftest/container/label.py @@ -6,11 +6,10 @@ import string import uuid -from avocado.core.exceptions import TestFail - from apricot import TestWithServers -from general_utils import report_errors, get_random_string +from avocado.core.exceptions import TestFail from exception_utils import CommandFailure +from general_utils import get_random_string, report_errors class ContainerLabelTest(TestWithServers): diff --git a/src/tests/ftest/container/multiple_delete.py b/src/tests/ftest/container/multiple_delete.py index 4c54becd1d7..4abe99ec6af 100644 --- a/src/tests/ftest/container/multiple_delete.py +++ b/src/tests/ftest/container/multiple_delete.py @@ -5,8 +5,8 @@ """ import time -from ior_test_base import IorTestBase from general_utils import DaosTestError +from ior_test_base import IorTestBase SCM_THRESHOLD = 400000 diff --git a/src/tests/ftest/container/open.py b/src/tests/ftest/container/open.py index ba561ef08cb..aa20abf7348 100644 --- a/src/tests/ftest/container/open.py +++ b/src/tests/ftest/container/open.py @@ -6,9 +6,8 @@ import traceback import uuid -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail RESULT_PASS = "PASS" # nosec RESULT_FAIL = "FAIL" diff --git a/src/tests/ftest/container/query_attribute.py b/src/tests/ftest/container/query_attribute.py index 7a8cdd19501..60e137e94e6 100644 --- a/src/tests/ftest/container/query_attribute.py +++ b/src/tests/ftest/container/query_attribute.py @@ -4,10 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import base64 + from apricot import TestWithServers from general_utils import report_errors - # Test container set-attr, get-attr, and list-attrs with different # types of characters. # pylint: disable=anomalous-backslash-in-string diff --git a/src/tests/ftest/container/query_properties.py b/src/tests/ftest/container/query_properties.py index 12cb84b36c0..92393a6d15f 100644 --- a/src/tests/ftest/container/query_properties.py +++ b/src/tests/ftest/container/query_properties.py @@ -5,9 +5,8 @@ ''' import ctypes -from pydaos.raw import daos_cref, DaosApiError, conversion, DaosContPropEnum - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContPropEnum, conversion, daos_cref from test_utils_container import TestContainer diff --git a/src/tests/ftest/container/snapshot.py b/src/tests/ftest/container/snapshot.py index 4b7918f2f71..498d19f7b6f 100644 --- a/src/tests/ftest/container/snapshot.py +++ b/src/tests/ftest/container/snapshot.py @@ -5,10 +5,9 @@ """ import traceback -from pydaos.raw import DaosContainer, DaosSnapshot, DaosApiError, c_uuid_to_str - from apricot import TestWithServers from general_utils import get_random_bytes +from pydaos.raw import DaosApiError, DaosContainer, DaosSnapshot, c_uuid_to_str # pylint: disable=broad-except diff --git a/src/tests/ftest/control/config_generate_output.py b/src/tests/ftest/control/config_generate_output.py index d45f12fea01..a65790e9b73 100644 --- a/src/tests/ftest/control/config_generate_output.py +++ b/src/tests/ftest/control/config_generate_output.py @@ -4,11 +4,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' from collections import defaultdict -import yaml +import yaml from apricot import TestWithServers -from exception_utils import CommandFailure from dmg_utils import DmgCommand +from exception_utils import CommandFailure class ConfigGenerateOutput(TestWithServers): diff --git a/src/tests/ftest/control/config_generate_run.py b/src/tests/ftest/control/config_generate_run.py index ddc26966cf9..571e23c103e 100644 --- a/src/tests/ftest/control/config_generate_run.py +++ b/src/tests/ftest/control/config_generate_run.py @@ -5,7 +5,6 @@ ''' import yaml - from apricot import TestWithServers from server_utils import ServerFailed diff --git a/src/tests/ftest/control/daos_agent_config.py b/src/tests/ftest/control/daos_agent_config.py index fb5d6738b8b..d2f6c69ecc8 100644 --- a/src/tests/ftest/control/daos_agent_config.py +++ b/src/tests/ftest/control/daos_agent_config.py @@ -5,8 +5,8 @@ """ -from apricot import TestWithServers from agent_utils import include_local_host +from apricot import TestWithServers from exception_utils import CommandFailure diff --git a/src/tests/ftest/control/daos_server_helper.py b/src/tests/ftest/control/daos_server_helper.py index 44e6eec9270..dae932edcaf 100644 --- a/src/tests/ftest/control/daos_server_helper.py +++ b/src/tests/ftest/control/daos_server_helper.py @@ -3,9 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import getpass import os import stat -import getpass from apricot import TestWithServers from server_utils import ServerFailed diff --git a/src/tests/ftest/control/dmg_nvme_scan_test.py b/src/tests/ftest/control/dmg_nvme_scan_test.py index 04f3b3bcd02..25a41e14ec2 100644 --- a/src/tests/ftest/control/dmg_nvme_scan_test.py +++ b/src/tests/ftest/control/dmg_nvme_scan_test.py @@ -5,10 +5,9 @@ """ import os +from apricot import TestWithServers from avocado.utils import process - from dmg_utils import DmgCommand -from apricot import TestWithServers class DmgNvmeScanTest(TestWithServers): diff --git a/src/tests/ftest/control/dmg_pool_query_ranks.py b/src/tests/ftest/control/dmg_pool_query_ranks.py index d480f473f2e..d0cadef0754 100644 --- a/src/tests/ftest/control/dmg_pool_query_ranks.py +++ b/src/tests/ftest/control/dmg_pool_query_ranks.py @@ -5,8 +5,8 @@ """ import time -from exception_utils import CommandFailure from control_test_base import ControlTestBase +from exception_utils import CommandFailure class DmgPoolQueryRanks(ControlTestBase): diff --git a/src/tests/ftest/control/dmg_pool_query_test.py b/src/tests/ftest/control/dmg_pool_query_test.py index e71760a4d7b..200ea396cbf 100644 --- a/src/tests/ftest/control/dmg_pool_query_test.py +++ b/src/tests/ftest/control/dmg_pool_query_test.py @@ -5,8 +5,8 @@ """ from copy import deepcopy -from ior_test_base import IorTestBase from control_test_base import ControlTestBase +from ior_test_base import IorTestBase class DmgPoolQueryTest(ControlTestBase, IorTestBase): diff --git a/src/tests/ftest/control/dmg_storage_query.py b/src/tests/ftest/control/dmg_storage_query.py index 9021aeea0bc..f1d8b76bef0 100644 --- a/src/tests/ftest/control/dmg_storage_query.py +++ b/src/tests/ftest/control/dmg_storage_query.py @@ -7,11 +7,10 @@ import re import avocado - from control_test_base import ControlTestBase -from dmg_utils import get_storage_query_pool_info, get_storage_query_device_info +from dmg_utils import get_storage_query_device_info, get_storage_query_pool_info from exception_utils import CommandFailure -from general_utils import list_to_str, dict_to_str +from general_utils import dict_to_str, list_to_str class DmgStorageQuery(ControlTestBase): diff --git a/src/tests/ftest/control/dmg_storage_scan_scm.py b/src/tests/ftest/control/dmg_storage_scan_scm.py index 8f69f9f0edc..4c3373c0d0f 100644 --- a/src/tests/ftest/control/dmg_storage_scan_scm.py +++ b/src/tests/ftest/control/dmg_storage_scan_scm.py @@ -5,8 +5,8 @@ """ import os -from general_utils import pcmd, run_pcmd from control_test_base import ControlTestBase +from general_utils import pcmd, run_pcmd class DmgStorageScanSCMTest(ControlTestBase): diff --git a/src/tests/ftest/control/dmg_system_cleanup.py b/src/tests/ftest/control/dmg_system_cleanup.py index de72667d0c5..9051dd28dab 100644 --- a/src/tests/ftest/control/dmg_system_cleanup.py +++ b/src/tests/ftest/control/dmg_system_cleanup.py @@ -4,9 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from socket import gethostname + +from apricot import TestWithServers from avocado.core.exceptions import TestFail from pydaos.raw import DaosPool -from apricot import TestWithServers class DmgSystemCleanupTest(TestWithServers): diff --git a/src/tests/ftest/control/dmg_system_reformat.py b/src/tests/ftest/control/dmg_system_reformat.py index bdf5d583052..d20354018c8 100644 --- a/src/tests/ftest/control/dmg_system_reformat.py +++ b/src/tests/ftest/control/dmg_system_reformat.py @@ -5,9 +5,8 @@ """ import time -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail from exception_utils import CommandFailure from test_utils_pool import add_pool, get_size_params diff --git a/src/tests/ftest/control/dmg_telemetry_io_latency.py b/src/tests/ftest/control/dmg_telemetry_io_latency.py index 034389270be..82ad47eb2fb 100644 --- a/src/tests/ftest/control/dmg_telemetry_io_latency.py +++ b/src/tests/ftest/control/dmg_telemetry_io_latency.py @@ -4,11 +4,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from general_utils import report_errors from ior_test_base import IorTestBase +from oclass_utils import extract_redundancy_factor from telemetry_test_base import TestWithTelemetry from telemetry_utils import TelemetryUtils -from oclass_utils import extract_redundancy_factor -from general_utils import report_errors def convert_to_number(size): diff --git a/src/tests/ftest/control/dmg_telemetry_nvme.py b/src/tests/ftest/control/dmg_telemetry_nvme.py index d6cdc26197a..0e211d8d617 100644 --- a/src/tests/ftest/control/dmg_telemetry_nvme.py +++ b/src/tests/ftest/control/dmg_telemetry_nvme.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from telemetry_test_base import TestWithTelemetry from apricot import TestWithServers +from telemetry_test_base import TestWithTelemetry from telemetry_utils import TelemetryUtils diff --git a/src/tests/ftest/control/log_entry.py b/src/tests/ftest/control/log_entry.py index 641b49d8e97..c932f775765 100644 --- a/src/tests/ftest/control/log_entry.py +++ b/src/tests/ftest/control/log_entry.py @@ -6,9 +6,8 @@ import contextlib import re -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet from general_utils import get_journalctl, journalctl_time, wait_for_result from run_utils import run_remote diff --git a/src/tests/ftest/control/ms_failover.py b/src/tests/ftest/control/ms_failover.py index 54fc7d90b3a..f12c0fe27e5 100644 --- a/src/tests/ftest/control/ms_failover.py +++ b/src/tests/ftest/control/ms_failover.py @@ -6,9 +6,8 @@ import socket import time -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet class ManagementServiceFailover(TestWithServers): diff --git a/src/tests/ftest/control/ms_resilience.py b/src/tests/ftest/control/ms_resilience.py index e0ca68ed575..4409c7beff6 100644 --- a/src/tests/ftest/control/ms_resilience.py +++ b/src/tests/ftest/control/ms_resilience.py @@ -6,9 +6,8 @@ import socket import time -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet from run_utils import stop_processes diff --git a/src/tests/ftest/control/ssd_socket.py b/src/tests/ftest/control/ssd_socket.py index acebfdeab0f..2a899dfb446 100644 --- a/src/tests/ftest/control/ssd_socket.py +++ b/src/tests/ftest/control/ssd_socket.py @@ -5,8 +5,9 @@ """ import os from textwrap import wrap -from general_utils import pcmd, run_pcmd + from control_test_base import ControlTestBase +from general_utils import pcmd, run_pcmd class SSDSocketTest(ControlTestBase): diff --git a/src/tests/ftest/control/super_block_versioning.py b/src/tests/ftest/control/super_block_versioning.py index 3b8576caf79..5a9141c5f95 100644 --- a/src/tests/ftest/control/super_block_versioning.py +++ b/src/tests/ftest/control/super_block_versioning.py @@ -7,8 +7,8 @@ import os -from general_utils import check_file_exists, pcmd from apricot import TestWithServers +from general_utils import check_file_exists, pcmd class SuperBlockVersioning(TestWithServers): diff --git a/src/tests/ftest/control/version.py b/src/tests/ftest/control/version.py index 93a5110c490..247b7072953 100644 --- a/src/tests/ftest/control/version.py +++ b/src/tests/ftest/control/version.py @@ -3,11 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import re import json +import re from apricot import TestWithServers -from general_utils import run_pcmd, report_errors, append_error +from general_utils import append_error, report_errors, run_pcmd from server_utils_base import DaosServerCommandRunner diff --git a/src/tests/ftest/daos_racer/parallel.py b/src/tests/ftest/daos_racer/parallel.py index bf3116891f9..6dba46ef7d2 100755 --- a/src/tests/ftest/daos_racer/parallel.py +++ b/src/tests/ftest/daos_racer/parallel.py @@ -6,8 +6,8 @@ """ from apricot import TestWithServers -from exception_utils import CommandFailure from daos_racer_utils import DaosRacerCommand +from exception_utils import CommandFailure class DaosRacerParallelTest(TestWithServers): diff --git a/src/tests/ftest/daos_test/dfs.py b/src/tests/ftest/daos_test/dfs.py index 04eb6073666..c5d5d4adaef 100644 --- a/src/tests/ftest/daos_test/dfs.py +++ b/src/tests/ftest/daos_test/dfs.py @@ -5,6 +5,7 @@ """ import os + from daos_core_base import DaosCoreBase diff --git a/src/tests/ftest/daos_vol/bigio.py b/src/tests/ftest/daos_vol/bigio.py index 54bbf925471..75094dc3537 100644 --- a/src/tests/ftest/daos_vol/bigio.py +++ b/src/tests/ftest/daos_vol/bigio.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from vol_test_base import VolTestBase from job_manager_utils import get_job_manager +from vol_test_base import VolTestBase class DaosVolBigIO(VolTestBase): diff --git a/src/tests/ftest/daos_vol/h5_suite.py b/src/tests/ftest/daos_vol/h5_suite.py index a1eba76677b..fd4d339fb86 100644 --- a/src/tests/ftest/daos_vol/h5_suite.py +++ b/src/tests/ftest/daos_vol/h5_suite.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from vol_test_base import VolTestBase from job_manager_utils import get_job_manager +from vol_test_base import VolTestBase class DaosVol(VolTestBase): diff --git a/src/tests/ftest/datamover/copy_procs.py b/src/tests/ftest/datamover/copy_procs.py index 4119fa723ae..ce980f373f4 100644 --- a/src/tests/ftest/datamover/copy_procs.py +++ b/src/tests/ftest/datamover/copy_procs.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' from os.path import join + from data_mover_test_base import DataMoverTestBase diff --git a/src/tests/ftest/datamover/dst_create.py b/src/tests/ftest/datamover/dst_create.py index e06c46d129c..b2956045e4b 100644 --- a/src/tests/ftest/datamover/dst_create.py +++ b/src/tests/ftest/datamover/dst_create.py @@ -6,10 +6,9 @@ from os.path import join import avocado -from pydaos.raw import DaosApiError - from data_mover_test_base import DataMoverTestBase from duns_utils import format_path +from pydaos.raw import DaosApiError class DmvrDstCreate(DataMoverTestBase): diff --git a/src/tests/ftest/datamover/large_dir.py b/src/tests/ftest/datamover/large_dir.py index 720e95f458d..17751a3d5f4 100644 --- a/src/tests/ftest/datamover/large_dir.py +++ b/src/tests/ftest/datamover/large_dir.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import os + from data_mover_test_base import DataMoverTestBase from duns_utils import format_path diff --git a/src/tests/ftest/datamover/large_file.py b/src/tests/ftest/datamover/large_file.py index 355ec5eecc1..6fc9faf03e0 100644 --- a/src/tests/ftest/datamover/large_file.py +++ b/src/tests/ftest/datamover/large_file.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import os + from data_mover_test_base import DataMoverTestBase diff --git a/src/tests/ftest/datamover/negative.py b/src/tests/ftest/datamover/negative.py index 32142ffe654..5d9b36adc60 100644 --- a/src/tests/ftest/datamover/negative.py +++ b/src/tests/ftest/datamover/negative.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from os.path import join import uuid +from os.path import join from data_mover_test_base import DataMoverTestBase from duns_utils import format_path diff --git a/src/tests/ftest/datamover/negative_space.py b/src/tests/ftest/datamover/negative_space.py index 0c9af6cd48b..69d7fd7ce5b 100644 --- a/src/tests/ftest/datamover/negative_space.py +++ b/src/tests/ftest/datamover/negative_space.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' from os.path import join + from data_mover_test_base import DataMoverTestBase diff --git a/src/tests/ftest/datamover/obj_small.py b/src/tests/ftest/datamover/obj_small.py index 80812015198..00fae2a6257 100644 --- a/src/tests/ftest/datamover/obj_small.py +++ b/src/tests/ftest/datamover/obj_small.py @@ -3,10 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from pydaos.raw import DaosApiError import avocado - from data_mover_test_base import DataMoverTestBase +from pydaos.raw import DaosApiError class DmvrObjSmallTest(DataMoverTestBase): diff --git a/src/tests/ftest/datamover/posix_meta_entry.py b/src/tests/ftest/datamover/posix_meta_entry.py index a39b404ac1c..40268fc7485 100644 --- a/src/tests/ftest/datamover/posix_meta_entry.py +++ b/src/tests/ftest/datamover/posix_meta_entry.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from os.path import join + from data_mover_test_base import DataMoverTestBase from exception_utils import CommandFailure diff --git a/src/tests/ftest/datamover/posix_preserve_props.py b/src/tests/ftest/datamover/posix_preserve_props.py index c05e609a4f3..07fa6bd66c0 100644 --- a/src/tests/ftest/datamover/posix_preserve_props.py +++ b/src/tests/ftest/datamover/posix_preserve_props.py @@ -5,11 +5,10 @@ ''' from os.path import join -from pydaos.raw import DaosApiError import avocado - from data_mover_test_base import DataMoverTestBase from duns_utils import format_path +from pydaos.raw import DaosApiError class DmvrPreserveProps(DataMoverTestBase): diff --git a/src/tests/ftest/datamover/posix_subsets.py b/src/tests/ftest/datamover/posix_subsets.py index 791156307cb..ad62672d096 100644 --- a/src/tests/ftest/datamover/posix_subsets.py +++ b/src/tests/ftest/datamover/posix_subsets.py @@ -3,8 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from os.path import join import re +from os.path import join + from data_mover_test_base import DataMoverTestBase diff --git a/src/tests/ftest/datamover/posix_symlinks.py b/src/tests/ftest/datamover/posix_symlinks.py index ae938783870..cdffdd9085e 100644 --- a/src/tests/ftest/datamover/posix_symlinks.py +++ b/src/tests/ftest/datamover/posix_symlinks.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' from os.path import join + from data_mover_test_base import DataMoverTestBase diff --git a/src/tests/ftest/datamover/serial_small.py b/src/tests/ftest/datamover/serial_small.py index afae5e76451..9d1b6243d3f 100644 --- a/src/tests/ftest/datamover/serial_small.py +++ b/src/tests/ftest/datamover/serial_small.py @@ -3,10 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from pydaos.raw import DaosApiError import avocado - from data_mover_test_base import DataMoverTestBase +from pydaos.raw import DaosApiError class DmvrSerialSmall(DataMoverTestBase): diff --git a/src/tests/ftest/dbench/dbench.py b/src/tests/ftest/dbench/dbench.py index 907bcc51d18..05081fb4039 100644 --- a/src/tests/ftest/dbench/dbench.py +++ b/src/tests/ftest/dbench/dbench.py @@ -5,9 +5,9 @@ """ from ClusterShell.NodeSet import NodeSet +from dbench_utils import Dbench from dfuse_test_base import DfuseTestBase from exception_utils import CommandFailure -from dbench_utils import Dbench class DbenchTest(DfuseTestBase): diff --git a/src/tests/ftest/deployment/agent_failure.py b/src/tests/ftest/deployment/agent_failure.py index 13525b53bf7..3bac851f4ef 100644 --- a/src/tests/ftest/deployment/agent_failure.py +++ b/src/tests/ftest/deployment/agent_failure.py @@ -3,16 +3,15 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import os import threading +import time from ClusterShell.NodeSet import NodeSet - +from command_utils_base import CommandFailure +from general_utils import get_journalctl, journalctl_time, report_errors from ior_test_base import IorTestBase from ior_utils import IorCommand -from general_utils import report_errors, get_journalctl, journalctl_time -from command_utils_base import CommandFailure from job_manager_utils import get_job_manager from run_utils import stop_processes diff --git a/src/tests/ftest/deployment/basic_checkout.py b/src/tests/ftest/deployment/basic_checkout.py index 3a700f27223..52a828e8329 100644 --- a/src/tests/ftest/deployment/basic_checkout.py +++ b/src/tests/ftest/deployment/basic_checkout.py @@ -4,9 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from performance_test_base import PerformanceTestBase from data_mover_test_base import DataMoverTestBase from exception_utils import CommandFailure +from performance_test_base import PerformanceTestBase class BasicCheckout(PerformanceTestBase): diff --git a/src/tests/ftest/deployment/critical_integration.py b/src/tests/ftest/deployment/critical_integration.py index a714ed28daf..8afdf18cfcc 100644 --- a/src/tests/ftest/deployment/critical_integration.py +++ b/src/tests/ftest/deployment/critical_integration.py @@ -6,16 +6,12 @@ import json +# Imports need to be split or python fails to import +from apricot import TestWithoutServers, TestWithServers from ClusterShell.NodeSet import NodeSet - -from general_utils import run_command, DaosTestError, get_journalctl, journalctl_time -from ior_test_base import IorTestBase from exception_utils import CommandFailure - -# Imports need to be split or python fails to import -from apricot import TestWithServers -from apricot import TestWithoutServers - +from general_utils import DaosTestError, get_journalctl, journalctl_time, run_command +from ior_test_base import IorTestBase # pylint: disable-next=fixme # TODO Provision all daos nodes using provisioning tool provided by HPCM diff --git a/src/tests/ftest/deployment/disk_failure.py b/src/tests/ftest/deployment/disk_failure.py index 79c37ef3cf7..23f2132171c 100644 --- a/src/tests/ftest/deployment/disk_failure.py +++ b/src/tests/ftest/deployment/disk_failure.py @@ -9,12 +9,11 @@ from avocado import fail_on from ClusterShell.NodeSet import NodeSet - -from dmg_utils import get_storage_query_device_info, get_dmg_response +from dmg_utils import get_dmg_response, get_storage_query_device_info from exception_utils import CommandFailure from general_utils import list_to_str -from test_utils_pool import add_pool from osa_utils import OSAUtils +from test_utils_pool import add_pool class DiskFailureTest(OSAUtils): diff --git a/src/tests/ftest/deployment/io_sys_admin.py b/src/tests/ftest/deployment/io_sys_admin.py index 0c85d01ea5a..2eaaf8be324 100644 --- a/src/tests/ftest/deployment/io_sys_admin.py +++ b/src/tests/ftest/deployment/io_sys_admin.py @@ -6,10 +6,10 @@ import time -from file_count_test_base import FileCountTestBase +import security_test_base as secTestBase from data_mover_test_base import DataMoverTestBase +from file_count_test_base import FileCountTestBase from general_utils import human_to_bytes -import security_test_base as secTestBase from test_utils_pool import check_pool_creation diff --git a/src/tests/ftest/deployment/ior_per_rank.py b/src/tests/ftest/deployment/ior_per_rank.py index 86f53978a31..f914216f326 100644 --- a/src/tests/ftest/deployment/ior_per_rank.py +++ b/src/tests/ftest/deployment/ior_per_rank.py @@ -5,10 +5,9 @@ """ from avocado.core.exceptions import TestFail - +from general_utils import DaosTestError, percent_change from ior_test_base import IorTestBase from ior_utils import IorCommand, IorMetrics -from general_utils import percent_change, DaosTestError class IorPerRank(IorTestBase): diff --git a/src/tests/ftest/deployment/network_failure.py b/src/tests/ftest/deployment/network_failure.py index b52a6c5bf88..8c3de5cd414 100644 --- a/src/tests/ftest/deployment/network_failure.py +++ b/src/tests/ftest/deployment/network_failure.py @@ -6,15 +6,15 @@ import os import time from collections import defaultdict -from ClusterShell.NodeSet import NodeSet +from ClusterShell.NodeSet import NodeSet +from command_utils_base import CommandFailure +from dmg_utils import check_system_query_status +from general_utils import report_errors, run_pcmd from ior_test_base import IorTestBase from ior_utils import IorCommand -from general_utils import report_errors, run_pcmd -from command_utils_base import CommandFailure from job_manager_utils import get_job_manager from network_utils import update_network_interface -from dmg_utils import check_system_query_status class NetworkFailureTest(IorTestBase): diff --git a/src/tests/ftest/deployment/server_rank_failure.py b/src/tests/ftest/deployment/server_rank_failure.py index 8392d4283d7..51d6c2e660f 100644 --- a/src/tests/ftest/deployment/server_rank_failure.py +++ b/src/tests/ftest/deployment/server_rank_failure.py @@ -3,16 +3,15 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import os import threading +import time from ClusterShell.NodeSet import NodeSet - +from command_utils_base import CommandFailure +from general_utils import report_errors from ior_test_base import IorTestBase from ior_utils import IorCommand -from general_utils import report_errors -from command_utils_base import CommandFailure from job_manager_utils import get_job_manager from run_utils import stop_processes diff --git a/src/tests/ftest/deployment/target_failure.py b/src/tests/ftest/deployment/target_failure.py index 16db78cb796..1751148f073 100644 --- a/src/tests/ftest/deployment/target_failure.py +++ b/src/tests/ftest/deployment/target_failure.py @@ -3,14 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import os import threading +import time +from command_utils_base import CommandFailure +from general_utils import report_errors from ior_test_base import IorTestBase from ior_utils import IorCommand -from general_utils import report_errors -from command_utils_base import CommandFailure from job_manager_utils import get_job_manager diff --git a/src/tests/ftest/dfuse/bash.py b/src/tests/ftest/dfuse/bash.py index 4f2b2aa165b..db1b3bbfe24 100644 --- a/src/tests/ftest/dfuse/bash.py +++ b/src/tests/ftest/dfuse/bash.py @@ -4,9 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -from ClusterShell.NodeSet import NodeSet import general_utils +from ClusterShell.NodeSet import NodeSet from dfuse_test_base import DfuseTestBase from exception_utils import CommandFailure diff --git a/src/tests/ftest/dfuse/caching_check.py b/src/tests/ftest/dfuse/caching_check.py index 8806b9d588d..bcd2db544e0 100644 --- a/src/tests/ftest/dfuse/caching_check.py +++ b/src/tests/ftest/dfuse/caching_check.py @@ -4,9 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from general_utils import percent_change from ior_test_base import IorTestBase from ior_utils import IorCommand, IorMetrics -from general_utils import percent_change class DfuseCachingCheck(IorTestBase): diff --git a/src/tests/ftest/dfuse/container_type.py b/src/tests/ftest/dfuse/container_type.py index 0f93a616beb..56ea7a25a08 100644 --- a/src/tests/ftest/dfuse/container_type.py +++ b/src/tests/ftest/dfuse/container_type.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from avocado.core.exceptions import TestFail - from dfuse_test_base import DfuseTestBase diff --git a/src/tests/ftest/dfuse/enospace.py b/src/tests/ftest/dfuse/enospace.py index 5ef5a4cba19..4f4c1136baa 100644 --- a/src/tests/ftest/dfuse/enospace.py +++ b/src/tests/ftest/dfuse/enospace.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import os import errno +import os from dfuse_test_base import DfuseTestBase diff --git a/src/tests/ftest/dfuse/find.py b/src/tests/ftest/dfuse/find.py index 180adc7a89b..f583e303816 100644 --- a/src/tests/ftest/dfuse/find.py +++ b/src/tests/ftest/dfuse/find.py @@ -4,12 +4,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -import sys -import string import random -from ClusterShell.NodeSet import NodeSet +import string +import sys import general_utils +from ClusterShell.NodeSet import NodeSet from dfuse_test_base import DfuseTestBase from dfuse_utils import get_dfuse, start_dfuse from exception_utils import CommandFailure diff --git a/src/tests/ftest/dfuse/mu_mount.py b/src/tests/ftest/dfuse/mu_mount.py index 8cf47b8c9b4..8083cb684db 100644 --- a/src/tests/ftest/dfuse/mu_mount.py +++ b/src/tests/ftest/dfuse/mu_mount.py @@ -7,9 +7,8 @@ from getpass import getuser from ClusterShell.NodeSet import NodeSet - -from run_utils import run_remote, command_as_user from dfuse_test_base import DfuseTestBase +from run_utils import command_as_user, run_remote class DfuseMUMount(DfuseTestBase): diff --git a/src/tests/ftest/dfuse/mu_perms.py b/src/tests/ftest/dfuse/mu_perms.py index 9d571972bc9..f9cbe4450d8 100644 --- a/src/tests/ftest/dfuse/mu_perms.py +++ b/src/tests/ftest/dfuse/mu_perms.py @@ -3,16 +3,15 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -from itertools import product -import time import re +import time +from itertools import product from ClusterShell.NodeSet import NodeSet - from dfuse_test_base import DfuseTestBase -from dfuse_utils import get_dfuse, start_dfuse, VerifyPermsCommand +from dfuse_utils import VerifyPermsCommand, get_dfuse, start_dfuse +from run_utils import command_as_user, run_remote from user_utils import get_chown_command -from run_utils import run_remote, command_as_user class DfuseMUPerms(DfuseTestBase): diff --git a/src/tests/ftest/dfuse/posix_stat.py b/src/tests/ftest/dfuse/posix_stat.py index e8dc75c3bee..57030f6567a 100644 --- a/src/tests/ftest/dfuse/posix_stat.py +++ b/src/tests/ftest/dfuse/posix_stat.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ior_test_base import IorTestBase from general_utils import get_remote_file_size, run_pcmd +from ior_test_base import IorTestBase class POSIXStatTest(IorTestBase): diff --git a/src/tests/ftest/dfuse/simul.py b/src/tests/ftest/dfuse/simul.py index e8b5d920ba5..aef80b5d651 100644 --- a/src/tests/ftest/dfuse/simul.py +++ b/src/tests/ftest/dfuse/simul.py @@ -5,10 +5,11 @@ """ import os + from avocado import fail_on -from exception_utils import MPILoadError from dfuse_test_base import DfuseTestBase from env_modules import load_mpi +from exception_utils import MPILoadError from general_utils import DaosTestError, run_command diff --git a/src/tests/ftest/dfuse/sparse_file.py b/src/tests/ftest/dfuse/sparse_file.py index 70e6460fd67..6b8521614b1 100644 --- a/src/tests/ftest/dfuse/sparse_file.py +++ b/src/tests/ftest/dfuse/sparse_file.py @@ -4,10 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from getpass import getuser import os -import paramiko +from getpass import getuser +import paramiko from general_utils import get_remote_file_size from ior_test_base import IorTestBase diff --git a/src/tests/ftest/dtx/basic.py b/src/tests/ftest/dtx/basic.py index 6c27cbd6377..96ecc16049c 100644 --- a/src/tests/ftest/dtx/basic.py +++ b/src/tests/ftest/dtx/basic.py @@ -6,9 +6,8 @@ import time import traceback -from pydaos.raw import DaosContainer, DaosApiError, c_uuid_to_str - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer, c_uuid_to_str class BasicTxTest(TestWithServers): diff --git a/src/tests/ftest/erasurecode/offline_rebuild_aggregation.py b/src/tests/ftest/erasurecode/offline_rebuild_aggregation.py index 7b5ede44aaf..5dc238f7a2e 100644 --- a/src/tests/ftest/erasurecode/offline_rebuild_aggregation.py +++ b/src/tests/ftest/erasurecode/offline_rebuild_aggregation.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import time + from ec_utils import ErasureCodeIor diff --git a/src/tests/ftest/erasurecode/rebuild_disabled.py b/src/tests/ftest/erasurecode/rebuild_disabled.py index bdbecd288a7..7310dcee731 100644 --- a/src/tests/ftest/erasurecode/rebuild_disabled.py +++ b/src/tests/ftest/erasurecode/rebuild_disabled.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import time + from ec_utils import ErasureCodeIor diff --git a/src/tests/ftest/erasurecode/rebuild_disabled_single.py b/src/tests/ftest/erasurecode/rebuild_disabled_single.py index 1c6fe2dfe5f..ff8f77d3165 100644 --- a/src/tests/ftest/erasurecode/rebuild_disabled_single.py +++ b/src/tests/ftest/erasurecode/rebuild_disabled_single.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import time + from ec_utils import ErasureCodeSingle diff --git a/src/tests/ftest/erasurecode/space_usage.py b/src/tests/ftest/erasurecode/space_usage.py index 9abff06443e..7fa40365e76 100644 --- a/src/tests/ftest/erasurecode/space_usage.py +++ b/src/tests/ftest/erasurecode/space_usage.py @@ -3,9 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ior_test_base import IorTestBase -from data_utils import list_stats, dict_subtract, dict_extract_values +from data_utils import dict_extract_values, dict_subtract, list_stats from general_utils import percent_change +from ior_test_base import IorTestBase from oclass_utils import calculate_ec_targets_used diff --git a/src/tests/ftest/erasurecode/truncate.py b/src/tests/ftest/erasurecode/truncate.py index 3218b5c86b9..dc8ff91f67a 100644 --- a/src/tests/ftest/erasurecode/truncate.py +++ b/src/tests/ftest/erasurecode/truncate.py @@ -6,7 +6,7 @@ import os from fio_test_base import FioBase -from general_utils import run_pcmd, get_remote_file_size +from general_utils import get_remote_file_size, run_pcmd class Ecodtruncate(FioBase): diff --git a/src/tests/ftest/fault_domain/fault_domain.py b/src/tests/ftest/fault_domain/fault_domain.py index 431d055db56..c4e893383f7 100644 --- a/src/tests/ftest/fault_domain/fault_domain.py +++ b/src/tests/ftest/fault_domain/fault_domain.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers, skipForTicket +from ClusterShell.NodeSet import NodeSet class FaultDomain(TestWithServers): diff --git a/src/tests/ftest/fault_injection/ec.py b/src/tests/ftest/fault_injection/ec.py index 7e3411d0eb8..873ddd1741c 100644 --- a/src/tests/ftest/fault_injection/ec.py +++ b/src/tests/ftest/fault_injection/ec.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from ior_test_base import IorTestBase from fio_test_base import FioBase +from ior_test_base import IorTestBase class EcodFaultInjection(IorTestBase, FioBase): diff --git a/src/tests/ftest/fault_injection/pool.py b/src/tests/ftest/fault_injection/pool.py index 05aae07d543..907772d7f36 100644 --- a/src/tests/ftest/fault_injection/pool.py +++ b/src/tests/ftest/fault_injection/pool.py @@ -5,6 +5,7 @@ """ from random import randint + from apricot import TestWithServers from exception_utils import CommandFailure from general_utils import DaosTestError diff --git a/src/tests/ftest/harness/advanced.py b/src/tests/ftest/harness/advanced.py index cc97d60e53b..f1c4a7f79be 100644 --- a/src/tests/ftest/harness/advanced.py +++ b/src/tests/ftest/harness/advanced.py @@ -6,14 +6,13 @@ import os from random import choice -from ClusterShell.NodeSet import NodeSet from apricot import TestWithServers - +from ClusterShell.NodeSet import NodeSet +from dfuse_utils import get_dfuse, start_dfuse from general_utils import get_avocado_config_value from run_utils import run_remote from test_utils_pool import POOL_TIMEOUT_INCREMENT from user_utils import get_chown_command -from dfuse_utils import get_dfuse, start_dfuse class HarnessAdvancedTest(TestWithServers): diff --git a/src/tests/ftest/harness/basic.py b/src/tests/ftest/harness/basic.py index e3997f1054a..e0e39e15c6c 100644 --- a/src/tests/ftest/harness/basic.py +++ b/src/tests/ftest/harness/basic.py @@ -9,7 +9,7 @@ from cmocka_utils import CmockaUtils from command_utils import SubProcessCommand from exception_utils import CommandFailure -from job_manager_utils import Orterun, Mpirun +from job_manager_utils import Mpirun, Orterun class HarnessBasicTest(TestWithoutServers): diff --git a/src/tests/ftest/harness/config.py b/src/tests/ftest/harness/config.py index 7904164c780..e6237c26e0b 100644 --- a/src/tests/ftest/harness/config.py +++ b/src/tests/ftest/harness/config.py @@ -4,11 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import yaml - from apricot import TestWithServers - -from general_utils import nodeset_append_suffix from dmg_utils import DmgCommand +from general_utils import nodeset_append_suffix class HarnessConfigTest(TestWithServers): diff --git a/src/tests/ftest/harness/core_files.py b/src/tests/ftest/harness/core_files.py index 4b3dac67d10..fed038a6a82 100644 --- a/src/tests/ftest/harness/core_files.py +++ b/src/tests/ftest/harness/core_files.py @@ -7,10 +7,9 @@ from random import choice from re import findall -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers -from run_utils import run_remote, run_local, RunException +from ClusterShell.NodeSet import NodeSet +from run_utils import RunException, run_local, run_remote class HarnessCoreFilesTest(TestWithServers): diff --git a/src/tests/ftest/harness/launch_setup.py b/src/tests/ftest/harness/launch_setup.py index 758cafcf891..e4dc1ae0282 100644 --- a/src/tests/ftest/harness/launch_setup.py +++ b/src/tests/ftest/harness/launch_setup.py @@ -4,8 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithoutServers - -from run_utils import run_remote, command_as_user +from run_utils import command_as_user, run_remote from user_utils import get_group_id, get_user_groups diff --git a/src/tests/ftest/harness/skip_list.py b/src/tests/ftest/harness/skip_list.py index e8a73bc959f..fc3e7023cf9 100644 --- a/src/tests/ftest/harness/skip_list.py +++ b/src/tests/ftest/harness/skip_list.py @@ -3,8 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import os import errno +import os + from apricot import Test diff --git a/src/tests/ftest/harness/slurm.py b/src/tests/ftest/harness/slurm.py index a552a662ed9..7924f8b5574 100644 --- a/src/tests/ftest/harness/slurm.py +++ b/src/tests/ftest/harness/slurm.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithoutServers - from host_utils import get_local_host from slurm_utils import sinfo diff --git a/src/tests/ftest/harness/unit.py b/src/tests/ftest/harness/unit.py index 51828a0d03c..1f4524674f6 100644 --- a/src/tests/ftest/harness/unit.py +++ b/src/tests/ftest/harness/unit.py @@ -3,12 +3,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithoutServers -from data_utils import list_unique, list_flatten, list_stats, \ - dict_extract_values, dict_subtract -from run_utils import run_remote, ResultData +from ClusterShell.NodeSet import NodeSet +from data_utils import dict_extract_values, dict_subtract, list_flatten, list_stats, list_unique +from run_utils import ResultData, run_remote class HarnessUnitTest(TestWithoutServers): diff --git a/src/tests/ftest/interoperability/upgrade_downgrade_base.py b/src/tests/ftest/interoperability/upgrade_downgrade_base.py index a168ab1a4d8..bc295372115 100644 --- a/src/tests/ftest/interoperability/upgrade_downgrade_base.py +++ b/src/tests/ftest/interoperability/upgrade_downgrade_base.py @@ -3,17 +3,17 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' +import base64 import os -import traceback import random -import base64 import time +import traceback -from pydaos.raw import DaosApiError -from general_utils import get_random_bytes, pcmd, run_pcmd from agent_utils import include_local_host from command_utils_base import CommandFailure +from general_utils import get_random_bytes, pcmd, run_pcmd from ior_test_base import IorTestBase +from pydaos.raw import DaosApiError class UpgradeDowngradeBase(IorTestBase): diff --git a/src/tests/ftest/io/macsio_test.py b/src/tests/ftest/io/macsio_test.py index 95b003378b9..01daf179712 100644 --- a/src/tests/ftest/io/macsio_test.py +++ b/src/tests/ftest/io/macsio_test.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from general_utils import list_to_str from dfuse_test_base import DfuseTestBase +from general_utils import list_to_str from macsio_test_base import MacsioTestBase diff --git a/src/tests/ftest/io/parallel_io.py b/src/tests/ftest/io/parallel_io.py index db6bfa0df65..d4dbcae06a7 100644 --- a/src/tests/ftest/io/parallel_io.py +++ b/src/tests/ftest/io/parallel_io.py @@ -4,12 +4,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import threading +import os import subprocess # nosec +import threading import time from getpass import getuser -import os - from exception_utils import CommandFailure from fio_test_base import FioBase diff --git a/src/tests/ftest/ior/crash.py b/src/tests/ftest/ior/crash.py index 43f430e4ed1..a41dc920f62 100644 --- a/src/tests/ftest/ior/crash.py +++ b/src/tests/ftest/ior/crash.py @@ -6,8 +6,8 @@ import time -from ior_test_base import IorTestBase from dmg_utils import check_system_query_status +from ior_test_base import IorTestBase class IorCrash(IorTestBase): diff --git a/src/tests/ftest/ior/intercept_messages.py b/src/tests/ftest/ior/intercept_messages.py index 146e623aa17..a431f2fe47b 100644 --- a/src/tests/ftest/ior/intercept_messages.py +++ b/src/tests/ftest/ior/intercept_messages.py @@ -4,8 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import re import os +import re + from ior_test_base import IorTestBase diff --git a/src/tests/ftest/ior/intercept_messages_pil4dfs.py b/src/tests/ftest/ior/intercept_messages_pil4dfs.py index 3cbecf21d09..854907c3f8a 100644 --- a/src/tests/ftest/ior/intercept_messages_pil4dfs.py +++ b/src/tests/ftest/ior/intercept_messages_pil4dfs.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import re import os +import re from ior_test_base import IorTestBase diff --git a/src/tests/ftest/ior/small.py b/src/tests/ftest/ior/small.py index e5efab32da1..e7cb58e2dfd 100644 --- a/src/tests/ftest/ior/small.py +++ b/src/tests/ftest/ior/small.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from avocado.core.exceptions import TestFail - from ior_test_base import IorTestBase diff --git a/src/tests/ftest/launch.py b/src/tests/ftest/launch.py index dc5f9f84247..5650622bf20 100755 --- a/src/tests/ftest/launch.py +++ b/src/tests/ftest/launch.py @@ -4,6 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +# pylint: disable=too-many-lines + import getpass import json import logging diff --git a/src/tests/ftest/network/cart_self_test.py b/src/tests/ftest/network/cart_self_test.py index 003fa17ac99..c3421ed9539 100644 --- a/src/tests/ftest/network/cart_self_test.py +++ b/src/tests/ftest/network/cart_self_test.py @@ -4,10 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithServers - +from command_utils import ExecutableCommand from command_utils_base import EnvironmentVariables, FormattedParameter from exception_utils import CommandFailure -from command_utils import ExecutableCommand from host_utils import get_local_host from job_manager_utils import get_job_manager diff --git a/src/tests/ftest/nvme/fault.py b/src/tests/ftest/nvme/fault.py index 984bcaccf2f..5cb30c3815e 100644 --- a/src/tests/ftest/nvme/fault.py +++ b/src/tests/ftest/nvme/fault.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from nvme_utils import ServerFillUp from exception_utils import CommandFailure +from nvme_utils import ServerFillUp class NvmeFault(ServerFillUp): diff --git a/src/tests/ftest/nvme/fragmentation.py b/src/tests/ftest/nvme/fragmentation.py index 5bda262538c..65dd3406ef6 100644 --- a/src/tests/ftest/nvme/fragmentation.py +++ b/src/tests/ftest/nvme/fragmentation.py @@ -3,16 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import queue import time -from threading import Thread from itertools import product -import queue +from threading import Thread from apricot import TestWithServers -from write_host_file import write_host_file -from ior_utils import IorCommand from exception_utils import CommandFailure +from ior_utils import IorCommand from job_manager_utils import get_job_manager +from write_host_file import write_host_file class NvmeFragmentation(TestWithServers): diff --git a/src/tests/ftest/nvme/health.py b/src/tests/ftest/nvme/health.py index cf06704ea0c..f1de78a7546 100644 --- a/src/tests/ftest/nvme/health.py +++ b/src/tests/ftest/nvme/health.py @@ -7,10 +7,9 @@ from avocado import fail_on from avocado.core.exceptions import TestFail - -from dmg_utils import get_storage_query_pool_info, get_dmg_smd_info -from nvme_utils import ServerFillUp, get_device_ids +from dmg_utils import get_dmg_smd_info, get_storage_query_pool_info from exception_utils import CommandFailure +from nvme_utils import ServerFillUp, get_device_ids class NvmeHealth(ServerFillUp): diff --git a/src/tests/ftest/nvme/io_verification.py b/src/tests/ftest/nvme/io_verification.py index c513ac6cefb..bd660fd2f8a 100644 --- a/src/tests/ftest/nvme/io_verification.py +++ b/src/tests/ftest/nvme/io_verification.py @@ -4,10 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import avocado - -from pydaos.raw import DaosApiError -from ior_test_base import IorTestBase from dmg_utils import check_system_query_status +from ior_test_base import IorTestBase +from pydaos.raw import DaosApiError class NvmeIoVerification(IorTestBase): diff --git a/src/tests/ftest/nvme/object.py b/src/tests/ftest/nvme/object.py index 516c17c0ffd..4c473401731 100644 --- a/src/tests/ftest/nvme/object.py +++ b/src/tests/ftest/nvme/object.py @@ -4,12 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import avocado - -from pydaos.raw import DaosApiError - from apricot import TestWithServers -from thread_manager import ThreadManager from general_utils import report_errors +from pydaos.raw import DaosApiError +from thread_manager import ThreadManager class NvmeObject(TestWithServers): diff --git a/src/tests/ftest/nvme/pool_capacity.py b/src/tests/ftest/nvme/pool_capacity.py index 815ab1910f0..2f88d012092 100644 --- a/src/tests/ftest/nvme/pool_capacity.py +++ b/src/tests/ftest/nvme/pool_capacity.py @@ -3,16 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time +import queue import threading +import time from itertools import product -import queue from apricot import TestWithServers -from write_host_file import write_host_file +from exception_utils import CommandFailure from ior_utils import IorCommand from job_manager_utils import get_job_manager -from exception_utils import CommandFailure +from write_host_file import write_host_file class NvmePoolCapacity(TestWithServers): diff --git a/src/tests/ftest/nvme/pool_exclude.py b/src/tests/ftest/nvme/pool_exclude.py index aad66411fb3..578070c1c47 100644 --- a/src/tests/ftest/nvme/pool_exclude.py +++ b/src/tests/ftest/nvme/pool_exclude.py @@ -3,17 +3,17 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from multiprocessing import Queue -import time import random -import threading import re +import threading +import time +from multiprocessing import Queue from exception_utils import CommandFailure from ior_utils import run_ior, thread_run_ior from job_manager_utils import get_job_manager -from test_utils_pool import add_pool from osa_utils import OSAUtils +from test_utils_pool import add_pool from write_host_file import write_host_file diff --git a/src/tests/ftest/nvme/pool_extend.py b/src/tests/ftest/nvme/pool_extend.py index 44ef7575ddf..18e48ced8a1 100644 --- a/src/tests/ftest/nvme/pool_extend.py +++ b/src/tests/ftest/nvme/pool_extend.py @@ -3,12 +3,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import threading +import time +from dmg_utils import check_system_query_status from osa_utils import OSAUtils from write_host_file import write_host_file -from dmg_utils import check_system_query_status class NvmePoolExtend(OSAUtils): diff --git a/src/tests/ftest/object/array.py b/src/tests/ftest/object/array.py index be7f46807ff..b14e51306de 100644 --- a/src/tests/ftest/object/array.py +++ b/src/tests/ftest/object/array.py @@ -3,13 +3,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' +import logging import time import traceback -import logging - -from pydaos.raw import DaosContainer, DaosApiError, c_uuid_to_str from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer, c_uuid_to_str class ArrayObjTest(TestWithServers): diff --git a/src/tests/ftest/object/create_many_dkeys.py b/src/tests/ftest/object/create_many_dkeys.py index 52d073fe2dd..c4feb0bdf48 100644 --- a/src/tests/ftest/object/create_many_dkeys.py +++ b/src/tests/ftest/object/create_many_dkeys.py @@ -3,14 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import sys import ctypes -import avocado - -from pydaos.raw import DaosContainer, IORequest, DaosApiError +import sys +import avocado from apricot import TestWithServers from general_utils import create_string_buffer +from pydaos.raw import DaosApiError, DaosContainer, IORequest class CreateManyDkeys(TestWithServers): diff --git a/src/tests/ftest/object/fetch_bad_param.py b/src/tests/ftest/object/fetch_bad_param.py index 919fc42b831..8c2f91055f0 100644 --- a/src/tests/ftest/object/fetch_bad_param.py +++ b/src/tests/ftest/object/fetch_bad_param.py @@ -6,9 +6,8 @@ import time import traceback -from pydaos.raw import DaosContainer, DaosApiError - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer class ObjFetchBadParam(TestWithServers): diff --git a/src/tests/ftest/object/integrity.py b/src/tests/ftest/object/integrity.py index 097d7f17460..8f37422b106 100644 --- a/src/tests/ftest/object/integrity.py +++ b/src/tests/ftest/object/integrity.py @@ -5,11 +5,11 @@ """ import ctypes import time -import avocado -from pydaos.raw import (DaosContainer, IORequest, DaosObj, DaosApiError) +import avocado from apricot import TestWithServers from general_utils import create_string_buffer +from pydaos.raw import DaosApiError, DaosContainer, DaosObj, IORequest class ObjectDataValidation(TestWithServers): diff --git a/src/tests/ftest/object/open_bad_param.py b/src/tests/ftest/object/open_bad_param.py index 045fcdd0332..8815d6e4aa2 100644 --- a/src/tests/ftest/object/open_bad_param.py +++ b/src/tests/ftest/object/open_bad_param.py @@ -5,9 +5,8 @@ """ import traceback -from pydaos.raw import DaosContainer, DaosApiError, DaosObjId - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer, DaosObjId class ObjOpenBadParam(TestWithServers): diff --git a/src/tests/ftest/object/punch_test.py b/src/tests/ftest/object/punch_test.py index a55fd8dad03..ca1d300a07d 100644 --- a/src/tests/ftest/object/punch_test.py +++ b/src/tests/ftest/object/punch_test.py @@ -5,9 +5,8 @@ ''' import traceback -from pydaos.raw import DaosContainer, DaosApiError - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer class PunchTest(TestWithServers): diff --git a/src/tests/ftest/object/same_key_different_value.py b/src/tests/ftest/object/same_key_different_value.py index e37c69456d1..a35222c21f4 100644 --- a/src/tests/ftest/object/same_key_different_value.py +++ b/src/tests/ftest/object/same_key_different_value.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from pydaos.raw import DaosApiError - from apricot import TestWithServers +from pydaos.raw import DaosApiError class SameKeyDifferentValue(TestWithServers): diff --git a/src/tests/ftest/object/update_bad_param.py b/src/tests/ftest/object/update_bad_param.py index 6462bdd09ba..5a58a0292af 100644 --- a/src/tests/ftest/object/update_bad_param.py +++ b/src/tests/ftest/object/update_bad_param.py @@ -5,9 +5,8 @@ ''' import traceback -from pydaos.raw import DaosContainer, DaosApiError - from apricot import TestWithServers +from pydaos.raw import DaosApiError, DaosContainer class ObjUpdateBadParam(TestWithServers): diff --git a/src/tests/ftest/osa/offline_drain.py b/src/tests/ftest/osa/offline_drain.py index 0a67edc2f7f..26be76d95a8 100644 --- a/src/tests/ftest/osa/offline_drain.py +++ b/src/tests/ftest/osa/offline_drain.py @@ -5,9 +5,9 @@ """ import random +from nvme_utils import ServerFillUp from osa_utils import OSAUtils from test_utils_pool import add_pool -from nvme_utils import ServerFillUp from write_host_file import write_host_file diff --git a/src/tests/ftest/osa/offline_extend.py b/src/tests/ftest/osa/offline_extend.py index 0f8bcc35fe5..471bdc64fd5 100644 --- a/src/tests/ftest/osa/offline_extend.py +++ b/src/tests/ftest/osa/offline_extend.py @@ -3,9 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from dmg_utils import check_system_query_status from osa_utils import OSAUtils from test_utils_pool import add_pool -from dmg_utils import check_system_query_status class OSAOfflineExtend(OSAUtils): diff --git a/src/tests/ftest/osa/offline_parallel_test.py b/src/tests/ftest/osa/offline_parallel_test.py index 8dd19354dbb..c0d2c7f14a4 100644 --- a/src/tests/ftest/osa/offline_parallel_test.py +++ b/src/tests/ftest/osa/offline_parallel_test.py @@ -3,13 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time -import random -import threading import copy import queue -from osa_utils import OSAUtils +import random +import threading +import time + from dmg_utils import check_system_query_status +from osa_utils import OSAUtils from test_utils_pool import add_pool diff --git a/src/tests/ftest/osa/offline_reintegration.py b/src/tests/ftest/osa/offline_reintegration.py index dab5f237d8b..6491a477d7a 100644 --- a/src/tests/ftest/osa/offline_reintegration.py +++ b/src/tests/ftest/osa/offline_reintegration.py @@ -5,8 +5,8 @@ """ import random -from osa_utils import OSAUtils from nvme_utils import ServerFillUp +from osa_utils import OSAUtils from test_utils_pool import add_pool from write_host_file import write_host_file diff --git a/src/tests/ftest/osa/online_drain.py b/src/tests/ftest/osa/online_drain.py index bd4c26b0b53..bf5804c3767 100644 --- a/src/tests/ftest/osa/online_drain.py +++ b/src/tests/ftest/osa/online_drain.py @@ -3,13 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import random import threading +import time -from write_host_file import write_host_file from osa_utils import OSAUtils from test_utils_pool import add_pool +from write_host_file import write_host_file class OSAOnlineDrain(OSAUtils): diff --git a/src/tests/ftest/osa/online_extend.py b/src/tests/ftest/osa/online_extend.py index a2d6569c528..bb17ecbd388 100644 --- a/src/tests/ftest/osa/online_extend.py +++ b/src/tests/ftest/osa/online_extend.py @@ -3,14 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import threading +import time -from test_utils_pool import add_pool -from write_host_file import write_host_file from daos_racer_utils import DaosRacerCommand from dmg_utils import check_system_query_status from osa_utils import OSAUtils +from test_utils_pool import add_pool +from write_host_file import write_host_file class OSAOnlineExtend(OSAUtils): diff --git a/src/tests/ftest/osa/online_parallel_test.py b/src/tests/ftest/osa/online_parallel_test.py index 6d8f36e35b7..c99a45dbd66 100644 --- a/src/tests/ftest/osa/online_parallel_test.py +++ b/src/tests/ftest/osa/online_parallel_test.py @@ -3,18 +3,18 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time +import copy +import queue import random import threading -import copy - +import time from itertools import product -import queue -from write_host_file import write_host_file -from exception_utils import CommandFailure + +from apricot import skipForTicket from daos_racer_utils import DaosRacerCommand +from exception_utils import CommandFailure from osa_utils import OSAUtils -from apricot import skipForTicket +from write_host_file import write_host_file class OSAOnlineParallelTest(OSAUtils): diff --git a/src/tests/ftest/osa/online_reintegration.py b/src/tests/ftest/osa/online_reintegration.py index 106e9bda2a9..3e4cda5eb3c 100644 --- a/src/tests/ftest/osa/online_reintegration.py +++ b/src/tests/ftest/osa/online_reintegration.py @@ -3,15 +3,15 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time +import queue import random import threading -import queue +import time -from test_utils_pool import add_pool -from write_host_file import write_host_file from daos_racer_utils import DaosRacerCommand from osa_utils import OSAUtils +from test_utils_pool import add_pool +from write_host_file import write_host_file class OSAOnlineReintegration(OSAUtils): diff --git a/src/tests/ftest/performance/mdtest_hard.py b/src/tests/ftest/performance/mdtest_hard.py index 971a2aad471..1d6107670ad 100644 --- a/src/tests/ftest/performance/mdtest_hard.py +++ b/src/tests/ftest/performance/mdtest_hard.py @@ -5,6 +5,7 @@ ''' import os + from performance_test_base import PerformanceTestBase diff --git a/src/tests/ftest/pool/api_attribute.py b/src/tests/ftest/pool/api_attribute.py index d2e0541ba56..f83f6e1e370 100644 --- a/src/tests/ftest/pool/api_attribute.py +++ b/src/tests/ftest/pool/api_attribute.py @@ -5,10 +5,9 @@ ''' import traceback -from pydaos.raw import DaosApiError - from apricot import TestWithServers from general_utils import get_random_bytes +from pydaos.raw import DaosApiError from test_utils_base import CallbackHandler diff --git a/src/tests/ftest/pool/bad_connect.py b/src/tests/ftest/pool/bad_connect.py index 69ac2974651..4d524259c0c 100644 --- a/src/tests/ftest/pool/bad_connect.py +++ b/src/tests/ftest/pool/bad_connect.py @@ -3,10 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import traceback import ctypes -from avocado.core.exceptions import TestFail +import traceback + from apricot import TestWithServers +from avocado.core.exceptions import TestFail class BadConnectTest(TestWithServers): diff --git a/src/tests/ftest/pool/bad_query.py b/src/tests/ftest/pool/bad_query.py index 0b212aaa5b4..964cba97209 100644 --- a/src/tests/ftest/pool/bad_query.py +++ b/src/tests/ftest/pool/bad_query.py @@ -5,9 +5,8 @@ ''' import traceback -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail class BadQueryTest(TestWithServers): diff --git a/src/tests/ftest/pool/create.py b/src/tests/ftest/pool/create.py index 787363ddc3e..433dd88110d 100644 --- a/src/tests/ftest/pool/create.py +++ b/src/tests/ftest/pool/create.py @@ -4,7 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithServers -from test_utils_pool import add_pool, get_size_params, check_pool_creation +from test_utils_pool import add_pool, check_pool_creation, get_size_params class PoolCreateTests(TestWithServers): diff --git a/src/tests/ftest/pool/destroy.py b/src/tests/ftest/pool/destroy.py index a7d5b192925..cec6ef7c9f9 100644 --- a/src/tests/ftest/pool/destroy.py +++ b/src/tests/ftest/pool/destroy.py @@ -5,11 +5,10 @@ """ import os -from avocado.core.exceptions import TestFail - from apricot import TestWithServers -from general_utils import get_default_config_file, check_pool_files +from avocado.core.exceptions import TestFail from dmg_utils import get_dmg_command +from general_utils import check_pool_files, get_default_config_file class DestroyTests(TestWithServers): diff --git a/src/tests/ftest/pool/dynamic_server_pool.py b/src/tests/ftest/pool/dynamic_server_pool.py index 2d48f25e3ad..c4f8c56b2ca 100644 --- a/src/tests/ftest/pool/dynamic_server_pool.py +++ b/src/tests/ftest/pool/dynamic_server_pool.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet from general_utils import check_for_pool diff --git a/src/tests/ftest/pool/evict.py b/src/tests/ftest/pool/evict.py index 285fe81bd56..539d0e2a412 100644 --- a/src/tests/ftest/pool/evict.py +++ b/src/tests/ftest/pool/evict.py @@ -3,12 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from apricot import TestWithServers from avocado.core.exceptions import TestFail from ClusterShell.NodeSet import NodeSet from pydaos.raw import DaosApiError, c_uuid_to_str -from apricot import TestWithServers - class PoolEvictTest(TestWithServers): """ diff --git a/src/tests/ftest/pool/label.py b/src/tests/ftest/pool/label.py index 9b6a3131846..09743aa960e 100644 --- a/src/tests/ftest/pool/label.py +++ b/src/tests/ftest/pool/label.py @@ -5,11 +5,10 @@ """ import string -from avocado.core.exceptions import TestFail - from apricot import TestWithServers -from general_utils import report_errors, get_random_string +from avocado.core.exceptions import TestFail from exception_utils import CommandFailure +from general_utils import get_random_string, report_errors class Label(TestWithServers): diff --git a/src/tests/ftest/pool/list_pools.py b/src/tests/ftest/pool/list_pools.py index c0127b155c8..b1a85a0125d 100644 --- a/src/tests/ftest/pool/list_pools.py +++ b/src/tests/ftest/pool/list_pools.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail from exception_utils import CommandFailure diff --git a/src/tests/ftest/pool/list_verbose.py b/src/tests/ftest/pool/list_verbose.py index f60e01abd48..656ec80c5a6 100644 --- a/src/tests/ftest/pool/list_verbose.py +++ b/src/tests/ftest/pool/list_verbose.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ior_test_base import IorTestBase from general_utils import report_errors +from ior_test_base import IorTestBase class ListVerboseTest(IorTestBase): diff --git a/src/tests/ftest/pool/management_race.py b/src/tests/ftest/pool/management_race.py index 725ced287b0..1a86c31b4e5 100644 --- a/src/tests/ftest/pool/management_race.py +++ b/src/tests/ftest/pool/management_race.py @@ -3,14 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import random - -from pydaos.raw import DaosApiError +import time from apricot import TestWithServers -from thread_manager import ThreadManager from command_utils_base import CommandFailure +from pydaos.raw import DaosApiError +from thread_manager import ThreadManager class PoolManagementRace(TestWithServers): diff --git a/src/tests/ftest/pool/multi_server_create_delete.py b/src/tests/ftest/pool/multi_server_create_delete.py index d2207db693f..0e2fb334cd5 100644 --- a/src/tests/ftest/pool/multi_server_create_delete.py +++ b/src/tests/ftest/pool/multi_server_create_delete.py @@ -5,12 +5,10 @@ """ import os -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet from general_utils import check_for_pool - RESULT_PASS = "PASS" # nosec RESULT_FAIL = "FAIL" diff --git a/src/tests/ftest/pool/permission.py b/src/tests/ftest/pool/permission.py index 3784a02a778..f2f247d9bba 100644 --- a/src/tests/ftest/pool/permission.py +++ b/src/tests/ftest/pool/permission.py @@ -3,10 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from pydaos.raw import DaosContainer, DaosApiError -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail +from pydaos.raw import DaosApiError, DaosContainer RESULT_PASS = "PASS" # nosec RESULT_FAIL = "FAIL" diff --git a/src/tests/ftest/pool/query_attribute.py b/src/tests/ftest/pool/query_attribute.py index e90b631f92e..988e1d18d1b 100644 --- a/src/tests/ftest/pool/query_attribute.py +++ b/src/tests/ftest/pool/query_attribute.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import base64 + from apricot import TestWithServers diff --git a/src/tests/ftest/pool/svc.py b/src/tests/ftest/pool/svc.py index f60a8ed7a7b..c459eba4e8a 100644 --- a/src/tests/ftest/pool/svc.py +++ b/src/tests/ftest/pool/svc.py @@ -3,9 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail class PoolSvc(TestWithServers): diff --git a/src/tests/ftest/pool/verify_space.py b/src/tests/ftest/pool/verify_space.py index a463c6ef5a0..1fd8469fddb 100644 --- a/src/tests/ftest/pool/verify_space.py +++ b/src/tests/ftest/pool/verify_space.py @@ -7,7 +7,6 @@ import re from apricot import TestWithServers - from exception_utils import CommandFailure from general_utils import human_to_bytes from ior_utils import run_ior diff --git a/src/tests/ftest/rebuild/cascading_failures.py b/src/tests/ftest/rebuild/cascading_failures.py index 4edd0064b6a..a17c6c6ffac 100644 --- a/src/tests/ftest/rebuild/cascading_failures.py +++ b/src/tests/ftest/rebuild/cascading_failures.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from rebuild_test_base import RebuildTestBase from daos_utils import DaosCommand +from rebuild_test_base import RebuildTestBase class RbldCascadingFailures(RebuildTestBase): diff --git a/src/tests/ftest/rebuild/container_create_race.py b/src/tests/ftest/rebuild/container_create_race.py index 6684d89ad53..c98103dd00d 100644 --- a/src/tests/ftest/rebuild/container_create_race.py +++ b/src/tests/ftest/rebuild/container_create_race.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from avocado.core.exceptions import TestFail - from ior_test_base import IorTestBase diff --git a/src/tests/ftest/rebuild/no_cap.py b/src/tests/ftest/rebuild/no_cap.py index 8bb773bb1a5..5b7f81fed00 100644 --- a/src/tests/ftest/rebuild/no_cap.py +++ b/src/tests/ftest/rebuild/no_cap.py @@ -5,8 +5,9 @@ ''' from time import sleep, time + from apricot import TestWithServers -from general_utils import get_random_bytes, DaosTestError +from general_utils import DaosTestError, get_random_bytes from test_utils_container import TestContainerData diff --git a/src/tests/ftest/rebuild/read_array.py b/src/tests/ftest/rebuild/read_array.py index 58207839a17..6b72bf45bbd 100644 --- a/src/tests/ftest/rebuild/read_array.py +++ b/src/tests/ftest/rebuild/read_array.py @@ -3,9 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from rebuild_test_base import RebuildTestBase from daos_utils import DaosCommand from general_utils import DaosTestError +from rebuild_test_base import RebuildTestBase class RbldReadArrayTest(RebuildTestBase): diff --git a/src/tests/ftest/rebuild/widely_striped.py b/src/tests/ftest/rebuild/widely_striped.py index 01eee76014c..470926df05d 100644 --- a/src/tests/ftest/rebuild/widely_striped.py +++ b/src/tests/ftest/rebuild/widely_striped.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import time + from mdtest_test_base import MdtestBase diff --git a/src/tests/ftest/scrubber/csum_fault.py b/src/tests/ftest/scrubber/csum_fault.py index 6e1b9a968db..4003f6a13f1 100644 --- a/src/tests/ftest/scrubber/csum_fault.py +++ b/src/tests/ftest/scrubber/csum_fault.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import time + from scrubber_test_base import TestWithScrubber diff --git a/src/tests/ftest/security/cont_acl.py b/src/tests/ftest/security/cont_acl.py index dd3a6a69733..166da1e459b 100644 --- a/src/tests/ftest/security/cont_acl.py +++ b/src/tests/ftest/security/cont_acl.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os + import security_test_base as secTestBase from cont_security_test_base import ContSecurityTestBase from pool_security_test_base import PoolSecurityTestBase diff --git a/src/tests/ftest/security/cont_delete_acl.py b/src/tests/ftest/security/cont_delete_acl.py index 3415c7e2b6a..a8b729fc357 100644 --- a/src/tests/ftest/security/cont_delete_acl.py +++ b/src/tests/ftest/security/cont_delete_acl.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from avocado import fail_on - from cont_security_test_base import ContSecurityTestBase from exception_utils import CommandFailure diff --git a/src/tests/ftest/security/cont_get_acl.py b/src/tests/ftest/security/cont_get_acl.py index 86a675fab5d..b703d7e8d93 100644 --- a/src/tests/ftest/security/cont_get_acl.py +++ b/src/tests/ftest/security/cont_get_acl.py @@ -6,10 +6,9 @@ import os from avocado import fail_on - from cont_security_test_base import ContSecurityTestBase -from security_test_base import read_acl_file from exception_utils import CommandFailure +from security_test_base import read_acl_file class GetContainerACLTest(ContSecurityTestBase): diff --git a/src/tests/ftest/security/cont_overwrite_acl.py b/src/tests/ftest/security/cont_overwrite_acl.py index 9a494c58ebf..10de9a06883 100644 --- a/src/tests/ftest/security/cont_overwrite_acl.py +++ b/src/tests/ftest/security/cont_overwrite_acl.py @@ -6,10 +6,9 @@ import os from avocado import fail_on - from cont_security_test_base import ContSecurityTestBase -from security_test_base import create_acl_file from exception_utils import CommandFailure +from security_test_base import create_acl_file class OverwriteContainerACLTest(ContSecurityTestBase): diff --git a/src/tests/ftest/security/cont_update_acl.py b/src/tests/ftest/security/cont_update_acl.py index c0f971dd260..82a4922ff97 100644 --- a/src/tests/ftest/security/cont_update_acl.py +++ b/src/tests/ftest/security/cont_update_acl.py @@ -6,10 +6,9 @@ import os from avocado import fail_on - from cont_security_test_base import ContSecurityTestBase -from security_test_base import create_acl_file from exception_utils import CommandFailure +from security_test_base import create_acl_file class UpdateContainerACLTest(ContSecurityTestBase): diff --git a/src/tests/ftest/security/pool_acl.py b/src/tests/ftest/security/pool_acl.py index d46635083b4..5d8cc6728c2 100644 --- a/src/tests/ftest/security/pool_acl.py +++ b/src/tests/ftest/security/pool_acl.py @@ -3,12 +3,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' +import grp import os import pwd -import grp -from security_test_base import acl_entry from pool_security_test_base import PoolSecurityTestBase +from security_test_base import acl_entry PERMISSIONS = ["", "r", "w", "rw"] diff --git a/src/tests/ftest/security/pool_connect_init.py b/src/tests/ftest/security/pool_connect_init.py index 70ffd5b33fc..4ddd30ddacb 100644 --- a/src/tests/ftest/security/pool_connect_init.py +++ b/src/tests/ftest/security/pool_connect_init.py @@ -6,9 +6,8 @@ import os import traceback -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail class PoolSecurityTest(TestWithServers): diff --git a/src/tests/ftest/security/pool_groups.py b/src/tests/ftest/security/pool_groups.py index fe02295b75d..f16b9863796 100644 --- a/src/tests/ftest/security/pool_groups.py +++ b/src/tests/ftest/security/pool_groups.py @@ -3,11 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import os import grp +import os -from security_test_base import acl_entry from pool_security_test_base import PoolSecurityTestBase +from security_test_base import acl_entry PERMISSIONS = ["", "r", "w", "rw"] diff --git a/src/tests/ftest/server/daos_server_restart.py b/src/tests/ftest/server/daos_server_restart.py index ce58fcf29f0..be3583f9459 100644 --- a/src/tests/ftest/server/daos_server_restart.py +++ b/src/tests/ftest/server/daos_server_restart.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from avocado import fail_on from apricot import TestWithServers +from avocado import fail_on from exception_utils import CommandFailure from server_utils import ServerFailed diff --git a/src/tests/ftest/server/metadata.py b/src/tests/ftest/server/metadata.py index 50ef7991a99..18cf9edf70f 100644 --- a/src/tests/ftest/server/metadata.py +++ b/src/tests/ftest/server/metadata.py @@ -3,14 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import traceback import time - -from avocado.core.exceptions import TestFail +import traceback from apricot import TestWithServers -from ior_utils import IorCommand +from avocado.core.exceptions import TestFail from exception_utils import CommandFailure +from ior_utils import IorCommand from job_manager_utils import get_job_manager from thread_manager import ThreadManager diff --git a/src/tests/ftest/server/multiengine_persocket.py b/src/tests/ftest/server/multiengine_persocket.py index f838610e91c..7c897415fe9 100644 --- a/src/tests/ftest/server/multiengine_persocket.py +++ b/src/tests/ftest/server/multiengine_persocket.py @@ -6,14 +6,13 @@ import base64 import traceback -from pydaos.raw import DaosApiError - -from general_utils import get_random_bytes, wait_for_result, check_ping, check_ssh -from run_utils import run_remote, run_local +from general_utils import check_ping, check_ssh, get_random_bytes, wait_for_result from ior_test_base import IorTestBase from mdtest_test_base import MdtestBase +from pydaos.raw import DaosApiError +from run_utils import run_local, run_remote from server_utils_base import DaosServerCommand -from storage_utils import StorageInfo, StorageException +from storage_utils import StorageException, StorageInfo class MultiEnginesPerSocketTest(IorTestBase, MdtestBase): diff --git a/src/tests/ftest/server/replay.py b/src/tests/ftest/server/replay.py index ce7cf569952..1b9f08b114c 100644 --- a/src/tests/ftest/server/replay.py +++ b/src/tests/ftest/server/replay.py @@ -7,7 +7,6 @@ import time from apricot import TestWithServers - from dfuse_utils import get_dfuse, start_dfuse, stop_dfuse from general_utils import join from ior_utils import get_ior diff --git a/src/tests/ftest/server/storage_tiers.py b/src/tests/ftest/server/storage_tiers.py index 7db0e495e5b..e627b8d62c2 100644 --- a/src/tests/ftest/server/storage_tiers.py +++ b/src/tests/ftest/server/storage_tiers.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import os -import yaml +import yaml from apricot import TestWithServers from command_utils_base import CommonConfig from server_utils import DaosServerTransportCredentials, DaosServerYamlParameters diff --git a/src/tests/ftest/telemetry/dkey_akey_enum_punch.py b/src/tests/ftest/telemetry/dkey_akey_enum_punch.py index b88d1b0c5c2..99481406d8e 100644 --- a/src/tests/ftest/telemetry/dkey_akey_enum_punch.py +++ b/src/tests/ftest/telemetry/dkey_akey_enum_punch.py @@ -5,10 +5,9 @@ ''' import ctypes -from pydaos.raw import DaosContainer, IORequest, DaosObjClass - -from telemetry_test_base import TestWithTelemetry from general_utils import create_string_buffer +from pydaos.raw import DaosContainer, DaosObjClass, IORequest +from telemetry_test_base import TestWithTelemetry class DkeyAkeyEnumPunch(TestWithTelemetry): diff --git a/src/tests/ftest/util/agent_utils.py b/src/tests/ftest/util/agent_utils.py index 4252d5aeb3d..dec6f0d2e3a 100644 --- a/src/tests/ftest/util/agent_utils.py +++ b/src/tests/ftest/util/agent_utils.py @@ -3,21 +3,17 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import socket -import re import os +import re +import socket from getpass import getuser +from agent_utils_params import DaosAgentTransportCredentials, DaosAgentYamlParameters from ClusterShell.NodeSet import NodeSet - -from command_utils_base import \ - FormattedParameter, EnvironmentVariables, \ - CommonConfig +from command_utils import CommandWithSubCommand, SubprocessManager, YamlCommand +from command_utils_base import CommonConfig, EnvironmentVariables, FormattedParameter from exception_utils import CommandFailure -from command_utils import YamlCommand, CommandWithSubCommand, SubprocessManager from general_utils import get_log_file, run_pcmd -from agent_utils_params import \ - DaosAgentTransportCredentials, DaosAgentYamlParameters def include_local_host(hosts): diff --git a/src/tests/ftest/util/agent_utils_params.py b/src/tests/ftest/util/agent_utils_params.py index c16dedfd443..46b793f31ef 100644 --- a/src/tests/ftest/util/agent_utils_params.py +++ b/src/tests/ftest/util/agent_utils_params.py @@ -5,7 +5,7 @@ """ import os -from command_utils_base import BasicParameter, LogParameter, YamlParameters, TransportCredentials +from command_utils_base import BasicParameter, LogParameter, TransportCredentials, YamlParameters class DaosAgentTransportCredentials(TransportCredentials): diff --git a/src/tests/ftest/util/apricot/apricot/__init__.py b/src/tests/ftest/util/apricot/apricot/__init__.py index 1d7b29f52c9..ad62cf1297a 100644 --- a/src/tests/ftest/util/apricot/apricot/__init__.py +++ b/src/tests/ftest/util/apricot/apricot/__init__.py @@ -1,5 +1,4 @@ """apricot __init__.""" __all__ = ['Test', 'TestWithServers', 'TestWithoutServers', 'skipForTicket'] -from apricot.test import Test, TestWithServers, TestWithoutServers -from apricot.test import skipForTicket +from apricot.test import Test, TestWithoutServers, TestWithServers, skipForTicket diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index f0996196951..aed0ea77ce5 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -5,41 +5,39 @@ """ # pylint: disable=too-many-lines -from ast import literal_eval -import os import json +import os +import random import re import sys +from ast import literal_eval from time import time -import random -from avocado import fail_on, skip, TestFail +from agent_utils import DaosAgentManager, include_local_host from avocado import Test as avocadoTest +from avocado import TestFail, fail_on, skip from avocado.core import exceptions -from ClusterShell.NodeSet import NodeSet -from pydaos.raw import DaosContext, DaosLog, DaosApiError - -from agent_utils import DaosAgentManager, include_local_host from cart_ctl_utils import CartCtl +from ClusterShell.NodeSet import NodeSet from command_utils_base import EnvironmentVariables -from exception_utils import CommandFailure from daos_utils import DaosCommand from distro_utils import detect from dmg_utils import get_dmg_command +from exception_utils import CommandFailure from fault_config_utils import FaultInjection -from general_utils import \ - get_default_config_file, pcmd, get_file_listing, DaosTestError, run_command, \ - dump_engines_stacks, get_avocado_config_value, set_avocado_config_value, \ - nodeset_append_suffix, dict_to_str -from host_utils import get_local_host, get_host_parameters, HostRole, HostInfo, HostException +from general_utils import (DaosTestError, dict_to_str, dump_engines_stacks, + get_avocado_config_value, get_default_config_file, get_file_listing, + nodeset_append_suffix, pcmd, run_command, set_avocado_config_value) +from host_utils import HostException, HostInfo, HostRole, get_host_parameters, get_local_host +from job_manager_utils import get_job_manager from logger_utils import TestLogger +from pydaos.raw import DaosApiError, DaosContext, DaosLog +from run_utils import command_as_user, run_remote, stop_processes from server_utils import DaosServerManager -from run_utils import stop_processes, run_remote, command_as_user -from slurm_utils import get_partition_hosts, get_reservation_hosts, SlurmFailed +from slurm_utils import SlurmFailed, get_partition_hosts, get_reservation_hosts from test_utils_container import TestContainer -from test_utils_pool import LabelGenerator, add_pool, POOL_NAMESPACE +from test_utils_pool import POOL_NAMESPACE, LabelGenerator, add_pool from write_host_file import write_host_file -from job_manager_utils import get_job_manager def skipForTicket(ticket): # pylint: disable=invalid-name diff --git a/src/tests/ftest/util/cart_ctl_utils.py b/src/tests/ftest/util/cart_ctl_utils.py index 5b5182bf21c..4d6272bb21a 100644 --- a/src/tests/ftest/util/cart_ctl_utils.py +++ b/src/tests/ftest/util/cart_ctl_utils.py @@ -4,9 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils_base import FormattedParameter -from command_utils_base import BasicParameter from command_utils import ExecutableCommand +from command_utils_base import BasicParameter, FormattedParameter # pylint: disable=too-few-public-methods,too-many-instance-attributes diff --git a/src/tests/ftest/util/cmocka_utils.py b/src/tests/ftest/util/cmocka_utils.py index 78b8614daff..4bda40e9e95 100644 --- a/src/tests/ftest/util/cmocka_utils.py +++ b/src/tests/ftest/util/cmocka_utils.py @@ -9,7 +9,7 @@ from command_utils import ExecutableCommand from command_utils_base import EnvironmentVariables from exception_utils import CommandFailure -from results_utils import TestName, TestResult, Results, Job, create_xml +from results_utils import Job, Results, TestName, TestResult, create_xml from run_utils import get_clush_command, run_local, run_remote diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index 0744dc21ad9..9c21c21feeb 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -5,27 +5,25 @@ """ # pylint: disable=too-many-lines import contextlib -from datetime import datetime -from getpass import getuser import json -from logging import getLogger import os import re import signal import time +from datetime import datetime +from getpass import getuser +from logging import getLogger from avocado.utils import process from ClusterShell.NodeSet import NodeSet - -from command_utils_base import \ - BasicParameter, CommandWithParameters, EnvironmentVariables, LogParameter, ObjectWithParameters +from command_utils_base import (BasicParameter, CommandWithParameters, EnvironmentVariables, + LogParameter, ObjectWithParameters) from exception_utils import CommandFailure -from general_utils import check_file_exists, \ - run_command, DaosTestError, get_job_manager_class, create_directory, \ - distribute_files, change_file_owner, get_file_listing, run_pcmd, \ - get_subprocess_stdout -from user_utils import get_primary_group +from general_utils import (DaosTestError, change_file_owner, check_file_exists, create_directory, + distribute_files, get_file_listing, get_job_manager_class, + get_subprocess_stdout, run_command, run_pcmd) from run_utils import command_as_user +from user_utils import get_primary_group from yaml_utils import get_yaml_data diff --git a/src/tests/ftest/util/command_utils_base.py b/src/tests/ftest/util/command_utils_base.py index 5f3d4585192..07672cbdf5f 100644 --- a/src/tests/ftest/util/command_utils_base.py +++ b/src/tests/ftest/util/command_utils_base.py @@ -3,8 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import os +from logging import getLogger + import yaml from exception_utils import CommandFailure diff --git a/src/tests/ftest/util/configuration_utils.py b/src/tests/ftest/util/configuration_utils.py index 928caa71264..689ac8456c4 100644 --- a/src/tests/ftest/util/configuration_utils.py +++ b/src/tests/ftest/util/configuration_utils.py @@ -3,13 +3,12 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import re +from logging import getLogger from command_utils_base import BasicParameter, ObjectWithParameters from general_utils import get_host_data - DATA_ERROR = "[ERROR]" diff --git a/src/tests/ftest/util/cont_security_test_base.py b/src/tests/ftest/util/cont_security_test_base.py index 6bfb1ef9801..ce9c3bf93a5 100644 --- a/src/tests/ftest/util/cont_security_test_base.py +++ b/src/tests/ftest/util/cont_security_test_base.py @@ -3,15 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import grp import os import pwd -import grp import re -from avocado.core.exceptions import TestFail - -from apricot import TestWithServers import general_utils +from apricot import TestWithServers +from avocado.core.exceptions import TestFail from general_utils import DaosTestError from security_test_base import acl_entry diff --git a/src/tests/ftest/util/container_rf_test_base.py b/src/tests/ftest/util/container_rf_test_base.py index feb6adc4e06..53fcbe79319 100644 --- a/src/tests/ftest/util/container_rf_test_base.py +++ b/src/tests/ftest/util/container_rf_test_base.py @@ -5,9 +5,9 @@ """ import re -from rebuild_test_base import RebuildTestBase -from general_utils import DaosTestError from daos_utils import DaosCommand +from general_utils import DaosTestError +from rebuild_test_base import RebuildTestBase class ContRedundancyFactor(RebuildTestBase): diff --git a/src/tests/ftest/util/control_test_base.py b/src/tests/ftest/util/control_test_base.py index c99d954e5a9..eff064f53f2 100644 --- a/src/tests/ftest/util/control_test_base.py +++ b/src/tests/ftest/util/control_test_base.py @@ -4,9 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ClusterShell.NodeSet import NodeSet - from apricot import TestWithServers +from ClusterShell.NodeSet import NodeSet class ControlTestBase(TestWithServers): diff --git a/src/tests/ftest/util/daos_core_base.py b/src/tests/ftest/util/daos_core_base.py index cc560e3e0fd..86fbaf3edff 100644 --- a/src/tests/ftest/util/daos_core_base.py +++ b/src/tests/ftest/util/daos_core_base.py @@ -6,12 +6,12 @@ import os import shutil -from avocado import fail_on from apricot import TestWithServers -from general_utils import get_log_file +from avocado import fail_on from cmocka_utils import CmockaUtils from exception_utils import CommandFailure +from general_utils import get_log_file from job_manager_utils import get_job_manager from test_utils_pool import POOL_TIMEOUT_INCREMENT diff --git a/src/tests/ftest/util/daos_io_conf.py b/src/tests/ftest/util/daos_io_conf.py index 4e5dab0dc3e..0f37479b8e8 100644 --- a/src/tests/ftest/util/daos_io_conf.py +++ b/src/tests/ftest/util/daos_io_conf.py @@ -9,8 +9,8 @@ from apricot import TestWithServers from command_utils import ExecutableCommand from command_utils_base import BasicParameter, FormattedParameter -from exception_utils import CommandFailure, MPILoadError from env_modules import load_mpi +from exception_utils import CommandFailure, MPILoadError from job_manager_utils import Orterun diff --git a/src/tests/ftest/util/daos_perf_utils.py b/src/tests/ftest/util/daos_perf_utils.py index b4ad3a87682..c336b73b5c6 100644 --- a/src/tests/ftest/util/daos_perf_utils.py +++ b/src/tests/ftest/util/daos_perf_utils.py @@ -5,8 +5,8 @@ """ import os -from command_utils_base import FormattedParameter from command_utils import ExecutableCommand +from command_utils_base import FormattedParameter class DaosPerfCommand(ExecutableCommand): diff --git a/src/tests/ftest/util/daos_racer_utils.py b/src/tests/ftest/util/daos_racer_utils.py index 5191ddcab8f..a1ebd6e92d6 100644 --- a/src/tests/ftest/util/daos_racer_utils.py +++ b/src/tests/ftest/util/daos_racer_utils.py @@ -6,12 +6,11 @@ import os from ClusterShell.NodeSet import NodeSet - -from command_utils_base import BasicParameter, FormattedParameter -from exception_utils import CommandFailure, MPILoadError from command_utils import ExecutableCommand -from general_utils import pcmd, get_log_file +from command_utils_base import BasicParameter, FormattedParameter from env_modules import load_mpi +from exception_utils import CommandFailure, MPILoadError +from general_utils import get_log_file, pcmd class DaosRacerCommand(ExecutableCommand): diff --git a/src/tests/ftest/util/daos_utils.py b/src/tests/ftest/util/daos_utils.py index 429945f8b00..679ca432414 100644 --- a/src/tests/ftest/util/daos_utils.py +++ b/src/tests/ftest/util/daos_utils.py @@ -7,7 +7,7 @@ import traceback from daos_utils_base import DaosCommandBase -from general_utils import list_to_str, dict_to_str +from general_utils import dict_to_str, list_to_str class DaosCommand(DaosCommandBase): diff --git a/src/tests/ftest/util/daos_utils_base.py b/src/tests/ftest/util/daos_utils_base.py index 7cec7fefece..9130b214dac 100644 --- a/src/tests/ftest/util/daos_utils_base.py +++ b/src/tests/ftest/util/daos_utils_base.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils_base import FormattedParameter, CommandWithParameters, BasicParameter from command_utils import CommandWithSubCommand +from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter class DaosCommandBase(CommandWithSubCommand): diff --git a/src/tests/ftest/util/data_mover_test_base.py b/src/tests/ftest/util/data_mover_test_base.py index 88eb2d21577..4a839bb587f 100644 --- a/src/tests/ftest/util/data_mover_test_base.py +++ b/src/tests/ftest/util/data_mover_test_base.py @@ -3,24 +3,22 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import ctypes # pylint: disable=too-many-lines import os -from os.path import join import re -import ctypes - -from pydaos.raw import str_to_c_uuid, DaosContainer, DaosObj, IORequest +from os.path import join +from command_utils_base import BasicParameter, EnvironmentVariables +from data_mover_utils import (ContClone, DcpCommand, DdeserializeCommand, DserializeCommand, + DsyncCommand, FsCopy, uuid_from_obj) +from duns_utils import format_path from exception_utils import CommandFailure -from test_utils_container import TestContainer +from general_utils import create_string_buffer, get_log_file from ior_test_base import IorTestBase from mdtest_test_base import MdtestBase -from data_mover_utils import DcpCommand, DsyncCommand, FsCopy, ContClone -from data_mover_utils import DserializeCommand, DdeserializeCommand -from data_mover_utils import uuid_from_obj -from duns_utils import format_path -from general_utils import create_string_buffer, get_log_file -from command_utils_base import BasicParameter, EnvironmentVariables +from pydaos.raw import DaosContainer, DaosObj, IORequest, str_to_c_uuid +from test_utils_container import TestContainer class DataMoverTestBase(IorTestBase, MdtestBase): diff --git a/src/tests/ftest/util/data_mover_utils.py b/src/tests/ftest/util/data_mover_utils.py index b07ee3af98a..15c89b6651b 100644 --- a/src/tests/ftest/util/data_mover_utils.py +++ b/src/tests/ftest/util/data_mover_utils.py @@ -4,9 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils_base import FormattedParameter -from command_utils_base import BasicParameter from command_utils import ExecutableCommand +from command_utils_base import BasicParameter, FormattedParameter from job_manager_utils import Mpirun diff --git a/src/tests/ftest/util/dbench_utils.py b/src/tests/ftest/util/dbench_utils.py index 897f77efe13..40313dd6d1c 100644 --- a/src/tests/ftest/util/dbench_utils.py +++ b/src/tests/ftest/util/dbench_utils.py @@ -5,9 +5,8 @@ """ -from command_utils_base import FormattedParameter -from command_utils_base import BasicParameter from command_utils import ExecutableCommand +from command_utils_base import BasicParameter, FormattedParameter from job_manager_utils import Mpirun diff --git a/src/tests/ftest/util/dfuse_test_base.py b/src/tests/ftest/util/dfuse_test_base.py index 16d5e9aeb1c..df1f6c8d721 100644 --- a/src/tests/ftest/util/dfuse_test_base.py +++ b/src/tests/ftest/util/dfuse_test_base.py @@ -4,10 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from agent_utils import include_local_host - from apricot import TestWithServers -from exception_utils import CommandFailure from dfuse_utils import get_dfuse, start_dfuse +from exception_utils import CommandFailure class DfuseTestBase(TestWithServers): diff --git a/src/tests/ftest/util/dfuse_utils.py b/src/tests/ftest/util/dfuse_utils.py index 17c67f7be79..afea1ae187b 100644 --- a/src/tests/ftest/util/dfuse_utils.py +++ b/src/tests/ftest/util/dfuse_utils.py @@ -4,15 +4,15 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import time import os -from ClusterShell.NodeSet import NodeSet +import time -from command_utils_base import FormattedParameter, BasicParameter -from exception_utils import CommandFailure +from ClusterShell.NodeSet import NodeSet from command_utils import ExecutableCommand +from command_utils_base import BasicParameter, FormattedParameter +from exception_utils import CommandFailure from general_utils import check_file_exists, get_log_file -from run_utils import run_remote, command_as_user +from run_utils import command_as_user, run_remote class DfuseCommand(ExecutableCommand): diff --git a/src/tests/ftest/util/distro_utils.py b/src/tests/ftest/util/distro_utils.py index 90565365853..78d26e651f1 100644 --- a/src/tests/ftest/util/distro_utils.py +++ b/src/tests/ftest/util/distro_utils.py @@ -4,6 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ import re + # pylint: disable-next=wildcard-import, unused-wildcard-import from avocado.utils.distro import * # noqa: F403 diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index 43192a6c7aa..b1680d17e46 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -3,16 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import re +from grp import getgrgid # pylint: disable=too-many-lines from logging import getLogger -from grp import getgrgid from pwd import getpwuid -import re -from exception_utils import CommandFailure from dmg_utils_base import DmgCommandBase -from general_utils import get_numeric_list, dict_to_str -from dmg_utils_params import DmgYamlParameters, DmgTransportCredentials +from dmg_utils_params import DmgTransportCredentials, DmgYamlParameters +from exception_utils import CommandFailure +from general_utils import dict_to_str, get_numeric_list class DmgJsonCommandFailure(CommandFailure): diff --git a/src/tests/ftest/util/dmg_utils_base.py b/src/tests/ftest/util/dmg_utils_base.py index b563f8ec622..78c685eb3dd 100644 --- a/src/tests/ftest/util/dmg_utils_base.py +++ b/src/tests/ftest/util/dmg_utils_base.py @@ -6,9 +6,8 @@ from socket import gethostname from ClusterShell.NodeSet import NodeSet - -from command_utils_base import FormattedParameter, CommandWithParameters, BasicParameter from command_utils import CommandWithSubCommand, YamlCommand +from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter from general_utils import nodeset_append_suffix diff --git a/src/tests/ftest/util/dmg_utils_params.py b/src/tests/ftest/util/dmg_utils_params.py index 248b398d163..ff9048a3b62 100644 --- a/src/tests/ftest/util/dmg_utils_params.py +++ b/src/tests/ftest/util/dmg_utils_params.py @@ -4,8 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils_base import \ - BasicParameter, LogParameter, YamlParameters, TransportCredentials +from command_utils_base import BasicParameter, LogParameter, TransportCredentials, YamlParameters class DmgTransportCredentials(TransportCredentials): diff --git a/src/tests/ftest/util/ec_utils.py b/src/tests/ftest/util/ec_utils.py index df50dc49881..d706b32320f 100644 --- a/src/tests/ftest/util/ec_utils.py +++ b/src/tests/ftest/util/ec_utils.py @@ -3,20 +3,19 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import queue import re import threading -import queue import time -from pydaos.raw import DaosApiError - -from nvme_utils import ServerFillUp -from daos_utils import DaosCommand from apricot import TestWithServers -from mdtest_test_base import MdtestBase -from fio_test_base import FioBase +from daos_utils import DaosCommand from exception_utils import CommandFailure +from fio_test_base import FioBase from general_utils import DaosTestError, run_pcmd +from mdtest_test_base import MdtestBase +from nvme_utils import ServerFillUp +from pydaos.raw import DaosApiError def get_data_parity_number(log, oclass): diff --git a/src/tests/ftest/util/exception_utils.py b/src/tests/ftest/util/exception_utils.py index 3f1be6dfdc3..d50b73a0167 100644 --- a/src/tests/ftest/util/exception_utils.py +++ b/src/tests/ftest/util/exception_utils.py @@ -5,7 +5,8 @@ """ import os -from env_modules import show_avail, get_module_list + +from env_modules import get_module_list, show_avail from general_utils import run_command diff --git a/src/tests/ftest/util/fault_config_utils.py b/src/tests/ftest/util/fault_config_utils.py index e0f6b4e37e3..d398997320e 100644 --- a/src/tests/ftest/util/fault_config_utils.py +++ b/src/tests/ftest/util/fault_config_utils.py @@ -5,8 +5,9 @@ """ import os + import yaml -from general_utils import distribute_files, run_command, DaosTestError +from general_utils import DaosTestError, distribute_files, run_command from run_utils import get_clush_command # a lookup table of predefined faults diff --git a/src/tests/ftest/util/file_count_test_base.py b/src/tests/ftest/util/file_count_test_base.py index 90793fc8ec2..88e1ceb1f27 100644 --- a/src/tests/ftest/util/file_count_test_base.py +++ b/src/tests/ftest/util/file_count_test_base.py @@ -4,7 +4,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from avocado.core.exceptions import TestFail - from ior_test_base import IorTestBase from mdtest_test_base import MdtestBase from oclass_utils import extract_redundancy_factor diff --git a/src/tests/ftest/util/fio_utils.py b/src/tests/ftest/util/fio_utils.py index 6c8e1bc31aa..7a0fa3a60ef 100644 --- a/src/tests/ftest/util/fio_utils.py +++ b/src/tests/ftest/util/fio_utils.py @@ -4,11 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from ClusterShell.NodeSet import NodeSet - -from run_utils import run_remote -from command_utils_base import BasicParameter, FormattedParameter, CommandWithParameters -from exception_utils import CommandFailure from command_utils import ExecutableCommand +from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter +from exception_utils import CommandFailure +from run_utils import run_remote class FioCommand(ExecutableCommand): diff --git a/src/tests/ftest/util/general_utils.py b/src/tests/ftest/util/general_utils.py index 7622853c148..754d80c7411 100644 --- a/src/tests/ftest/util/general_utils.py +++ b/src/tests/ftest/util/general_utils.py @@ -5,27 +5,26 @@ """ # pylint: disable=too-many-lines -from logging import getLogger +import ctypes +import math import os -import re import random +import re import string import time -import ctypes -import math +from datetime import datetime from getpass import getuser from importlib import import_module +from logging import getLogger from socket import gethostname -from datetime import datetime from avocado.core.settings import settings from avocado.core.version import MAJOR from avocado.utils import process -from ClusterShell.Task import task_self from ClusterShell.NodeSet import NodeSet - +from ClusterShell.Task import task_self +from run_utils import RunException, get_clush_command, run_local, run_remote from user_utils import get_chown_command, get_primary_group -from run_utils import get_clush_command, run_remote, run_local, RunException class DaosTestError(Exception): diff --git a/src/tests/ftest/util/io_utilities.py b/src/tests/ftest/util/io_utilities.py index 307ac94cb23..125248c8e20 100644 --- a/src/tests/ftest/util/io_utilities.py +++ b/src/tests/ftest/util/io_utilities.py @@ -3,17 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import os import random import shutil import tempfile import time +from logging import getLogger +from general_utils import DaosTestError, get_random_bytes from pydaos.raw import DaosApiError -from general_utils import get_random_bytes, DaosTestError - class DirTree(): """ diff --git a/src/tests/ftest/util/ior_intercept_test_base.py b/src/tests/ftest/util/ior_intercept_test_base.py index 7cec0fd32db..fb28c7356ca 100644 --- a/src/tests/ftest/util/ior_intercept_test_base.py +++ b/src/tests/ftest/util/ior_intercept_test_base.py @@ -5,9 +5,10 @@ """ import os + +from general_utils import percent_change from ior_test_base import IorTestBase from ior_utils import IorCommand, IorMetrics -from general_utils import percent_change class IorInterceptTestBase(IorTestBase): diff --git a/src/tests/ftest/util/ior_test_base.py b/src/tests/ftest/util/ior_test_base.py index bd24286a243..bc593dd7a17 100644 --- a/src/tests/ftest/util/ior_test_base.py +++ b/src/tests/ftest/util/ior_test_base.py @@ -6,12 +6,11 @@ import os from ClusterShell.NodeSet import NodeSet - from dfuse_test_base import DfuseTestBase -from ior_utils import IorCommand from exception_utils import CommandFailure +from general_utils import get_random_string, pcmd +from ior_utils import IorCommand from job_manager_utils import get_job_manager -from general_utils import pcmd, get_random_string class IorTestBase(DfuseTestBase): diff --git a/src/tests/ftest/util/ior_utils.py b/src/tests/ftest/util/ior_utils.py index 4040ce9a9a9..fb56993a260 100644 --- a/src/tests/ftest/util/ior_utils.py +++ b/src/tests/ftest/util/ior_utils.py @@ -8,11 +8,11 @@ from enum import IntEnum from avocado.utils.process import CmdResult -from command_utils_base import FormattedParameter, BasicParameter -from exception_utils import CommandFailure from command_utils import SubProcessCommand -from general_utils import get_log_file +from command_utils_base import BasicParameter, FormattedParameter from duns_utils import format_path +from exception_utils import CommandFailure +from general_utils import get_log_file def get_ior(test, manager, hosts, path, slots, namespace="/run/ior/*", ior_params=None): diff --git a/src/tests/ftest/util/job_manager_utils.py b/src/tests/ftest/util/job_manager_utils.py index a55f4c9f184..afb9b3e4bdd 100644 --- a/src/tests/ftest/util/job_manager_utils.py +++ b/src/tests/ftest/util/job_manager_utils.py @@ -3,20 +3,19 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -# pylint: disable=too-many-lines -from distutils.spawn import find_executable # pylint: disable=deprecated-module import os import re import time +# pylint: disable=too-many-lines +from distutils.spawn import find_executable # pylint: disable=deprecated-module from ClusterShell.NodeSet import NodeSet - from command_utils import ExecutableCommand, SystemctlCommand -from command_utils_base import FormattedParameter, EnvironmentVariables -from exception_utils import CommandFailure, MPILoadError +from command_utils_base import EnvironmentVariables, FormattedParameter from env_modules import load_mpi -from general_utils import pcmd, run_pcmd, get_job_manager_class, get_journalctl_command, \ - journalctl_time +from exception_utils import CommandFailure, MPILoadError +from general_utils import (get_job_manager_class, get_journalctl_command, journalctl_time, pcmd, + run_pcmd) from run_utils import run_remote, stop_processes from write_host_file import write_host_file diff --git a/src/tests/ftest/util/macsio_test_base.py b/src/tests/ftest/util/macsio_test_base.py index f5565b8aaab..e7519a51eb3 100644 --- a/src/tests/ftest/util/macsio_test_base.py +++ b/src/tests/ftest/util/macsio_test_base.py @@ -5,8 +5,8 @@ """ from apricot import TestWithServers from exception_utils import CommandFailure -from macsio_util import MacsioCommand from general_utils import get_log_file +from macsio_util import MacsioCommand class MacsioTestBase(TestWithServers): diff --git a/src/tests/ftest/util/macsio_util.py b/src/tests/ftest/util/macsio_util.py index de95ac7081d..cf64dbcb030 100644 --- a/src/tests/ftest/util/macsio_util.py +++ b/src/tests/ftest/util/macsio_util.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils_base import FormattedParameter from command_utils import ExecutableCommand +from command_utils_base import FormattedParameter from general_utils import get_log_file, pcmd diff --git a/src/tests/ftest/util/mdtest_test_base.py b/src/tests/ftest/util/mdtest_test_base.py index df5703dc910..04aa44e5814 100644 --- a/src/tests/ftest/util/mdtest_test_base.py +++ b/src/tests/ftest/util/mdtest_test_base.py @@ -5,10 +5,11 @@ """ import os + from dfuse_test_base import DfuseTestBase -from mdtest_utils import MdtestCommand from exception_utils import CommandFailure from job_manager_utils import get_job_manager +from mdtest_utils import MdtestCommand class MdtestBase(DfuseTestBase): diff --git a/src/tests/ftest/util/mdtest_utils.py b/src/tests/ftest/util/mdtest_utils.py index d54f4a605cc..4c6d16b29a1 100644 --- a/src/tests/ftest/util/mdtest_utils.py +++ b/src/tests/ftest/util/mdtest_utils.py @@ -7,8 +7,8 @@ import os import re -from command_utils_base import FormattedParameter from command_utils import ExecutableCommand +from command_utils_base import FormattedParameter from general_utils import get_log_file diff --git a/src/tests/ftest/util/mpiio_test_base.py b/src/tests/ftest/util/mpiio_test_base.py index e83887c0382..b3fd058de13 100644 --- a/src/tests/ftest/util/mpiio_test_base.py +++ b/src/tests/ftest/util/mpiio_test_base.py @@ -7,11 +7,10 @@ import os from apricot import TestWithServers - from command_utils_base import CommandFailure, EnvironmentVariables -from job_manager_utils import get_job_manager -from mpiio_utils import LLNLCommand, Mpi4pyCommand, RomioCommand, Hdf5Command from duns_utils import format_path +from job_manager_utils import get_job_manager +from mpiio_utils import Hdf5Command, LLNLCommand, Mpi4pyCommand, RomioCommand class MpiioTests(TestWithServers): diff --git a/src/tests/ftest/util/mpiio_utils.py b/src/tests/ftest/util/mpiio_utils.py index 2c596d38643..44b40a98551 100644 --- a/src/tests/ftest/util/mpiio_utils.py +++ b/src/tests/ftest/util/mpiio_utils.py @@ -4,8 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from sys import version_info -from command_utils_base import FormattedParameter + from command_utils import ExecutableCommand +from command_utils_base import FormattedParameter class LLNLCommand(ExecutableCommand): diff --git a/src/tests/ftest/util/nvme_utils.py b/src/tests/ftest/util/nvme_utils.py index 3ba6fe2ec53..e92128b3d10 100644 --- a/src/tests/ftest/util/nvme_utils.py +++ b/src/tests/ftest/util/nvme_utils.py @@ -3,14 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -import threading import re +import threading import time from avocado import fail_on from avocado.core.exceptions import TestFail - -from dmg_utils import get_storage_query_device_uuids, get_dmg_response +from dmg_utils import get_dmg_response, get_storage_query_device_uuids from exception_utils import CommandFailure from ior_test_base import IorTestBase from ior_utils import IorCommand diff --git a/src/tests/ftest/util/osa_utils.py b/src/tests/ftest/util/osa_utils.py index 493fcf189a7..cc5d119bf09 100644 --- a/src/tests/ftest/util/osa_utils.py +++ b/src/tests/ftest/util/osa_utils.py @@ -5,17 +5,16 @@ """ import ctypes import queue -import time -import threading import re - -from pydaos.raw import DaosContainer, IORequest, DaosObj, DaosApiError +import threading +import time from avocado import fail_on -from ior_test_base import IorTestBase -from mdtest_test_base import MdtestBase from exception_utils import CommandFailure from general_utils import create_string_buffer, run_command +from ior_test_base import IorTestBase +from mdtest_test_base import MdtestBase +from pydaos.raw import DaosApiError, DaosContainer, DaosObj, IORequest class OSAUtils(MdtestBase, IorTestBase): diff --git a/src/tests/ftest/util/performance_test_base.py b/src/tests/ftest/util/performance_test_base.py index 4e9ef61004d..bf7a24907ef 100644 --- a/src/tests/ftest/util/performance_test_base.py +++ b/src/tests/ftest/util/performance_test_base.py @@ -6,15 +6,14 @@ import os import time +import oclass_utils from avocado.core.exceptions import TestFail - +from exception_utils import CommandFailure +from general_utils import get_subprocess_stdout from ior_test_base import IorTestBase +from ior_utils import IorMetrics from mdtest_test_base import MdtestBase from mdtest_utils import MdtestMetrics -from general_utils import get_subprocess_stdout -from ior_utils import IorMetrics -import oclass_utils -from exception_utils import CommandFailure class PerformanceTestBase(IorTestBase, MdtestBase): diff --git a/src/tests/ftest/util/pool_create_all_base.py b/src/tests/ftest/util/pool_create_all_base.py index 2ab16d42485..81b7b4b079b 100644 --- a/src/tests/ftest/util/pool_create_all_base.py +++ b/src/tests/ftest/util/pool_create_all_base.py @@ -5,9 +5,8 @@ """ import sys -from avocado.core.exceptions import TestFail - from apricot import TestWithServers +from avocado.core.exceptions import TestFail from general_utils import bytes_to_human diff --git a/src/tests/ftest/util/pool_security_test_base.py b/src/tests/ftest/util/pool_security_test_base.py index 4dc4250af77..c861248f17a 100644 --- a/src/tests/ftest/util/pool_security_test_base.py +++ b/src/tests/ftest/util/pool_security_test_base.py @@ -3,14 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import grp import os import random -import grp import re -from apricot import TestWithServers import agent_utils as agu import security_test_base as secTestBase +from apricot import TestWithServers PERMISSIONS = ["", "r", "w", "rw"] DENY_ACCESS = "-1001" diff --git a/src/tests/ftest/util/rebuild_test_base.py b/src/tests/ftest/util/rebuild_test_base.py index 1435aa1815e..bdb31ea9807 100644 --- a/src/tests/ftest/util/rebuild_test_base.py +++ b/src/tests/ftest/util/rebuild_test_base.py @@ -4,7 +4,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithServers -from command_utils_base import ObjectWithParameters, BasicParameter +from command_utils_base import BasicParameter, ObjectWithParameters from daos_utils import DaosCommand diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index 730dbb3bbd1..b382a893522 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -3,10 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from socket import gethostname -import subprocess # nosec import shlex +import subprocess # nosec import time +from socket import gethostname + from ClusterShell.NodeSet import NodeSet from ClusterShell.Task import task_self diff --git a/src/tests/ftest/util/scrubber_test_base.py b/src/tests/ftest/util/scrubber_test_base.py index 95a2eb770a8..7e938662e9f 100644 --- a/src/tests/ftest/util/scrubber_test_base.py +++ b/src/tests/ftest/util/scrubber_test_base.py @@ -5,8 +5,8 @@ """ import threading -from scrubber_utils import ScrubberUtils from ior_test_base import IorTestBase +from scrubber_utils import ScrubberUtils class TestWithScrubber(IorTestBase): diff --git a/src/tests/ftest/util/security_test_base.py b/src/tests/ftest/util/security_test_base.py index 85c8ad2e050..a0b83426777 100644 --- a/src/tests/ftest/util/security_test_base.py +++ b/src/tests/ftest/util/security_test_base.py @@ -6,6 +6,7 @@ import os import random + from general_utils import pcmd diff --git a/src/tests/ftest/util/server_utils.py b/src/tests/ftest/util/server_utils.py index c8c36862c0f..8f20bf6c783 100644 --- a/src/tests/ftest/util/server_utils.py +++ b/src/tests/ftest/util/server_utils.py @@ -5,25 +5,24 @@ """ # pylint: disable=too-many-lines -from collections import defaultdict -from getpass import getuser import os +import random import re import time -import random +from collections import defaultdict +from getpass import getuser from avocado import fail_on - -from command_utils_base import CommonConfig, BasicParameter from command_utils import SubprocessManager +from command_utils_base import BasicParameter, CommonConfig from dmg_utils import get_dmg_command from exception_utils import CommandFailure -from general_utils import pcmd, get_log_file, list_to_str, get_display_size, run_pcmd +from general_utils import get_display_size, get_log_file, list_to_str, pcmd, run_pcmd from host_utils import get_local_host -from server_utils_base import ServerFailed, DaosServerCommand, DaosServerInformation +from run_utils import run_remote, stop_processes +from server_utils_base import DaosServerCommand, DaosServerInformation, ServerFailed from server_utils_params import DaosServerTransportCredentials, DaosServerYamlParameters from user_utils import get_chown_command -from run_utils import run_remote, stop_processes def get_server_command(group, cert_dir, bin_dir, config_file, config_temp=None): diff --git a/src/tests/ftest/util/server_utils_base.py b/src/tests/ftest/util/server_utils_base.py index ab68914b24c..068c62919f0 100644 --- a/src/tests/ftest/util/server_utils_base.py +++ b/src/tests/ftest/util/server_utils_base.py @@ -3,14 +3,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import os import re +from logging import getLogger from ClusterShell.NodeSet import NodeSet - -from command_utils_base import FormattedParameter, CommandWithParameters -from command_utils import YamlCommand, CommandWithSubCommand +from command_utils import CommandWithSubCommand, YamlCommand +from command_utils_base import CommandWithParameters, FormattedParameter from data_utils import dict_extract_values, list_flatten from dmg_utils import get_dmg_response from exception_utils import CommandFailure diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index 8f456706c04..d22b9cc5cb3 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -5,8 +5,7 @@ """ import os -from command_utils_base import \ - BasicParameter, LogParameter, YamlParameters, TransportCredentials +from command_utils_base import BasicParameter, LogParameter, TransportCredentials, YamlParameters MAX_STORAGE_TIERS = 5 diff --git a/src/tests/ftest/util/slurm_utils.py b/src/tests/ftest/util/slurm_utils.py index d43208e3d61..9ba92b0f8c9 100644 --- a/src/tests/ftest/util/slurm_utils.py +++ b/src/tests/ftest/util/slurm_utils.py @@ -5,9 +5,9 @@ """ import os import random -import time -import threading import re +import threading +import time from ClusterShell.NodeSet import NodeSet, NodeSetParseError # pylint: disable=import-error,no-name-in-module diff --git a/src/tests/ftest/util/soak_test_base.py b/src/tests/ftest/util/soak_test_base.py index dc0e1912bf3..03c01106894 100644 --- a/src/tests/ftest/util/soak_test_base.py +++ b/src/tests/ftest/util/soak_test_base.py @@ -4,33 +4,32 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import multiprocessing import os +import random +import socket +import threading import time from datetime import datetime, timedelta -import multiprocessing -import threading -import random from filecmp import cmp from getpass import getuser -import socket +import slurm_utils +from agent_utils import include_local_host from apricot import TestWithServers from ClusterShell.NodeSet import NodeSet -from agent_utils import include_local_host +from dmg_utils import DmgCommand from exception_utils import CommandFailure from general_utils import journalctl_time from host_utils import get_local_host -import slurm_utils -from dmg_utils import DmgCommand -from run_utils import run_local, run_remote, RunException -from soak_utils import ddhhmmss_format, add_pools, \ - launch_snapshot, launch_exclude_reintegrate, launch_extend, \ - create_ior_cmdline, cleanup_dfuse, create_fio_cmdline, \ - build_job_script, SoakTestError, launch_server_stop_start, get_harassers, \ - create_racer_cmdline, run_event_check, run_monitor_check, \ - create_mdtest_cmdline, reserved_file_copy, run_metrics_check, \ - get_journalctl, get_daos_server_logs, create_macsio_cmdline, \ - create_app_cmdline, create_dm_cmdline, launch_vmd_identify_check +from run_utils import RunException, run_local, run_remote +from soak_utils import (SoakTestError, add_pools, build_job_script, cleanup_dfuse, + create_app_cmdline, create_dm_cmdline, create_fio_cmdline, + create_ior_cmdline, create_macsio_cmdline, create_mdtest_cmdline, + create_racer_cmdline, ddhhmmss_format, get_daos_server_logs, get_harassers, + get_journalctl, launch_exclude_reintegrate, launch_extend, + launch_server_stop_start, launch_snapshot, launch_vmd_identify_check, + reserved_file_copy, run_event_check, run_metrics_check, run_monitor_check) class SoakTestBase(TestWithServers): diff --git a/src/tests/ftest/util/soak_utils.py b/src/tests/ftest/util/soak_utils.py index 577fde705e3..9aaec2fdbf1 100644 --- a/src/tests/ftest/util/soak_utils.py +++ b/src/tests/ftest/util/soak_utils.py @@ -6,33 +6,31 @@ # pylint: disable=too-many-lines import os -import time import random -import threading import re +import threading +import time from itertools import product +import slurm_utils from avocado.core.exceptions import TestFail -from pydaos.raw import DaosSnapshot, DaosApiError - -from ior_utils import IorCommand -from fio_utils import FioCommand -from mdtest_utils import MdtestCommand +from command_utils_base import EnvironmentVariables from daos_racer_utils import DaosRacerCommand from data_mover_utils import DcpCommand, FsCopy from dfuse_utils import Dfuse from dmg_utils import get_storage_query_device_info +from duns_utils import format_path +from fio_utils import FioCommand +from general_utils import (DaosTestError, get_host_data, get_log_file, get_random_bytes, + get_random_string, list_to_str, pcmd, run_command, run_pcmd) +from ior_utils import IorCommand from job_manager_utils import Mpirun -from general_utils import get_host_data, get_random_string, \ - run_command, DaosTestError, pcmd, get_random_bytes, \ - run_pcmd, list_to_str, get_log_file -from command_utils_base import EnvironmentVariables -import slurm_utils -from run_utils import run_remote -from test_utils_container import TestContainer from macsio_util import MacsioCommand +from mdtest_utils import MdtestCommand from oclass_utils import extract_redundancy_factor -from duns_utils import format_path +from pydaos.raw import DaosApiError, DaosSnapshot +from run_utils import run_remote +from test_utils_container import TestContainer H_LOCK = threading.Lock() diff --git a/src/tests/ftest/util/support_test_base.py b/src/tests/ftest/util/support_test_base.py new file mode 100644 index 00000000000..27b4a20e893 --- /dev/null +++ b/src/tests/ftest/util/support_test_base.py @@ -0,0 +1,139 @@ +""" +(C) Copyright 2023 Intel Corporation. + +SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import os +from datetime import datetime + +from control_test_base import ControlTestBase +from run_utils import command_as_user, run_remote + + +class SupportTestBase(ControlTestBase): + """Class for Support log collection """ + + def __init__(self, *args, **kwargs): + """Initialize a SupportTestBase object.""" + super().__init__(*args, **kwargs) + self.target_folder = None + self.custom_log_dir = None + self.custom_log_file = None + self.custom_log_data = None + self.log_hosts = None + self.run_user = 'root' + self.extract_dir = os.path.join(self.base_test_dir, "extracted_support_logs") + + def create_custom_log(self, folder_name): + """Create custom log directory with custom data file on each servers. + + Args: + folder_name (str): Name of the custom folder + """ + server_custom_log = self.base_test_dir + self.custom_log_dir = os.path.join(server_custom_log, folder_name) + self.custom_log_file = os.path.join(self.custom_log_dir, "Custom_File") + self.target_folder = os.path.join(self.base_test_dir, "DAOS_Support") + + # make the custom log dir on node (clients or servers) + mkdir_cmd = "mkdir -p {}".format(self.custom_log_dir) + result = run_remote(self.log, self.log_hosts, mkdir_cmd) + if not result.passed: + self.fail("Failed to create the custom log dir {} ".format(result)) + + # Get date-time object containing current date and time + now = datetime.now() + self.custom_log_data = now.strftime("%d/%m/%Y %H:%M:%S") + + # Create the custom log file on node (clients or servers) + create_file = " echo \'{}\' > {}".format(self.custom_log_data, self.custom_log_file) + result = run_remote(self.log, self.log_hosts, create_file) + if not result.passed: + self.fail("Failed to create the custom log file {} ".format(result)) + + def verify_custom_log_data(self): + """Verify custom log files is collected and part of archive. + + """ + getfilename = "find {} -name {}".format( + self.extract_dir, os.path.basename(self.custom_log_file)) + findcmd = command_as_user(getfilename, self.run_user) + readfiledata = command_as_user("xargs cat", self.run_user) + + result = run_remote(self.log, self.log_hosts, findcmd + "|" + readfiledata) + if not result.passed: + self.fail("Failed to read the custom log file {} ".format(result)) + + for output in result.all_stdout.values(): + if self.custom_log_data not in output: + self.fail("Expected custom_log_data {} not found in log file {}" + .format(self.custom_log_data, self.custom_log_file)) + + def extract_logs(self, tar_gz_filename): + """Extract the logs files which are in collected archive. + + Args: + tar_gz_filename (str): Log archive File name + + Raises: + Test Failure: If unable to run any command. + + """ + # Create the new extract directory + command = command_as_user("mkdir -p {}".format(self.extract_dir), self.run_user) + result = run_remote(self.log, self.log_hosts, command) + if not result.passed: + self.fail("cmd {} failed, result:{}".format(command, result)) + + # Extract The tar.gz file to newly created directory + command = command_as_user("tar -xf {} -C {}".format(tar_gz_filename, self.extract_dir), + self.run_user) + result = run_remote(self.log, self.log_hosts, command) + if not result.passed: + self.fail("Failed to extract the {} file, result:{}".format(tar_gz_filename, result)) + + def validate_server_log_files(self): + """Verify all the server logs files are collected and part of archive. + + Raises: + Test Failure: If unable to run any command. + + """ + log_files = [] + helper_log_file = self.server_managers[0].get_config_value("helper_log_file") + log_files.append(os.path.basename(helper_log_file)) + control_log_file = self.server_managers[0].get_config_value("control_log_file") + log_files.append(os.path.basename(control_log_file)) + log_files.append(self.params.get("log_file", "/run/server_config/engines/0/*")) + log_files.append(self.params.get("log_file", "/run/server_config/engines/1/*")) + + # Verify server log files are collected. + for log_file in log_files: + list_file = "ls -lsaRt {} | grep {}".format(self.extract_dir, log_file) + command = command_as_user(list_file, self.run_user) + result = run_remote(self.log, self.log_hosts, command) + if not result.passed: + self.fail("Failed to list the {} file from extracted folder{}".format( + result, self.extract_dir)) + + def cleanup_support_log(self, log_dir): + """ Test cleanup to remove the temporary directory + + Args: + log_dir (str): Name of the log directory to be removed + + Returns: + list: a list of any errors detected when deleting the logs + + """ + error_list = [] + # Remove the log and extract directory + folders = [log_dir, self.extract_dir] + for folder in folders: + delete_cmd = "sudo rm -rf {}*".format(folder) + result = run_remote(self.log, self.log_hosts, delete_cmd) + if not result.passed: + error_list.append( + "Failed to delete folder {} with result:{}".format(folder, result)) + + return error_list diff --git a/src/tests/ftest/util/telemetry_utils.py b/src/tests/ftest/util/telemetry_utils.py index cc1cee34127..021ba517b1b 100644 --- a/src/tests/ftest/util/telemetry_utils.py +++ b/src/tests/ftest/util/telemetry_utils.py @@ -3,8 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import re +from logging import getLogger + from ClusterShell.NodeSet import NodeSet diff --git a/src/tests/ftest/util/test_utils_base.py b/src/tests/ftest/util/test_utils_base.py index 49562802c53..af032a5160c 100644 --- a/src/tests/ftest/util/test_utils_base.py +++ b/src/tests/ftest/util/test_utils_base.py @@ -3,15 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from collections import defaultdict from logging import getLogger -from time import sleep from threading import Lock -from collections import defaultdict +from time import sleep +from command_utils_base import BasicParameter, ObjectWithParameters from pydaos.raw import DaosApiError -from command_utils_base import ObjectWithParameters, BasicParameter - class CallbackHandler(): """Defines a callback method to use with DaosApi class methods.""" diff --git a/src/tests/ftest/util/test_utils_container.py b/src/tests/ftest/util/test_utils_container.py index 6152870ab8f..bc7902dec3a 100644 --- a/src/tests/ftest/util/test_utils_container.py +++ b/src/tests/ftest/util/test_utils_container.py @@ -10,13 +10,11 @@ from time import time from avocado import fail_on -from pydaos.raw import (DaosApiError, DaosContainer, DaosInputParams, - c_uuid_to_str, str_to_c_uuid) - -from test_utils_base import TestDaosApiBase from command_utils_base import BasicParameter from exception_utils import CommandFailure -from general_utils import get_random_bytes, DaosTestError +from general_utils import DaosTestError, get_random_bytes +from pydaos.raw import DaosApiError, DaosContainer, DaosInputParams, c_uuid_to_str, str_to_c_uuid +from test_utils_base import TestDaosApiBase class TestContainerData(): diff --git a/src/tests/ftest/util/test_utils_pool.py b/src/tests/ftest/util/test_utils_pool.py index 37e9ddcfcde..2b4b14f427f 100644 --- a/src/tests/ftest/util/test_utils_pool.py +++ b/src/tests/ftest/util/test_utils_pool.py @@ -3,20 +3,19 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import ctypes +import json # pylint: disable=too-many-lines import os from time import sleep, time -import ctypes -import json -from avocado import fail_on, TestFail -from pydaos.raw import (DaosApiError, DaosPool, c_uuid_to_str, daos_cref) - -from test_utils_base import TestDaosApiBase, LabelGenerator +from avocado import TestFail, fail_on from command_utils import BasicParameter -from exception_utils import CommandFailure -from general_utils import check_pool_files, DaosTestError from dmg_utils import DmgCommand, DmgJsonCommandFailure +from exception_utils import CommandFailure +from general_utils import DaosTestError, check_pool_files +from pydaos.raw import DaosApiError, DaosPool, c_uuid_to_str, daos_cref +from test_utils_base import LabelGenerator, TestDaosApiBase POOL_NAMESPACE = "/run/pool/*" POOL_TIMEOUT_INCREMENT = 200 diff --git a/src/tests/ftest/util/thread_manager.py b/src/tests/ftest/util/thread_manager.py index f67719e8843..6e30c6bdee4 100644 --- a/src/tests/ftest/util/thread_manager.py +++ b/src/tests/ftest/util/thread_manager.py @@ -3,8 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import ThreadPoolExecutor from concurrent.futures import TimeoutError as FuturesTimeoutError +from concurrent.futures import as_completed from logging import getLogger from avocado.utils.process import CmdResult diff --git a/src/tests/ftest/util/vol_test_base.py b/src/tests/ftest/util/vol_test_base.py index 3b532f7219a..b45bcf62082 100644 --- a/src/tests/ftest/util/vol_test_base.py +++ b/src/tests/ftest/util/vol_test_base.py @@ -4,10 +4,10 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from dfuse_test_base import DfuseTestBase +from command_utils import ExecutableCommand from command_utils_base import EnvironmentVariables +from dfuse_test_base import DfuseTestBase from exception_utils import CommandFailure -from command_utils import ExecutableCommand class VolTestBase(DfuseTestBase): diff --git a/src/tests/ftest/util/write_host_file.py b/src/tests/ftest/util/write_host_file.py index 9f418137a5c..8a6e810c6ba 100644 --- a/src/tests/ftest/util/write_host_file.py +++ b/src/tests/ftest/util/write_host_file.py @@ -3,9 +3,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from logging import getLogger import os import random +from logging import getLogger def write_host_file(hosts, path='/tmp', slots=1): diff --git a/src/tests/ftest/vmd/fault_reintegration.py b/src/tests/ftest/vmd/fault_reintegration.py index 3d8a32e6c79..0380fa9effc 100644 --- a/src/tests/ftest/vmd/fault_reintegration.py +++ b/src/tests/ftest/vmd/fault_reintegration.py @@ -3,17 +3,16 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from multiprocessing import Queue import threading +from multiprocessing import Queue +from apricot import TestWithServers from avocado import fail_on - -from dmg_utils import get_storage_query_device_info, get_dmg_response +from dmg_utils import get_dmg_response, get_storage_query_device_info from exception_utils import CommandFailure from ior_utils import run_ior, thread_run_ior from job_manager_utils import get_job_manager from nvme_utils import set_device_faulty -from apricot import TestWithServers class NvmeFaultReintegrate(TestWithServers): diff --git a/src/tests/ftest/vmd/led.py b/src/tests/ftest/vmd/led.py index f69c808d701..168d1dc29ae 100644 --- a/src/tests/ftest/vmd/led.py +++ b/src/tests/ftest/vmd/led.py @@ -6,7 +6,6 @@ import time from avocado import fail_on - from dmg_utils import get_storage_query_device_uuids from exception_utils import CommandFailure from nvme_utils import set_device_faulty diff --git a/src/tests/suite/SConscript b/src/tests/suite/SConscript index a30f8bc432a..7ca9f0b5148 100644 --- a/src/tests/suite/SConscript +++ b/src/tests/suite/SConscript @@ -1,6 +1,6 @@ """Build test suite""" -import sys import subprocess # nosec +import sys TEST_CMOCKA_SKIP = """ #include diff --git a/src/utils/daos_hdlr.c b/src/utils/daos_hdlr.c index 2da8c3df4d4..1c145bc6b1c 100644 --- a/src/utils/daos_hdlr.c +++ b/src/utils/daos_hdlr.c @@ -1492,10 +1492,8 @@ dm_connect(struct cmd_args_s *ap, dattr.da_props = props; rc = duns_create_path(ca->dst_poh, path, &dattr); if (rc != 0) { - rc = daos_errno2der(rc); - DH_PERROR_DER(ap, rc, "provide a destination pool or UNS path " - "of the form:\n\t --dst | "); - D_GOTO(err, rc); + DH_PERROR_SYS(ap, rc, "failed to create destination UNS path"); + D_GOTO(err, rc = daos_errno2der(rc)); } snprintf(ca->dst_cont, DAOS_PROP_LABEL_MAX_LEN + 1, "%s", dattr.da_cont); } diff --git a/src/vos/storage_estimator/common/dfs_sb.py b/src/vos/storage_estimator/common/dfs_sb.py index 6aaa07e5c51..6b9f7fa4833 100644 --- a/src/vos/storage_estimator/common/dfs_sb.py +++ b/src/vos/storage_estimator/common/dfs_sb.py @@ -4,11 +4,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import os import ctypes +import os from pydaos.raw import daos_cref -from storage_estimator.vos_structures import ValType, Overhead, AKey, VosValue, DKey, VosObject +from storage_estimator.vos_structures import AKey, DKey, Overhead, ValType, VosObject, VosValue header = '''--- # Sample conflig file DFS files and directories diff --git a/src/vos/storage_estimator/common/explorer.py b/src/vos/storage_estimator/common/explorer.py index 97641419e34..aafc7371989 100644 --- a/src/vos/storage_estimator/common/explorer.py +++ b/src/vos/storage_estimator/common/explorer.py @@ -4,13 +4,13 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import os import copy +import os import sys -from storage_estimator.vos_structures import VosObject, AKey, DKey, Container, \ - VosValue, Overhead, ValType, KeyType from storage_estimator.util import CommonBase, ObjectClass +from storage_estimator.vos_structures import (AKey, Container, DKey, KeyType, Overhead, ValType, + VosObject, VosValue) class FileInfo(): diff --git a/src/vos/storage_estimator/common/parse_csv.py b/src/vos/storage_estimator/common/parse_csv.py index 3b296a9e990..8820a3e3aaa 100644 --- a/src/vos/storage_estimator/common/parse_csv.py +++ b/src/vos/storage_estimator/common/parse_csv.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -from storage_estimator.explorer import AverageFS from storage_estimator.dfs_sb import get_dfs_inode_akey +from storage_estimator.explorer import AverageFS from storage_estimator.util import ProcessBase FILE_SIZES = ['4k', '64k', '128k', '256k', '512k', '768k', '1m', '8m', '64m', diff --git a/src/vos/storage_estimator/common/tests/storage_estimator_test.py b/src/vos/storage_estimator/common/tests/storage_estimator_test.py index 5855995d783..66944e75968 100644 --- a/src/vos/storage_estimator/common/tests/storage_estimator_test.py +++ b/src/vos/storage_estimator/common/tests/storage_estimator_test.py @@ -3,16 +3,17 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import pytest -import unittest -import yaml import os +import unittest -from storage_estimator.vos_structures import VosObject, AKey, DKey, Container, Containers, \ - VosValue, Overhead, ValType, VosValueError +import pytest +import yaml from storage_estimator.explorer import FileSystemExplorer -from storage_estimator.util import ObjectClass from storage_estimator.parse_csv import ProcessCSV +from storage_estimator.util import ObjectClass +from storage_estimator.vos_structures import (AKey, Container, Containers, DKey, Overhead, ValType, + VosObject, VosValue, VosValueError) + from .util import FileGenerator diff --git a/src/vos/storage_estimator/common/tests/util.py b/src/vos/storage_estimator/common/tests/util.py index 92006f268bb..79eaea7a9bb 100644 --- a/src/vos/storage_estimator/common/tests/util.py +++ b/src/vos/storage_estimator/common/tests/util.py @@ -4,8 +4,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import os -import tempfile import shutil +import tempfile class FileGenerator(): diff --git a/src/vos/storage_estimator/common/util.py b/src/vos/storage_estimator/common/util.py index 6462c019181..b571d984a57 100644 --- a/src/vos/storage_estimator/common/util.py +++ b/src/vos/storage_estimator/common/util.py @@ -6,8 +6,8 @@ import os -import yaml +import yaml from storage_estimator.dfs_sb import VOS_SIZE, get_dfs_sb_obj from storage_estimator.vos_size import MetaOverhead from storage_estimator.vos_structures import Containers diff --git a/src/vos/storage_estimator/common/vos_size.py b/src/vos/storage_estimator/common/vos_size.py index 0b8cf9a9650..872fc76207a 100644 --- a/src/vos/storage_estimator/common/vos_size.py +++ b/src/vos/storage_estimator/common/vos_size.py @@ -3,8 +3,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' -import random import math +import random def convert(stat): diff --git a/src/vos/storage_estimator/daos_storage_estimator.py b/src/vos/storage_estimator/daos_storage_estimator.py index 2b9da5c4878..6e68df36f63 100755 --- a/src/vos/storage_estimator/daos_storage_estimator.py +++ b/src/vos/storage_estimator/daos_storage_estimator.py @@ -7,9 +7,9 @@ import argparse import sys -from storage_estimator.dfs_sb import get_dfs_example, print_daos_version, get_dfs_inode_akey -from storage_estimator.parse_csv import ProcessCSV +from storage_estimator.dfs_sb import get_dfs_example, get_dfs_inode_akey, print_daos_version from storage_estimator.explorer import FileSystemExplorer +from storage_estimator.parse_csv import ProcessCSV from storage_estimator.util import Common, ProcessBase tool_description = '''DAOS estimation tool diff --git a/src/vos/tests/evt_stress.py b/src/vos/tests/evt_stress.py index d172d0ada86..ed05f61cc3f 100755 --- a/src/vos/tests/evt_stress.py +++ b/src/vos/tests/evt_stress.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Run evt_ctl with a specific pattern that causes a segfault with the default sort algorithm""" +import argparse +import json import os from os.path import join -import json -import argparse class EVTStress(): diff --git a/utils/ansible/ftest/library/daos_hugepages.py b/utils/ansible/ftest/library/daos_hugepages.py index 49110232453..35d192b3035 100644 --- a/utils/ansible/ftest/library/daos_hugepages.py +++ b/utils/ansible/ftest/library/daos_hugepages.py @@ -9,9 +9,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import datetime import os import re -import datetime import subprocess # nosec B404 from ansible.module_utils.basic import AnsibleModule diff --git a/utils/cq/d_logging_check.py b/utils/cq/d_logging_check.py new file mode 100755 index 00000000000..fca019815b8 --- /dev/null +++ b/utils/cq/d_logging_check.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python + +"""Check and re-write DAOS logging macros to preferred coding style + +Apply some checks for the macros, partly for correctness but also for consistency. Some errors +can be fixed however not all can. Code formatting will be incorrect after fixing so clang-format +(or git-clang-format -f) should be used after this. + +Use --fix to apply any correctness fixes that can be applied. +Use --correct to apply any style changes that can be applied. Style changes will be applied +if fixes are being applied. +""" + +import argparse +import inspect +import io +import os +import re +import sys + +ARGS = None + + +class FileLine(): + """One line from a file""" + + def __init__(self, file_object, line, lineno): + self._fo = file_object + self._lineno = lineno + + # Text as it appears in the source + self._line = line + # Striped text + self._code = line.strip() + self.modified = False + self.corrected = False + + def startswith(self, string): + """Starts-with method""" + return self._code.startswith(string) + + def endswith(self, string): + """Ends-with method""" + return self._code.endswith(string) + + def count(self, string): + """Returns count of substring""" + return self._code.count(string) + + def __contains__(self, string): + return string in self._code + + def __str__(self): + if self.modified or (self.corrected and ARGS.correct): + return f'{self._code}\n' + return self._line + + def expand(self): + """Expand line to end""" + while not self._code.endswith(';'): + to_add = str(next(self._fo)) + self._code += to_add.strip() + self._line += to_add + + def write(self, flo): + """Write line to file""" + flo.write(str(self)) + + def correct(self, new_code): + """Apply corrections to a line. + + These will only be applied if fixes are also being applied. + """ + self._code = new_code + self.corrected = True + self.note('Optional fixes') + + def fix(self, new_code): + """Mark a line as updated""" + self.correct(new_code) + self.modified = True + self.note('Required fixes') + + def raw(self): + """Returns the code""" + return self._code + + def warning(self, msg): + """Show a warning""" + print(f'{self._fo.fname}:{self._lineno} {msg}') + if ARGS.github: + fn_name = inspect.stack()[1].function + fn_name = fn_name.replace('_', '-') + print(f'::warning file={self._fo.fname},line={self._lineno},::{fn_name}, {msg}') + + def note(self, msg): + """Show a note""" + print(f'{self._fo.fname}:{self._lineno} {msg}') + + +class FileParser: + """One source file""" + + def __init__(self, fname): + self.fname = fname + self.lines = [] + self._index = None + with open(fname, 'r') as fd: + lineno = 1 + for line in fd: + self.lines.append(FileLine(self, line, lineno)) + lineno += 1 + + def __iter__(self): + self._index = -1 + return self + + def __next__(self): + self._index += 1 + try: + return self.lines[self._index] + except IndexError as exc: + raise StopIteration from exc + + +# Logging macros where the new-line is added if missing. +PREFIXES = ['D_ERROR', 'D_WARN', 'D_INFO', 'D_NOTE', 'D_ALERT', 'D_CRIT', 'D_FATAT', 'D_EMIT', + 'D_TRACE_INFO', 'D_TRACE_NOTE', 'D_TRACE_WARN', 'D_TRACE_ERROR', 'D_TRACE_ALERT', + 'D_TRACE_CRIT', 'D_TRACE_FATAL', 'D_TRACE_EMIT', 'RPC_TRACE', 'RPC_ERROR', + 'VOS_TX_LOG_FAIL', 'VOS_TX_TRACE_FAIL', 'D_DEBUG', 'D_CDEBUG'] + +# Logging macros where a new-line is always added. +PREFIXES_NNL = ['DFUSE_LOG_WARNING', 'DFUSE_LOG_ERROR', 'DFUSE_LOG_DEBUG', 'DFUSE_LOG_INFO', + 'DFUSE_TRA_WARNING', 'DFUSE_TRA_ERROR', 'DFUSE_TRA_DEBUG', 'DFUSE_TRA_INFO', + 'DH_PERROR_SYS', 'DH_PERROR_DER', 'DL_CDEBUG'] + +for prefix in ['DL', 'DHL', 'DS', 'DHS']: + for suffix in ['ERROR', 'WARN', 'INFO']: + PREFIXES_NNL.append(f'{prefix}_{suffix}') + +PREFIXES_ALL = PREFIXES.copy() +PREFIXES_ALL.extend(PREFIXES_NNL) + + +class AllChecks(): + """All the checks in one class""" + + def __init__(self, file_object): + self._fo = file_object + self.line = '' + self._output = io.StringIO() + self.modified = False + self.corrected = False + + def run_all_checks(self): + """Run everything + + Iterate over the input file line by line checking for logging use and run checks on lines + where the macros are used. Ignore lines within macro definitions. + """ + prev_macro = False + for line in self._fo: + if line.endswith('\\'): + prev_macro = True + # line.note('Part of macro, not checking') + line.write(self._output) + continue + + if prev_macro: + prev_macro = False + line.write(self._output) + continue + + if not any(map(line.startswith, PREFIXES_ALL)): + line.write(self._output) + continue + + line.expand() + + self.check_print_string(line) + self.check_quote(line) + self.check_return(line) + self.check_df_rc_dot(line) + self.check_for_newline(line) + self.check_df_rc(line) + self.remove_trailing_period(line) + self.check_quote(line) + self.check_failed(line) + + line.write(self._output) + if line.modified: + self.modified = True + if line.corrected: + self.corrected = True + + def save(self, fname): + """Save new file to file""" + if not self.modified and not self.corrected: + return + with open(fname, 'w') as fd: + fd.write(self._output.getvalue()) + print('Changes saved, run git-clang-format to format them.') + + def check_print_string(self, line): + """Check for %s in message""" + if line.startswith('DH_PERROR'): + return + count = line.count('%s') + if count == 0: + return + if count == 1 and line.count('strerror') == 1: + return + if '?' in line: + line.note('Message uses %s with trigraph') + else: + line.note('Message uses %s without trigraph') + + def check_quote(self, line): + """Check for double quotes in message""" + if '""' not in line: + return + + # Do not remove if used in tri-graphs. + if ': ""' in line or '? ""' in line or '\\""' in line: + return + + line.correct(line.raw().replace('""', '')) + + def check_return(self, line): + """Check for one return character""" + max_newlines = 1 + code = line.raw() + if any(map(code.startswith, PREFIXES_NNL)): + max_newlines = 0 + + if '"%s",' in code: + line.note('Use of %s at end of log-line, unable to check') + return + + count = code.count('\\n') + # Some logging calls contain multiple new-line characters and that's OK, as long as one of + # them isn't at the end of a line. + if max_newlines == 0 and count > max_newlines: + line.warning("Line contains too many newlines") + + def check_df_rc_dot(self, line): + """Check there is no . after DF_RC""" + code = line.raw() + count = code.count('DF_RC') + if count == 0: + return + code = code.replace('DF_RC ', 'DF_RC') + if 'DF_RC".\\n"' not in code: + return + code = code.replace('DF_RC".\\n"', 'DF_RC"\\n"') + line.warning('Extra . after DP_RC macro') + line.fix(code) + + def check_df_rc(self, line): + r"""Check for text before DF_RC macro + + Re-flow lines that use DF_RC so that they are of the form '...: " DF_RC,' + There should be a ": " before the DF_RC. + There should not be a space before the : + There should not be other special characters used. + The variable name should not be printed + + """ + code = line.raw() + count = code.count('DF_RC') + if count == 0: + return + if count != 1: + line.note('Cannot check lines with multiple DF_RC') + return + if not code.endswith('));'): + line.note('Unable to check DF_RC') + return + + # Remove any spaces around macros as these may or may not be present. This updated input + # is used for the update check at the end so white-space differences here will not cause + # code to be re-written. + code = re.sub(r' ?DF_RC ?', 'DF_RC', code) + code = re.sub(r' ?DP_RC ?', 'DP_RC', code) + + # Check that DF_RC is at the end of the line, it should be. + + if '\\n' in code: + if 'DF_RC"\\n"' not in code: + line.note('DF_RC is not at end of line') + return + else: + if 'DF_RC,' not in code: + line.note('DF_RC is not at end of line') + return + + # Extract the variable name + parts = code[:-3].split('(') + if not parts[-2].endswith('DP_RC'): + line.note('Function in DF_RC call') + return + var_name = parts.pop() + new_code = '('.join(parts) + assert new_code.endswith('DP_RC') + new_code = new_code[:-5] + + # Strip out the string formatting message + parts = code.split('DF_RC') + assert len(parts) == 2 + msg = parts[0] + + assert msg.endswith('"') + msg = msg[:-1] + + # Check what comes before DF_RC in the message, and strip any trailing punctuation. + imsg = None + while imsg != msg: + imsg = msg + if any(map(msg.endswith, [' ', '=', '.', ',', ':', ';'])): + msg = msg[:-1] + if msg.endswith(var_name): + msg = msg[:-len(var_name)] + if msg.endswith('rc'): + msg = msg[:-2] + + # Put it all back together with consistent style. + new_code = f'{msg}: "DF_RC{parts[1]}' + + if any(map(line.startswith, ['D_ERROR', 'D_WARN', 'D_INFO'])): + new_code = f'{msg}"DF_RC{parts[1]}' + if line.startswith('D_ERROR'): + new_code = f'DL_ERROR({var_name}, {new_code[8:]}' + elif line.startswith('D_WARN'): + new_code = f'DL_WARN({var_name}, {new_code[7:]}' + else: + new_code = f'DL_INFO({var_name}, {new_code[7:]}' + new_code = new_code.replace('DF_RC', '') + new_code = new_code.replace(f',DP_RC({var_name})', '') + + if new_code != code: + line.correct(new_code) + + def check_for_newline(self, line): + """Remove optional new-lines""" + code = line.raw() + + if any(map(code.startswith, PREFIXES_NNL)): + return + + if '\\"\\n' in code: + line.note('Unable to check for newlines') + return + + new_code = code.replace('"\\n",', ',') + new_code = new_code.replace('\\n",', '",') + new_code = new_code.replace('"\\n")', ')') + new_code = new_code.replace('\\n")', '")') + if new_code != code: + line.correct(new_code) + + def remove_trailing_period(self, line): + """Remove . from the end of a line""" + code = line.raw() + + new_code = code + before_code = None + while new_code != before_code: + before_code = new_code + new_code = re.sub(r'\.",', '",', new_code) + new_code = re.sub(r'\."\)', '")', new_code) + + if new_code != code: + line.correct(new_code) + + def check_failed(self, line): + """Check for 'Failed' with uppercase F + + Lots of message are of the form 'function() failed' but some use Failed. + """ + code = line.raw() + + if 'Failed' not in code: + return + if '"Failed' in code: + return + if 'Failed to' in code: + return + + print(code) + line.note('Failed') + + +def one_entry(fname): + """Process one path entry + + Returns true if there are un-fixed errors. + """ + if not any(map(fname.endswith, ['.c', '.h'])): + return False + + if any(map(fname.endswith, ['pb-c.c', 'pb-c..h'])): + return False + + if ARGS.verbose: + print(f'Checking {fname}') + filep = FileParser(fname) + + checks = AllChecks(filep) + + checks.run_all_checks() + + if (ARGS.fix and checks.modified) or (ARGS.correct and checks.corrected): + print(f'Saving updates to {fname}') + checks.save(fname) + return False + + if checks.modified and not ARGS.fix: + return True + return False + + +def main(): + """Do something""" + parser = argparse.ArgumentParser(description='Verify DAOS logging in source tree') + parser.add_argument('--fix', action='store_true', help='Apply required fixes') + parser.add_argument('--correct', action='store_true', help='Apply optional fixes') + parser.add_argument('--github', action='store_true') + parser.add_argument('-v', '--verbose', action='store_true') + parser.add_argument('files', nargs='*') + + global ARGS + + ARGS = parser.parse_args() + unfixed_errors = False + + for fname in ARGS.files: + if os.path.isfile(fname): + one_entry(fname) + else: + for root, dirs, files in os.walk(fname): + for name in files: + if one_entry(os.path.join(root, name)): + unfixed_errors = True + if '.git' in dirs: + dirs.remove('.git') + if root == 'src/control' and 'vendor' in dirs: + dirs.remove('vendor') + if unfixed_errors: + print('Required fixes not applied') + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/utils/cq/daos_pylint.py b/utils/cq/daos_pylint.py index 05393e22df0..22e1fbc997c 100755 --- a/utils/cq/daos_pylint.py +++ b/utils/cq/daos_pylint.py @@ -1,21 +1,22 @@ #!/usr/bin/env python3 """Wrapper script for calling pylint""" +import argparse +import json import os -import sys import re -from collections import Counter -import tempfile import subprocess # nosec -import argparse -import json +import sys +import tempfile +from collections import Counter + for arg in sys.argv: if arg.startswith('--import='): sys.path.append(arg[9:]) try: + from pylint.constants import full_version from pylint.lint import Run from pylint.reporters.collecting_reporter import CollectingReporter - from pylint.constants import full_version except ImportError: if os.path.exists('venv'): @@ -23,9 +24,9 @@ f'python{sys.version_info.major}.{sys.version_info.minor}', 'site-packages')) try: + from pylint.constants import full_version from pylint.lint import Run from pylint.reporters.collecting_reporter import CollectingReporter - from pylint.constants import full_version except ImportError: print('detected venv unusable, install pylint to enable this check') sys.exit(0) diff --git a/utils/cq/requirements.txt b/utils/cq/requirements.txt index 071191dece9..a5d00a92fa7 100644 --- a/utils/cq/requirements.txt +++ b/utils/cq/requirements.txt @@ -1,4 +1,4 @@ -# Packages needed to pylint code. Most of these are needed for test rather than build. +# Packages needed to check code. These are needed for linting rather than build. # 94.0 is reporting a couple of errors so stick on 93. avocado-framework<94 avocado-framework-plugin-result-html<94 @@ -6,4 +6,10 @@ avocado-framework-plugin-varianter-yaml-to-mux<94 clustershell paramiko pyenchant -pylint +## flake8 6 removed --diff option which breaks flake precommit hook. +## https://github.com/pycqa/flake8/issues/1389 https://github.com/PyCQA/flake8/pull/1720 +flake8<6.0.0 +isort==5.12 +pylint==3.0.2 +yamllint==1.32.0 +codespell==2.2.6 diff --git a/utils/githooks/pre-commit.d/20-flake.sh b/utils/githooks/pre-commit.d/20-flake.sh index f9973bdabc4..8df37d63384 100755 --- a/utils/githooks/pre-commit.d/20-flake.sh +++ b/utils/githooks/pre-commit.d/20-flake.sh @@ -18,6 +18,7 @@ set -ue echo "Flake8:" if ! command -v flake8 > /dev/null 2>&1; then echo " No flake checking, install flake8 command to improve pre-commit checks" + echo "python3 -m pip install -r ./utils/cq/requirements.txt" exit 0 fi diff --git a/utils/githooks/pre-commit.d/20-isort.sh b/utils/githooks/pre-commit.d/20-isort.sh new file mode 100755 index 00000000000..8b2e496badc --- /dev/null +++ b/utils/githooks/pre-commit.d/20-isort.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Runs isort for the DAOS project. + +set -ue + +if ! command -v isort > /dev/null 2>&1 +then + echo "No isort checking, install isort command to improve pre-commit checks" + echo "python3 -m pip install -r ./utils/cq/requirements.txt" + exit 0 +fi + +echo "isort:" + +echo " Running isort for python imports." +isort --jobs 8 --check-only . diff --git a/utils/githooks/pre-commit.d/30-yamllint.sh b/utils/githooks/pre-commit.d/30-yamllint.sh index 3b5efdb7e93..d395dc25564 100755 --- a/utils/githooks/pre-commit.d/30-yamllint.sh +++ b/utils/githooks/pre-commit.d/30-yamllint.sh @@ -10,7 +10,7 @@ set -ue if ! command -v yamllint > /dev/null 2>&1 then echo "No yaml checking, install yamllint command to improve pre-commit checks" - echo "python3 -m pip install yamllint" + echo "python3 -m pip install -r ./utils/cq/requirements.txt" exit 0 fi diff --git a/utils/node_local_test.py b/utils/node_local_test.py index b05f0110929..1b80f1ae0a0 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -10,29 +10,30 @@ # pylint: disable=too-many-lines -import os -from os.path import join -import sys -import time -import uuid -import json +import argparse import copy -import signal -import pprint -import stat import errno -import argparse -import threading import functools -import traceback -import subprocess # nosec -import tempfile +import json +import os import pickle # nosec +import pprint import re import shutil -import xattr +import signal +import stat +import subprocess # nosec +import sys +import tempfile +import threading +import time +import traceback +import uuid +from os.path import join + import junit_xml import tabulate +import xattr import yaml diff --git a/utils/run_utest.py b/utils/run_utest.py index 6c2068b0698..4b0d7f941e1 100755 --- a/utils/run_utest.py +++ b/utils/run_utest.py @@ -6,18 +6,19 @@ Test script for running all DAOS unit tests """ +import argparse +import json # pylint: disable=broad-except import os -import sys -import json -import argparse -import shutil import re +import shutil import subprocess # nosec +import sys import tempfile import traceback -from junit_xml import TestSuite, TestCase + import yaml +from junit_xml import TestCase, TestSuite def check_version(): From 42865144ab7cd90f2681eb8fd74d7037ad18dca7 Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Tue, 5 Dec 2023 15:25:42 -0800 Subject: [PATCH 07/23] DAOS-14413 cart: Cherry-pick several patches for IV operations (#13412) The first patch makes the backport simpler. The rest are the IV inline buffer optimization and bug fixes of the same. Master commits: 636704ae4f7c40cd16eb92b75758e7ed3114d429 30fa0b512d11822bd95b014fd9b07659697be0d6 15fcd5eb576280db130f0c6de74584e1a19f063e f5c1707e18fd3871cac8ae07b5da719018836ef7 8ecbf440465e15c3bf205daf0f544856fed4b7b8 Remove return code for ivo_on_put. (#12637) Change on_put and ent_put to return void. The return code was never set and never read. DAOS-14413 cart: Using inline buffer for IV operation (#13091) If IV buffer size < 19K, let's use inline buffer for IV operation, instead of bulk to avoid extra overhead. DAOS-14492 cart: wrong private ptr for put (#13163) Wrong private ptr for iv_on_put(). DAOS-14663 iv: Fix erroneous version bump (#13354) The CRT_PROTO_IV_VERSION should have been bumped instead. Bumping this version breaks clients because it affects client RPCs as well. DAOS-14675 cart: missing iv put (#13373) missing iv put for inline IV update. Signed-off-by: Jeff Olivier Signed-off-by: Ashley Pittman Signed-off-by: Di Wang --- src/cart/crt_hg_proc.c | 41 +++ src/cart/crt_init.c | 2 + src/cart/crt_internal.h | 2 + src/cart/crt_internal_types.h | 1 + src/cart/crt_iv.c | 452 ++++++++++++++++++++----------- src/cart/crt_rpc.h | 14 +- src/common/misc.c | 11 +- src/container/container_iv.c | 4 +- src/container/oid_iv.c | 4 +- src/engine/server_iv.c | 20 +- src/include/cart/api.h | 12 + src/include/cart/iv.h | 6 +- src/include/daos_srv/iv.h | 3 +- src/include/gurt/common.h | 32 +++ src/object/obj_rpc.c | 46 ---- src/pool/srv_iv.c | 4 +- src/rebuild/rebuild_iv.c | 4 +- src/tests/ftest/cart/iv_server.c | 4 +- 18 files changed, 413 insertions(+), 249 deletions(-) diff --git a/src/cart/crt_hg_proc.c b/src/cart/crt_hg_proc.c index 6001c04b1a3..47d003ee822 100644 --- a/src/cart/crt_hg_proc.c +++ b/src/cart/crt_hg_proc.c @@ -254,6 +254,47 @@ crt_proc_d_iov_t(crt_proc_t proc, crt_proc_op_t proc_op, d_iov_t *div) return rc; } +int +crt_proc_d_sg_list_t(crt_proc_t proc, crt_proc_op_t proc_op, d_sg_list_t *p) +{ + int i; + int rc; + + if (FREEING(proc_op)) { + /* NB: don't need free in crt_proc_d_iov_t() */ + D_FREE(p->sg_iovs); + return 0; + } + + rc = crt_proc_uint32_t(proc, proc_op, &p->sg_nr); + if (unlikely(rc)) + return rc; + + rc = crt_proc_uint32_t(proc, proc_op, &p->sg_nr_out); + if (unlikely(rc)) + return rc; + + if (p->sg_nr == 0) + return 0; + + if (DECODING(proc_op)) { + D_ALLOC_ARRAY(p->sg_iovs, p->sg_nr); + if (p->sg_iovs == NULL) + return -DER_NOMEM; + } + + for (i = 0; i < p->sg_nr; i++) { + rc = crt_proc_d_iov_t(proc, proc_op, &p->sg_iovs[i]); + if (unlikely(rc)) { + if (DECODING(proc_op)) + D_FREE(p->sg_iovs); + return rc; + } + } + + return rc; +} + static inline int crt_proc_corpc_hdr(crt_proc_t proc, struct crt_corpc_hdr *hdr) { diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c index fcc62c207e5..308b9c946f4 100644 --- a/src/cart/crt_init.c +++ b/src/cart/crt_init.c @@ -49,6 +49,7 @@ crt_lib_init(void) crt_gdata.cg_rpcid = start_rpcid; crt_gdata.cg_num_cores = sysconf(_SC_NPROCESSORS_ONLN); + crt_gdata.cg_iv_inline_limit = 19456; /* 19KB */ } /* Library deinit */ @@ -844,6 +845,7 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) crt_self_test_init(); + crt_iv_init(opt); rc = crt_opc_map_create(); if (rc != 0) { D_ERROR("crt_opc_map_create() failed, "DF_RC"\n", DP_RC(rc)); diff --git a/src/cart/crt_internal.h b/src/cart/crt_internal.h index 1067128249c..64392ee1c00 100644 --- a/src/cart/crt_internal.h +++ b/src/cart/crt_internal.h @@ -91,4 +91,6 @@ crt_hdlr_ctl_get_hostname(crt_rpc_t *rpc_req); void crt_hdlr_ctl_get_pid(crt_rpc_t *rpc_req); +void +crt_iv_init(crt_init_options_t *ops); #endif /* __CRT_INTERNAL_H__ */ diff --git a/src/cart/crt_internal_types.h b/src/cart/crt_internal_types.h index 1041f62bae0..a09bc2b4c49 100644 --- a/src/cart/crt_internal_types.h +++ b/src/cart/crt_internal_types.h @@ -104,6 +104,7 @@ struct crt_gdata { /** credits limitation for #inflight RPCs per target EP CTX */ uint32_t cg_credit_ep_ctx; + uint32_t cg_iv_inline_limit; /** the global opcode map */ struct crt_opc_map *cg_opc_map; /** HG level global data */ diff --git a/src/cart/crt_iv.c b/src/cart/crt_iv.c index 3e65025733a..801959eeee2 100644 --- a/src/cart/crt_iv.c +++ b/src/cart/crt_iv.c @@ -135,6 +135,14 @@ struct crt_ivns_internal { void *cii_user_priv; }; +void +crt_iv_init(crt_init_options_t *ops) +{ + if (ops != NULL && ops->cio_max_unexpected_size > 1024) + crt_gdata.cg_iv_inline_limit = ops->cio_max_expected_size - 1024; + D_INFO("max inline buf size is %u\n", crt_gdata.cg_iv_inline_limit); +} + static void handle_response_cb(const struct crt_cb_info *cb_info); @@ -381,14 +389,13 @@ crt_ivf_finalize(struct iv_fetch_cb_info *iv_info, crt_iv_key_t *iv_key, if (rpc) { /* If there is child to respond to - bulk transfer to it */ - if (output_rc == 0) { + if (output_rc == 0 && iv_info->ifc_child_bulk != CRT_BULK_NULL) { /* Note: function will increment ref count on 'rpc' */ rc = crt_ivf_bulk_transfer(iv_info->ifc_ivns_internal, iv_info->ifc_class_id, iv_key, iv_value, iv_info->ifc_child_bulk, - rpc, - iv_info->ifc_user_priv); + rpc, iv_info->ifc_user_priv); if (rc != 0) D_ERROR("Bulk transfer failed for key=%p\n", iv_key); @@ -399,7 +406,7 @@ crt_ivf_finalize(struct iv_fetch_cb_info *iv_info, crt_iv_key_t *iv_key, output = crt_reply_get(rpc); output->ifo_rc = output_rc; - + output->ifo_sgl = *iv_value; /* Reply can fail */ crt_reply_send(rpc); } @@ -434,7 +441,6 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, struct iv_fetch_cb_info *iv_info; struct crt_iv_fetch_out *output; int rc = 0; - bool put_needed = false; iv_ops = crt_iv_ops_get(ivns_internal, class_id); D_ASSERT(iv_ops != NULL); @@ -450,7 +456,8 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, &kip_entry->kip_pending_fetch_list, struct pending_fetch, pf_link))) { - d_sg_list_t tmp_iv_value = {0}; + d_sg_list_t tmp_iv_value = {0}; + bool put_needed = false; iv_info = pending_fetch->pf_cb_info; @@ -489,8 +496,6 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, CRT_IV_PERM_READ, &tmp_iv_value, &iv_info->ifc_user_priv); - - put_needed = false; if (rc == 0) { put_needed = true; rc = iv_ops->ivo_on_fetch(ivns_internal, @@ -501,13 +506,26 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, if (rc == 0) { /* Function will do IVNS_ADDREF if needed */ - rc = crt_ivf_bulk_transfer(ivns_internal, - class_id, - &iv_info->ifc_iv_key, - &tmp_iv_value, - iv_info->ifc_child_bulk, - iv_info->ifc_child_rpc, - iv_info->ifc_user_priv); + if (iv_info->ifc_child_bulk != CRT_BULK_NULL) { + rc = crt_ivf_bulk_transfer(ivns_internal, class_id, + &iv_info->ifc_iv_key, + &tmp_iv_value, + iv_info->ifc_child_bulk, + iv_info->ifc_child_rpc, + iv_info->ifc_user_priv); + /* bulk transfer callback will put if succeed */ + if (rc == 0) + put_needed = false; + } else { + /* inline reply */ + output = crt_reply_get(iv_info->ifc_child_rpc); + output->ifo_rc = 0; + output->ifo_sgl = tmp_iv_value; + rc = crt_reply_send(iv_info->ifc_child_rpc); + if (rc != 0) + D_ERROR("crt_reply_send(): "DF_RC"\n", + DP_RC(rc)); + } } else { D_ERROR("Failed to process pending request\n"); @@ -517,7 +535,7 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, crt_reply_send(iv_info->ifc_child_rpc); } - if (rc != 0 && put_needed) + if (put_needed) iv_ops->ivo_on_put(ivns_internal, &tmp_iv_value, iv_info->ifc_user_priv); @@ -547,12 +565,8 @@ crt_ivf_pending_reqs_process(struct crt_ivns_internal *ivns_internal, 0, CRT_IV_PERM_READ, &tmp_iv_value, &iv_info->ifc_user_priv); - - put_needed = false; - if (rc == 0) { put_needed = true; - rc = iv_ops->ivo_on_fetch(ivns_internal, &iv_info->ifc_iv_key, 0x0, @@ -940,11 +954,8 @@ crt_ivf_bulk_transfer_done_cb(const struct crt_bulk_cb_info *info) cb_info->tci_class_id); D_ASSERT(iv_ops != NULL); - rc = iv_ops->ivo_on_put(cb_info->tci_ivns_internal, - &cb_info->tci_iv_value, - cb_info->tci_user_priv); - if (rc != 0) - D_ERROR("ivo_on_put(): "DF_RC"\n", DP_RC(rc)); + iv_ops->ivo_on_put(cb_info->tci_ivns_internal, &cb_info->tci_iv_value, + cb_info->tci_user_priv); rc = crt_reply_send(rpc); if (rc != 0) @@ -971,8 +982,7 @@ crt_ivf_bulk_transfer(struct crt_ivns_internal *ivns_internal, crt_bulk_opid_t opid; crt_bulk_t bulk_hdl; struct crt_iv_fetch_out *output; - int size; - int i; + size_t size; int rc2; int rc = 0; @@ -990,10 +1000,7 @@ crt_ivf_bulk_transfer(struct crt_ivns_internal *ivns_internal, } /* Calculate total size of all iovs in sg list */ - size = 0; - for (i = 0; i < iv_value->sg_nr; i++) - size += iv_value->sg_iovs[i].iov_buf_len; - + size = d_sgl_buf_size(iv_value); /* crt_req_decref done in crt_ivf_bulk_transfer_done_cb */ RPC_PUB_ADDREF(rpc); @@ -1053,6 +1060,7 @@ handle_ivfetch_response(const struct crt_cb_info *cb_info) struct crt_iv_ops *iv_ops; struct crt_ivns_internal *ivns; struct ivf_key_in_progress *kip_entry; + d_sg_list_t *iv_value = NULL; uint32_t class_id; int rc; @@ -1069,11 +1077,17 @@ handle_ivfetch_response(const struct crt_cb_info *cb_info) IV_DBG(&input->ifi_key, "response received, rc = %d\n", rc); + if (rc == 0) { + /* Inline IV fetch */ + if (iv_info->ifc_bulk_hdl == NULL) + d_sgl_buf_copy(&iv_info->ifc_iv_value, &output->ifo_sgl); + + iv_value = &iv_info->ifc_iv_value; + } + /* In case of a failure, call on_refresh with NULL iv_value */ - iv_ops->ivo_on_refresh(ivns, &input->ifi_key, - 0, /* TODO: iv_ver */ - rc == 0 ? &iv_info->ifc_iv_value : NULL, - false, rc, iv_info->ifc_user_priv); + iv_ops->ivo_on_refresh(ivns, &input->ifi_key, 0, /* TODO: iv_ver */ + iv_value, false, rc, iv_info->ifc_user_priv); if (iv_info->ifc_bulk_hdl) crt_bulk_free(iv_info->ifc_bulk_hdl); @@ -1171,11 +1185,14 @@ crt_ivf_rpc_issue(d_rank_t dest_node, crt_iv_key_t *iv_key, D_MUTEX_UNLOCK(&entry->kip_lock); D_MUTEX_UNLOCK(&ivns_internal->cii_lock); - rc = crt_bulk_create(ivns_internal->cii_ctx, iv_value, CRT_BULK_RW, - &local_bulk); - if (rc != 0) { - D_ERROR("crt_bulk_create(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(exit, rc); + if (d_sgl_buf_size(iv_value) > crt_gdata.cg_iv_inline_limit) { + IV_DBG(iv_key, "bulk transfer with size %zu\n", d_sgl_buf_size(iv_value)); + rc = crt_bulk_create(ivns_internal->cii_ctx, iv_value, CRT_BULK_RW, + &local_bulk); + if (rc != 0) { + D_ERROR("crt_bulk_create(): "DF_RC"\n", DP_RC(rc)); + D_GOTO(exit, rc); + } } /* Note: destination node is using global rank already */ @@ -1192,9 +1209,10 @@ crt_ivf_rpc_issue(d_rank_t dest_node, crt_iv_key_t *iv_key, input = crt_req_get(rpc); D_ASSERT(input != NULL); - input->ifi_value_bulk = local_bulk; - - cb_info->ifc_bulk_hdl = local_bulk; + if (local_bulk != CRT_BULK_NULL) { + input->ifi_value_bulk = local_bulk; + cb_info->ifc_bulk_hdl = local_bulk; + } d_iov_set(&input->ifi_key, iv_key->iov_buf, iv_key->iov_buf_len); input->ifi_class_id = cb_info->ifc_class_id; @@ -1310,12 +1328,11 @@ crt_hdlr_iv_fetch_aux(void *arg) d_sg_list_t iv_value = {0}; bool put_needed = false; void *user_priv = NULL; - crt_rpc_t *rpc_req; + crt_rpc_t *rpc_req = arg; uint32_t grp_ver_entry; uint32_t grp_ver_current; int rc = 0; - rpc_req = arg; input = crt_req_get(rpc_req); output = crt_reply_get(rpc_req); @@ -1326,7 +1343,7 @@ crt_hdlr_iv_fetch_aux(void *arg) ivns_internal = crt_ivns_internal_lookup(&ivns_id); if (ivns_internal == NULL) { D_ERROR("Failed to lookup ivns internal!\n"); - D_GOTO(send_error, rc = -DER_NONEXIST); + D_GOTO(reply_direct, rc = -DER_NONEXIST); } /* This function is called with ivns_internal ref count held. Since @@ -1349,13 +1366,13 @@ crt_hdlr_iv_fetch_aux(void *arg) "Group (%s) version mismatch. Local: %d Remote :%d\n", ivns_id.ii_group_name, grp_ver_entry, input->ifi_grp_ver); - D_GOTO(send_error, rc = -DER_GRPVER); + D_GOTO(reply_direct, rc = -DER_GRPVER); } iv_ops = crt_iv_ops_get(ivns_internal, input->ifi_class_id); if (iv_ops == NULL) { D_ERROR("Returned iv_ops were NULL\n"); - D_GOTO(send_error, rc = -DER_INVAL); + D_GOTO(reply_direct, rc = -DER_INVAL); } IV_DBG(&input->ifi_key, "fetch handler entered\n"); @@ -1363,23 +1380,28 @@ crt_hdlr_iv_fetch_aux(void *arg) 0, CRT_IV_PERM_READ, &iv_value, &user_priv); if (rc != 0) { D_ERROR("ivo_on_get(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(send_error, rc); + D_GOTO(reply_direct, rc); } put_needed = true; - rc = iv_ops->ivo_on_fetch(ivns_internal, &input->ifi_key, 0, 0x0, &iv_value, user_priv); if (rc == 0) { /* Note: This increments ref count on 'rpc_req' and ivns */ - rc = crt_ivf_bulk_transfer(ivns_internal, - input->ifi_class_id, - &input->ifi_key, - &iv_value, input->ifi_value_bulk, - rpc_req, user_priv); - if (rc != 0) { - D_ERROR("bulk transfer failed; "DF_RC"\n", DP_RC(rc)); - D_GOTO(send_error, rc); + if (input->ifi_value_bulk != CRT_BULK_NULL) { + rc = crt_ivf_bulk_transfer(ivns_internal, + input->ifi_class_id, + &input->ifi_key, + &iv_value, input->ifi_value_bulk, + rpc_req, user_priv); + if (rc != 0) { + DL_ERROR(rc, "bulk transfer failed"); + D_GOTO(reply_direct, rc); + } + put_needed = false; + } else { + output->ifo_sgl = iv_value; + D_GOTO(reply_direct, rc); } } else if (rc == -DER_IVCB_FORWARD) { /* Forward the request to the parent */ @@ -1389,15 +1411,11 @@ crt_hdlr_iv_fetch_aux(void *arg) if (ivns_internal->cii_grp_priv->gp_self == input->ifi_root_node) { D_ERROR("Forward requested for root node\n"); - D_GOTO(send_error, rc = -DER_INVAL); + D_GOTO(reply_direct, rc = -DER_INVAL); } - rc = iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); + iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); put_needed = false; - if (rc != 0) { - D_ERROR("ivo_on_put(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(send_error, rc); - } /* Reset the iv_value, since it maybe freed in on_put() */ memset(&iv_value, 0, sizeof(iv_value)); @@ -1406,7 +1424,7 @@ crt_hdlr_iv_fetch_aux(void *arg) &user_priv); if (rc != 0) { D_ERROR("ivo_on_get(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(send_error, rc); + D_GOTO(reply_direct, rc); } put_needed = true; @@ -1420,7 +1438,7 @@ crt_hdlr_iv_fetch_aux(void *arg) if (rc != 0) { D_DEBUG(DB_TRACE, "crt_iv_parent_get() returned %d\n", rc); - D_GOTO(send_error, rc = -DER_OOG); + D_GOTO(reply_direct, rc = -DER_OOG); } /* Check here for change in group */ @@ -1429,12 +1447,12 @@ crt_hdlr_iv_fetch_aux(void *arg) "On Entry: %d:: Changed To :%d\n", ivns_id.ii_group_name, grp_ver_entry, grp_ver_current); - D_GOTO(send_error, rc = -DER_GRPVER); + D_GOTO(reply_direct, rc = -DER_GRPVER); } D_ALLOC_PTR(cb_info); if (cb_info == NULL) - D_GOTO(send_error, rc = -DER_NOMEM); + D_GOTO(reply_direct, rc = -DER_NOMEM); cb_info->ifc_child_rpc = rpc_req; cb_info->ifc_child_bulk = input->ifi_value_bulk; @@ -1462,11 +1480,11 @@ crt_hdlr_iv_fetch_aux(void *arg) IVNS_DECREF(cb_info->ifc_ivns_internal); D_FREE(cb_info); - D_GOTO(send_error, rc); + D_GOTO(reply_direct, rc); } } else { D_ERROR("ERROR happened: "DF_RC"\n", DP_RC(rc)); - D_GOTO(send_error, rc); + D_GOTO(reply_direct, rc); } /* addref in crt_hdlr_iv_fetch */ @@ -1477,16 +1495,14 @@ crt_hdlr_iv_fetch_aux(void *arg) IVNS_DECREF(ivns_internal); return; -send_error: - if (put_needed && iv_ops) - iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); +reply_direct: output->ifo_rc = rc; rc = crt_reply_send(rpc_req); - if (rc != DER_SUCCESS) { + if (rc != DER_SUCCESS) D_ERROR("crt_reply_send(opc: %#x): "DF_RC"\n", rpc_req->cr_opc, DP_RC(rc)); - } - + if (put_needed && iv_ops) + iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); /* ADDREF done in lookup above */ if (ivns_internal) IVNS_DECREF(ivns_internal); @@ -1798,7 +1814,6 @@ crt_hdlr_iv_sync_aux(void *arg) struct crt_iv_ops *iv_ops = NULL; struct crt_ivns_id ivns_id; crt_iv_sync_t *sync_type; - d_sg_list_t iv_value = {0}; bool need_put = false; void *user_priv = NULL; crt_rpc_t *rpc_req; @@ -1849,7 +1864,7 @@ crt_hdlr_iv_sync_aux(void *arg) D_ASSERT(iv_ops != NULL); /* If bulk is not set, we issue invalidate call */ - if (rpc_req->cr_co_bulk_hdl == CRT_BULK_NULL) { + if (rpc_req->cr_co_bulk_hdl == CRT_BULK_NULL && input->ivs_sync_sgl.sg_nr == 0) { rc = iv_ops->ivo_on_refresh(ivns_internal, &input->ivs_key, 0, NULL, true, 0x0, NULL); D_GOTO(exit, rc); @@ -1860,10 +1875,10 @@ crt_hdlr_iv_sync_aux(void *arg) case CRT_IV_SYNC_EVENT_UPDATE: { d_sg_list_t tmp_iv; - d_iov_t *tmp_iovs; + d_iov_t tmp_iovs; rc = iv_ops->ivo_on_get(ivns_internal, &input->ivs_key, - 0, CRT_IV_PERM_READ, &iv_value, + 0, CRT_IV_PERM_READ | CRT_IV_NO_ALLOC, &tmp_iv, &user_priv); if (rc != 0) { D_ERROR("ivo_on_get(): "DF_RC"\n", DP_RC(rc)); @@ -1872,36 +1887,27 @@ crt_hdlr_iv_sync_aux(void *arg) need_put = true; - D_ALLOC_ARRAY(tmp_iovs, iv_value.sg_nr); - if (tmp_iovs == NULL) { - D_GOTO(exit, rc = -DER_NOMEM); - } - - tmp_iv.sg_nr = iv_value.sg_nr; - tmp_iv.sg_iovs = tmp_iovs; - - /* Populate tmp_iv.sg_iovs[0] to [sg_nr] */ - rc = crt_bulk_access(rpc_req->cr_co_bulk_hdl, &tmp_iv); - if (rc != 0) { - D_FREE(tmp_iovs); - D_ERROR("crt_bulk_access(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(exit, rc); + if (rpc_req->cr_co_bulk_hdl != CRT_BULK_NULL) { + tmp_iv.sg_iovs = &tmp_iovs; + /* Populate tmp_iv.sg_iovs[0] to [sg_nr] */ + rc = crt_bulk_access(rpc_req->cr_co_bulk_hdl, &tmp_iv); + if (rc != 0) { + D_ERROR("crt_bulk_access(): "DF_RC"\n", DP_RC(rc)); + D_GOTO(exit, rc); + } + } else if (input->ivs_sync_sgl.sg_nr > 0) { + tmp_iv = input->ivs_sync_sgl; } rc = iv_ops->ivo_on_refresh(ivns_internal, &input->ivs_key, 0, &tmp_iv, false, 0, user_priv); - D_FREE(tmp_iovs); if (rc != 0) { D_ERROR("ivo_on_refresh(): "DF_RC"\n", DP_RC(rc)); D_GOTO(exit, rc); } - rc = iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); + iv_ops->ivo_on_put(ivns_internal, NULL, user_priv); need_put = false; - if (rc != 0) { - D_ERROR("ivo_on_put(): "DF_RC"\n", DP_RC(rc)); - D_GOTO(exit, rc); - } break; } @@ -1924,7 +1930,7 @@ crt_hdlr_iv_sync_aux(void *arg) exit: if (need_put && iv_ops) - iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); + iv_ops->ivo_on_put(ivns_internal, NULL, user_priv); output->rc = rc; crt_reply_send(rpc_req); @@ -2072,6 +2078,8 @@ call_pre_sync_cb(struct crt_ivns_internal *ivns_internal, D_ERROR("crt_bulk_access(): "DF_RC"\n", DP_RC(rc)); D_GOTO(exit, rc); } + } else if (input->ivs_sync_sgl.sg_nr > 0) { + tmp_iv = input->ivs_sync_sgl; } D_DEBUG(DB_TRACE, "Executing ivo_pre_sync\n"); @@ -2264,8 +2272,9 @@ crt_ivsync_rpc_issue(struct crt_ivns_internal *ivns_internal, uint32_t class_id, } local_bulk = CRT_BULK_NULL; - if (iv_value != NULL) { - D_DEBUG(DB_TRACE, "Create Bulk\n"); + if (iv_value != NULL && d_sgl_buf_size(iv_value) > crt_gdata.cg_iv_inline_limit) { + IV_DBG(iv_key, "bulk transfer with size %zu\n", + d_sgl_buf_size(iv_value)); rc = crt_bulk_create(ivns_internal->cii_ctx, iv_value, CRT_BULK_RO, &local_bulk); if (rc != 0) { @@ -2290,10 +2299,8 @@ crt_ivsync_rpc_issue(struct crt_ivns_internal *ivns_internal, uint32_t class_id, D_ASSERT(input != NULL); D_ALLOC_PTR(iv_sync_cb); - if (iv_sync_cb == NULL) { - /* Avoid checkpatch warning */ + if (iv_sync_cb == NULL) D_GOTO(exit, rc = -DER_NOMEM); - } iv_sync_cb->isc_sync_type = *sync_type; input->ivs_ivns_id = ivns_internal->cii_gns.gn_ivns_id.ii_nsid; @@ -2305,6 +2312,10 @@ crt_ivsync_rpc_issue(struct crt_ivns_internal *ivns_internal, uint32_t class_id, d_iov_set(&input->ivs_sync_type, &iv_sync_cb->isc_sync_type, sizeof(crt_iv_sync_t)); + /* Set inline IV buffer if needed */ + if (iv_value != NULL && local_bulk == CRT_BULK_NULL) + input->ivs_sync_sgl = *iv_value; + input->ivs_class_id = class_id; iv_sync_cb->isc_bulk_hdl = local_bulk; @@ -2414,7 +2425,8 @@ finalize_transfer_back(struct update_cb_info *cb_info, int rc) crt_reply_send(cb_info->uci_child_rpc); /* ADDREF done in crt_hdlr_iv_update */ - crt_bulk_free(cb_info->uci_bulk_hdl); + if (cb_info->uci_bulk_hdl != CRT_BULK_NULL) + crt_bulk_free(cb_info->uci_bulk_hdl); iv_ops = crt_iv_ops_get(ivns, cb_info->uci_class_id); D_ASSERT(iv_ops != NULL); @@ -2439,16 +2451,16 @@ bulk_update_transfer_back_done(const struct crt_bulk_cb_info *info) } /* Helper function to transfer iv_value back to child */ -static -int transfer_back_to_child(crt_iv_key_t *key, struct update_cb_info *cb_info, - bool do_refresh, int update_rc) +static int +transfer_back_to_child(crt_iv_key_t *key, struct update_cb_info *cb_info, + int update_rc) { struct crt_bulk_desc bulk_desc = {0}; struct crt_iv_update_in *child_input; + struct crt_iv_update_out *child_output; struct crt_ivns_internal *ivns; struct crt_iv_ops *iv_ops; - int size = 0; - int i; + size_t size = 0; int rc = 0; ivns = cb_info->uci_ivns_internal; @@ -2456,11 +2468,6 @@ int transfer_back_to_child(crt_iv_key_t *key, struct update_cb_info *cb_info, iv_ops = crt_iv_ops_get(ivns, cb_info->uci_class_id); D_ASSERT(iv_ops != NULL); - if (do_refresh) - iv_ops->ivo_on_refresh(ivns, key, 0, - &cb_info->uci_iv_value, - false, update_rc, cb_info->uci_user_priv); - /* No more children -- we are the originator; call update_cb */ if (cb_info->uci_child_rpc == NULL) { cb_info->uci_comp_cb(ivns, cb_info->uci_class_id, key, NULL, @@ -2479,13 +2486,18 @@ int transfer_back_to_child(crt_iv_key_t *key, struct update_cb_info *cb_info, return 0; } - /* Perform bulk transfer back to the child */ child_input = crt_req_get(cb_info->uci_child_rpc); + if (child_input->ivu_iv_value_bulk == CRT_BULK_NULL) { + child_output = crt_reply_get(cb_info->uci_child_rpc); + D_ASSERT(update_rc != 0 || child_output->ivo_iv_sgl.sg_nr > 0); + finalize_transfer_back(cb_info, update_rc); + return 0; + } - /* Calculate size of iv value */ - for (i = 0; i < cb_info->uci_iv_value.sg_nr; i++) - size += cb_info->uci_iv_value.sg_iovs[i].iov_buf_len; + /* Perform bulk transfer back to the child */ + /* Calculate size of iv value */ + size = d_sgl_buf_size(&cb_info->uci_iv_value); bulk_desc.bd_rpc = cb_info->uci_child_rpc; bulk_desc.bd_bulk_op = CRT_BULK_PUT; bulk_desc.bd_remote_hdl = child_input->ivu_iv_value_bulk; @@ -2515,36 +2527,35 @@ handle_ivupdate_response(const struct crt_cb_info *cb_info) struct crt_iv_ops *iv_ops; int rc; + iv_ops = crt_iv_ops_get(iv_info->uci_ivns_internal, iv_info->uci_class_id); + D_ASSERT(iv_ops != NULL); + /* For bi-directional updates, transfer data back to child */ if (iv_info->uci_sync_type.ivs_flags & CRT_IV_SYNC_BIDIRECTIONAL) { - transfer_back_to_child(&input->ivu_key, iv_info, true, - cb_info->cci_rc ?: output->rc); + if (iv_info->uci_bulk_hdl == CRT_BULK_NULL) + d_sgl_buf_copy(&iv_info->uci_iv_value, &output->ivo_iv_sgl); + iv_ops->ivo_on_refresh(iv_info->uci_ivns_internal, &input->ivu_key, 0, + &iv_info->uci_iv_value, false, cb_info->cci_rc ?: output->rc, + iv_info->uci_user_priv); + + if (input->ivu_iv_value_bulk == CRT_BULK_NULL && + iv_info->uci_child_rpc != NULL) { + child_output = crt_reply_get(iv_info->uci_child_rpc); + child_output->ivo_iv_sgl = iv_info->uci_iv_value; + } + transfer_back_to_child(&input->ivu_key, iv_info, cb_info->cci_rc ?: output->rc); D_GOTO(exit, 0); } - iv_ops = crt_iv_ops_get(iv_info->uci_ivns_internal, - iv_info->uci_class_id); - D_ASSERT(iv_ops != NULL); - if (iv_info->uci_child_rpc) { child_output = crt_reply_get(iv_info->uci_child_rpc); /* uci_bulk_hdl will not be set for invalidate call */ - if (iv_info->uci_bulk_hdl != CRT_BULK_NULL) { + if (iv_info->uci_bulk_hdl != CRT_BULK_NULL) crt_bulk_free(iv_info->uci_bulk_hdl); - rc = iv_ops->ivo_on_put(iv_info->uci_ivns_internal, - &iv_info->uci_iv_value, - iv_info->uci_user_priv); - - if (rc != 0) { - D_ERROR("ivo_on_put(): "DF_RC"\n", DP_RC(rc)); - child_output->rc = rc; - } else { - child_output->rc = output->rc; - } - } else { - child_output->rc = output->rc; - } + iv_ops->ivo_on_put(iv_info->uci_ivns_internal, &iv_info->uci_iv_value, + iv_info->uci_user_priv); + child_output->rc = output->rc; if (cb_info->cci_rc != 0) child_output->rc = cb_info->cci_rc; @@ -2559,7 +2570,7 @@ handle_ivupdate_response(const struct crt_cb_info *cb_info) } else { d_sg_list_t *tmp_iv_value; - if (iv_info->uci_bulk_hdl == CRT_BULK_NULL) + if (iv_info->uci_iv_value.sg_nr == 0) tmp_iv_value = NULL; else tmp_iv_value = &iv_info->uci_iv_value; @@ -2583,9 +2594,8 @@ handle_ivupdate_response(const struct crt_cb_info *cb_info) if (iv_info->uci_bulk_hdl != CRT_BULK_NULL) crt_bulk_free(iv_info->uci_bulk_hdl); if (rc != 0) { - rc = iv_ops->ivo_on_put(iv_info->uci_ivns_internal, - tmp_iv_value, - iv_info->uci_user_priv); + iv_ops->ivo_on_put(iv_info->uci_ivns_internal, tmp_iv_value, + iv_info->uci_user_priv); } } @@ -2628,9 +2638,14 @@ crt_ivu_rpc_issue(d_rank_t dest_rank, crt_iv_key_t *iv_key, /* Update with NULL value is invalidate call */ if (iv_value) { - rc = crt_bulk_create(ivns_internal->cii_ctx, iv_value, - CRT_BULK_RW, &local_bulk); - + if (d_sgl_buf_size(iv_value) > crt_gdata.cg_iv_inline_limit) { + IV_DBG(iv_key, "bulk transfer with size %zu\n", + d_sgl_buf_size(iv_value)); + rc = crt_bulk_create(ivns_internal->cii_ctx, iv_value, + CRT_BULK_RW, &local_bulk); + } else { + input->ivu_iv_sgl = *iv_value; + } if (rc != 0) { D_ERROR("crt_bulk_create(): "DF_RC"\n", DP_RC(rc)); D_GOTO(exit, rc); @@ -2881,18 +2896,16 @@ bulk_update_transfer_done_aux(const struct crt_bulk_cb_info *info) } else if (update_rc == 0) { /* If sync was bi-directional - transfer value back */ if (sync_type->ivs_flags & CRT_IV_SYNC_BIDIRECTIONAL) { - rc = transfer_back_to_child(&input->ivu_key, - update_cb_info, - false, update_rc); + rc = transfer_back_to_child(&input->ivu_key, update_cb_info, + update_rc); if (rc == 0) rc = update_rc; D_GOTO(exit, rc); } crt_bulk_free(cb_info->buc_bulk_hdl); - output->rc = iv_ops->ivo_on_put(ivns_internal, - &cb_info->buc_iv_value, - cb_info->buc_user_priv); + output->rc = -DER_SUCCESS; + iv_ops->ivo_on_put(ivns_internal, &cb_info->buc_iv_value, cb_info->buc_user_priv); crt_reply_send(info->bci_bulk_desc->bd_rpc); RPC_PUB_DECREF(info->bci_bulk_desc->bd_rpc); @@ -3018,6 +3031,120 @@ bulk_update_transfer_done(const struct crt_bulk_cb_info *info) return rc; } +static void +crt_iv_update_inline_hdlr(crt_rpc_t *rpc, struct crt_ivns_internal *ivns_internal) +{ + struct crt_iv_ops *iv_ops; + struct crt_iv_update_in *input; + struct crt_iv_update_out *output; + d_sg_list_t iv_value = {0}; + struct update_cb_info *update_cb_info = NULL; + int rc = 0; + d_rank_t next_rank; + crt_iv_sync_t *sync_type; + void *user_priv; + uint32_t grp_ver; + + input = crt_req_get(rpc); + output = crt_reply_get(rpc); + + D_ASSERT(input != NULL); + D_ASSERT(output != NULL); + + iv_ops = crt_iv_ops_get(ivns_internal, input->ivu_class_id); + D_ASSERT(iv_ops != NULL); + + rc = iv_ops->ivo_on_get(ivns_internal, &input->ivu_key, 0, + CRT_IV_PERM_WRITE, &iv_value, &user_priv); + if (rc != 0) { + D_ERROR("ivo_on_get(): "DF_RC"\n", DP_RC(rc)); + D_GOTO(send_error, rc); + } + + d_sgl_buf_copy(&iv_value, &input->ivu_iv_sgl); + + rc = iv_ops->ivo_on_update(ivns_internal, &input->ivu_key, 0, false, + &iv_value, user_priv); + + sync_type = input->ivu_sync_type.iov_buf; + + if (rc == -DER_IVCB_FORWARD || + (rc == 0 && sync_type->ivs_flags & CRT_IV_SYNC_BIDIRECTIONAL)) { + + D_ALLOC_PTR(update_cb_info); + if (update_cb_info == NULL) + D_GOTO(put_error, rc = -DER_NOMEM); + + update_cb_info->uci_child_rpc = rpc; + + update_cb_info->uci_ivns_internal = ivns_internal; + IVNS_ADDREF(ivns_internal); + + update_cb_info->uci_class_id = input->ivu_class_id; + update_cb_info->uci_caller_rank = input->ivu_caller_node; + update_cb_info->uci_sync_type = *sync_type; + update_cb_info->uci_user_priv = user_priv; + update_cb_info->uci_iv_value = iv_value; + if (rc == -DER_IVCB_FORWARD) { + /* + * Forward request to the parent + * Get group version to associate with next_rank. + * Pass it down to crt_ivu_rpc_issue + */ + D_RWLOCK_RDLOCK(&ivns_internal->cii_grp_priv->gp_rwlock); + grp_ver = ivns_internal->cii_grp_priv->gp_membs_ver; + + rc = crt_iv_parent_get(ivns_internal, input->ivu_root_node, &next_rank); + D_RWLOCK_UNLOCK(&ivns_internal->cii_grp_priv->gp_rwlock); + + if (rc != 0) { + D_DEBUG(DB_TRACE, "crt_iv_parent_get() returned %d\n", rc); + D_GOTO(decref_error, rc = -DER_OOG); + } + + rc = crt_ivu_rpc_issue(next_rank, &input->ivu_key, + &iv_value, sync_type, + input->ivu_root_node, grp_ver, + update_cb_info); + if (rc != 0) { + D_ERROR("crt_ivu_rpc_issue(): "DF_RC"\n", DP_RC(rc)); + D_GOTO(decref_error, rc); + } + RPC_PUB_ADDREF(update_cb_info->uci_child_rpc); + } else { + int rc1; + + /* If sync was bi-directional - transfer value back */ + output->ivo_iv_sgl = iv_value; + RPC_PUB_ADDREF(update_cb_info->uci_child_rpc); + rc1 = transfer_back_to_child(&input->ivu_key, update_cb_info, rc); + if (rc == 0) + rc = rc1; + + D_GOTO(exit, rc); + } + } else if (rc == 0) { + output->rc = rc; + crt_reply_send(rpc); + iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); + } else { + D_GOTO(put_error, rc); + } +exit: + return; + +decref_error: + IVNS_DECREF(ivns_internal); +put_error: + iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); + +send_error: + output->rc = rc; + crt_reply_send(rpc); + if (update_cb_info) + D_FREE(update_cb_info); +} + /* IV UPDATE RPC handler */ void crt_hdlr_iv_update(crt_rpc_t *rpc_req) @@ -3034,9 +3161,8 @@ crt_hdlr_iv_update(crt_rpc_t *rpc_req) crt_iv_sync_t *sync_type; d_rank_t next_rank; struct update_cb_info *update_cb_info; - int size; + size_t size; void *user_priv; - int i; uint32_t grp_ver_entry; uint32_t grp_ver_current; bool put_needed = false; @@ -3078,7 +3204,7 @@ crt_hdlr_iv_update(crt_rpc_t *rpc_req) D_GOTO(send_error, rc = -DER_INVAL); } - if (input->ivu_iv_value_bulk == CRT_BULK_NULL) { + if (input->ivu_iv_value_bulk == CRT_BULK_NULL && input->ivu_iv_sgl.sg_nr == 0) { rc = iv_ops->ivo_on_refresh(ivns_internal, &input->ivu_key, 0, NULL, true, 0, NULL); if (rc == -DER_IVCB_FORWARD) { @@ -3156,6 +3282,11 @@ crt_hdlr_iv_update(crt_rpc_t *rpc_req) D_GOTO(exit, rc = 0); } + if (input->ivu_iv_value_bulk == CRT_BULK_NULL) { + crt_iv_update_inline_hdlr(rpc_req, ivns_internal); + D_GOTO(exit, rc); + } + rc = iv_ops->ivo_on_get(ivns_internal, &input->ivu_key, 0, CRT_IV_PERM_WRITE, &iv_value, &user_priv); if (rc != 0) { @@ -3164,10 +3295,7 @@ crt_hdlr_iv_update(crt_rpc_t *rpc_req) } put_needed = true; - size = 0; - for (i = 0; i < iv_value.sg_nr; i++) - size += iv_value.sg_iovs[i].iov_buf_len; - + size = d_sgl_buf_size(&iv_value); rc = crt_bulk_create(rpc_req->cr_ctx, &iv_value, CRT_BULK_RW, &local_bulk_handle); if (rc != 0) { @@ -3219,7 +3347,7 @@ crt_hdlr_iv_update(crt_rpc_t *rpc_req) crt_reply_send(rpc_req); if (put_needed) - iv_ops->ivo_on_put(ivns_internal, &iv_value, &user_priv); + iv_ops->ivo_on_put(ivns_internal, &iv_value, user_priv); /* ADDREF done in lookup above */ if (ivns_internal) diff --git a/src/cart/crt_rpc.h b/src/cart/crt_rpc.h index 6ddda306452..d297d2f13f0 100644 --- a/src/cart/crt_rpc.h +++ b/src/cart/crt_rpc.h @@ -220,7 +220,7 @@ crt_rpc_unlock(struct crt_rpc_priv *rpc_priv) #define CRT_PROTO_FI_VERSION 3 #define CRT_PROTO_ST_VERSION 1 #define CRT_PROTO_CTL_VERSION 1 -#define CRT_PROTO_IV_VERSION 1 +#define CRT_PROTO_IV_VERSION 2 /* LIST of internal RPCS in form of: * OPCODE, flags, FMT, handler, corpc_hdlr, @@ -308,7 +308,7 @@ crt_rpc_unlock(struct crt_rpc_priv *rpc_priv) crt_hdlr_iv_update, NULL) \ X(CRT_OPC_IV_SYNC, \ 0, &CQF_crt_iv_sync, \ - crt_hdlr_iv_sync, &crt_iv_sync_co_ops) \ + crt_hdlr_iv_sync, &crt_iv_sync_co_ops) /* Define for RPC enum population below */ #define X(a, b, c, d, e) a, @@ -476,7 +476,8 @@ CRT_RPC_DECLARE(crt_st_status_req, ((d_rank_t) (ifi_root_node) CRT_VAR) #define CRT_OSEQ_IV_FETCH /* output fields */ \ - ((int32_t) (ifo_rc) CRT_VAR) + ((d_sg_list_t) (ifo_sgl) CRT_VAR) \ + ((int32_t) (ifo_rc) CRT_VAR) \ CRT_RPC_DECLARE(crt_iv_fetch, CRT_ISEQ_IV_FETCH, CRT_OSEQ_IV_FETCH) @@ -491,6 +492,7 @@ CRT_RPC_DECLARE(crt_iv_fetch, CRT_ISEQ_IV_FETCH, CRT_OSEQ_IV_FETCH) ((d_iov_t) (ivu_sync_type) CRT_VAR) \ /* Bulk handle for iv value */ \ ((crt_bulk_t) (ivu_iv_value_bulk) CRT_VAR) \ + ((d_sg_list_t) (ivu_iv_sgl) CRT_VAR) \ /* Root node for IV UPDATE */ \ ((d_rank_t) (ivu_root_node) CRT_VAR) \ /* Original node that issued crt_iv_update call */ \ @@ -500,7 +502,8 @@ CRT_RPC_DECLARE(crt_iv_fetch, CRT_ISEQ_IV_FETCH, CRT_OSEQ_IV_FETCH) ((uint32_t) (padding) CRT_VAR) #define CRT_OSEQ_IV_UPDATE /* output fields */ \ - ((uint64_t) (rc) CRT_VAR) + ((uint64_t) (rc) CRT_VAR) \ + ((d_sg_list_t) (ivo_iv_sgl) CRT_VAR) CRT_RPC_DECLARE(crt_iv_update, CRT_ISEQ_IV_UPDATE, CRT_OSEQ_IV_UPDATE) @@ -513,8 +516,9 @@ CRT_RPC_DECLARE(crt_iv_update, CRT_ISEQ_IV_UPDATE, CRT_OSEQ_IV_UPDATE) ((d_iov_t) (ivs_key) CRT_VAR) \ /* IOV for sync type */ \ ((d_iov_t) (ivs_sync_type) CRT_VAR) \ + ((d_sg_list_t) (ivs_sync_sgl) CRT_VAR) \ /* IV Class ID */ \ - ((uint32_t) (ivs_class_id) CRT_VAR) + ((uint32_t) (ivs_class_id) CRT_VAR) \ #define CRT_OSEQ_IV_SYNC /* output fields */ \ ((int32_t) (rc) CRT_VAR) diff --git a/src/common/misc.c b/src/common/misc.c index dc910507d98..124b10b21ea 100644 --- a/src/common/misc.c +++ b/src/common/misc.c @@ -235,16 +235,9 @@ daos_sgl_data_len(d_sg_list_t *sgl) daos_size_t daos_sgl_buf_size(d_sg_list_t *sgl) { - daos_size_t size = 0; - int i; - - if (sgl == NULL || sgl->sg_iovs == NULL) + if (sgl == NULL) return 0; - - for (i = 0, size = 0; i < sgl->sg_nr; i++) - size += sgl->sg_iovs[i].iov_buf_len; - - return size; + return d_sgl_buf_size(sgl); } daos_size_t diff --git a/src/container/container_iv.c b/src/container/container_iv.c index bac555ac158..ff053c14e34 100644 --- a/src/container/container_iv.c +++ b/src/container/container_iv.c @@ -84,10 +84,10 @@ cont_iv_ent_get(struct ds_iv_entry *entry, void **priv) return 0; } -static int +static void cont_iv_ent_put(struct ds_iv_entry *entry, void *priv) { - return 0; + return; } static int diff --git a/src/container/oid_iv.c b/src/container/oid_iv.c index cf89be761b8..d1041184006 100644 --- a/src/container/oid_iv.c +++ b/src/container/oid_iv.c @@ -210,14 +210,12 @@ oid_iv_ent_get(struct ds_iv_entry *entry, void **_priv) return 0; } -static int +static void oid_iv_ent_put(struct ds_iv_entry *entry, void *priv) { D_ASSERT(priv != NULL); D_DEBUG(DB_TRACE, "%u: ON PUT\n", dss_self_rank()); D_FREE(priv); - - return 0; } static int diff --git a/src/engine/server_iv.c b/src/engine/server_iv.c index 29e9218eff4..05542d916b0 100644 --- a/src/engine/server_iv.c +++ b/src/engine/server_iv.c @@ -595,9 +595,13 @@ ivc_on_get(crt_iv_namespace_t ivns, crt_iv_key_t *iv_key, class = entry->iv_class; if (iv_value) { - rc = class->iv_class_ops->ivc_value_alloc(entry, &key, iv_value); - if (rc) - D_GOTO(out, rc); + if (permission & CRT_IV_NO_ALLOC) { + iv_value->sg_nr = 1; + } else { + rc = class->iv_class_ops->ivc_value_alloc(entry, &key, iv_value); + if (rc) + D_GOTO(out, rc); + } } rc = class->iv_class_ops->ivc_ent_get(entry, &entry_priv_val); @@ -626,7 +630,7 @@ ivc_on_get(crt_iv_namespace_t ivns, crt_iv_key_t *iv_key, return rc; } -static int +static void ivc_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *priv) { struct ds_iv_ns *ns = NULL; @@ -638,7 +642,7 @@ ivc_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *priv) if (rc != 0) { if (ns != NULL) ds_iv_ns_put(ns); /* balance ivc_on_get */ - return rc; + return; } D_ASSERT(ns != NULL); @@ -652,9 +656,7 @@ ivc_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *priv) /* Let's deal with iv_value first */ d_sgl_fini(iv_value, true); - rc = entry->iv_class->iv_class_ops->ivc_ent_put(entry, priv_entry->priv); - if (rc) - D_GOTO(put, rc); + entry->iv_class->iv_class_ops->ivc_ent_put(entry, priv_entry->priv); D_FREE(priv_entry); if (--entry->iv_ref > 0) @@ -668,7 +670,7 @@ ivc_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *priv) ds_iv_ns_put(ns); ds_iv_ns_put(ns); - return rc; + return; } static int diff --git a/src/include/cart/api.h b/src/include/cart/api.h index 6bf98811e47..402d17f83ed 100644 --- a/src/include/cart/api.h +++ b/src/include/cart/api.h @@ -1632,6 +1632,18 @@ crt_proc_d_rank_list_t(crt_proc_t proc, crt_proc_op_t proc_op, int crt_proc_d_iov_t(crt_proc_t proc, crt_proc_op_t proc_op, d_iov_t *data); +/** + * Generic processing routine. + * + * \param[in,out] proc abstract processor object + * \param[in] proc_op proc operation type + * \param[in,out] data pointer to data + * + * \return DER_SUCCESS on success, negative value if error + */ +int +crt_proc_d_sg_list_t(crt_proc_t proc, crt_proc_op_t proc_op, d_sg_list_t *data); + /** * Create the processor object. * diff --git a/src/include/cart/iv.h b/src/include/cart/iv.h index 1907ae791f7..db5ab93d34e 100644 --- a/src/include/cart/iv.h +++ b/src/include/cart/iv.h @@ -239,6 +239,7 @@ typedef int (*crt_iv_on_hash_cb_t)(crt_iv_namespace_t ivns, typedef enum { CRT_IV_PERM_READ = 0x1, CRT_IV_PERM_WRITE = 0x2, + CRT_IV_NO_ALLOC = 0x4, } crt_iv_perm_t; /** @@ -287,11 +288,8 @@ typedef int (*crt_iv_on_get_cb_t)(crt_iv_namespace_t ivns, * \param[in] iv_value iv_value buffers to return * \param[in] arg private user data * - * \return DER_SUCCESS on success, negative value if error */ -typedef int (*crt_iv_on_put_cb_t)(crt_iv_namespace_t ivns, - d_sg_list_t *iv_value, - void *arg); +typedef void (*crt_iv_on_put_cb_t)(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *arg); /** * Compares two passed iv keys 'key1' and 'key2' and returns either diff --git a/src/include/daos_srv/iv.h b/src/include/daos_srv/iv.h index 0895b0c0fa3..75e12481e7c 100644 --- a/src/include/daos_srv/iv.h +++ b/src/include/daos_srv/iv.h @@ -164,9 +164,8 @@ typedef int (*ds_iv_ent_get_t)(struct ds_iv_entry *ent, void **priv); * \param ent [IN] iv class entry to get. * \param priv [IN] private ptr from crt IV callback. * - * \return 0 if succeeds, error code otherwise. */ -typedef int (*ds_iv_ent_put_t)(struct ds_iv_entry *ent, void *priv); +typedef void (*ds_iv_ent_put_t)(struct ds_iv_entry *ent, void *priv); /** * Destroy the data attached to the entry. diff --git a/src/include/gurt/common.h b/src/include/gurt/common.h index b55e4c53cd6..94b794a6d55 100644 --- a/src/include/gurt/common.h +++ b/src/include/gurt/common.h @@ -462,6 +462,38 @@ d_sgl_fini(d_sg_list_t *sgl, bool free_iovs) sgl->sg_nr = 0; } +static inline size_t __attribute__((nonnull)) +d_sgl_buf_size(d_sg_list_t *sgl) +{ + size_t size = 0; + uint32_t i; + + if (sgl->sg_iovs == NULL) + return 0; + + for (i = 0, size = 0; i < sgl->sg_nr; i++) + size += sgl->sg_iovs[i].iov_buf_len; + + return size; +} + +static inline void +d_sgl_buf_copy(d_sg_list_t *dst_sgl, d_sg_list_t *src_sgl) +{ + uint32_t i; + + D_ASSERT(dst_sgl->sg_nr >= src_sgl->sg_nr); + for (i = 0; i < src_sgl->sg_nr; i++) { + D_ASSERT(dst_sgl->sg_iovs[i].iov_buf_len >= + src_sgl->sg_iovs[i].iov_buf_len); + + memcpy(dst_sgl->sg_iovs[i].iov_buf, src_sgl->sg_iovs[i].iov_buf, + src_sgl->sg_iovs[i].iov_buf_len); + dst_sgl->sg_iovs[i].iov_len = src_sgl->sg_iovs[i].iov_len; + dst_sgl->sg_iovs[i].iov_buf_len = src_sgl->sg_iovs[i].iov_buf_len; + } +} + void d_getenv_bool(const char *env, bool *bool_val); void d_getenv_char(const char *env, char *char_val); void d_getenv_int(const char *env, unsigned int *int_val); diff --git a/src/object/obj_rpc.c b/src/object/obj_rpc.c index 0137bc48dbc..bfff5544b40 100644 --- a/src/object/obj_rpc.c +++ b/src/object/obj_rpc.c @@ -491,52 +491,6 @@ crt_proc_struct_obj_iod_array(crt_proc_t proc, crt_proc_op_t proc_op, return rc; } -static int -crt_proc_d_sg_list_t(crt_proc_t proc, crt_proc_op_t proc_op, d_sg_list_t *p) -{ - int i; - int rc; - - if (FREEING(proc_op)) { - /* NB: don't need free in crt_proc_d_iov_t() */ - D_FREE(p->sg_iovs); - return 0; - } - - rc = crt_proc_uint32_t(proc, proc_op, &p->sg_nr); - if (unlikely(rc)) - return rc; - - rc = crt_proc_uint32_t(proc, proc_op, &p->sg_nr_out); - if (unlikely(rc)) - return rc; - - if (p->sg_nr == 0) - return 0; - - switch (proc_op) { - case CRT_PROC_DECODE: - D_ALLOC_ARRAY(p->sg_iovs, p->sg_nr); - if (p->sg_iovs == NULL) - return -DER_NOMEM; - /* fall through to fill sg_iovs */ - case CRT_PROC_ENCODE: - for (i = 0; i < p->sg_nr; i++) { - rc = crt_proc_d_iov_t(proc, proc_op, &p->sg_iovs[i]); - if (unlikely(rc)) { - if (DECODING(proc_op)) - D_FREE(p->sg_iovs); - return rc; - } - } - break; - default: - return -DER_INVAL; - } - - return rc; -} - static int crt_proc_struct_daos_shard_tgt(crt_proc_t proc, crt_proc_op_t proc_op, struct daos_shard_tgt *p) diff --git a/src/pool/srv_iv.c b/src/pool/srv_iv.c index 53908a490fc..3eaa05ce48e 100644 --- a/src/pool/srv_iv.c +++ b/src/pool/srv_iv.c @@ -613,10 +613,10 @@ pool_iv_ent_get(struct ds_iv_entry *entry, void **priv) return 0; } -static int +static void pool_iv_ent_put(struct ds_iv_entry *entry, void *priv) { - return 0; + return; } static int diff --git a/src/rebuild/rebuild_iv.c b/src/rebuild/rebuild_iv.c index 15a86ceb5fc..da306be72a6 100644 --- a/src/rebuild/rebuild_iv.c +++ b/src/rebuild/rebuild_iv.c @@ -66,10 +66,10 @@ rebuild_iv_ent_get(struct ds_iv_entry *entry, void **priv) return 0; } -static int +static void rebuild_iv_ent_put(struct ds_iv_entry *entry, void *priv) { - return 0; + return; } static int diff --git a/src/tests/ftest/cart/iv_server.c b/src/tests/ftest/cart/iv_server.c index 2e6488bf7aa..43927040a9a 100644 --- a/src/tests/ftest/cart/iv_server.c +++ b/src/tests/ftest/cart/iv_server.c @@ -623,7 +623,7 @@ iv_on_get(crt_iv_namespace_t ivns, crt_iv_key_t *iv_key, return 0; } -static int +static void iv_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *user_priv) { DBG_ENTRY(); @@ -635,8 +635,6 @@ iv_on_put(crt_iv_namespace_t ivns, d_sg_list_t *iv_value, void *user_priv) dump_all_keys("ON_PUTVALUE"); DBG_EXIT(); - - return 0; } static void From d5f2e0ba13b9bd5c5a1eefe7870d9755efa87c8a Mon Sep 17 00:00:00 2001 From: Tom Nabarro Date: Thu, 7 Dec 2023 14:44:19 +0000 Subject: [PATCH 08/23] DAOS-14713 bio: Address bio coverity issues (#13443) CID 1530973 unchecked return value CID 1530972 unused value Signed-off-by: Tom Nabarro --- src/bio/bio_xstream.c | 4 ++-- src/vos/tests/bio_ut.c | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/bio/bio_xstream.c b/src/bio/bio_xstream.c index c7819a84318..6242ad6f4ef 100644 --- a/src/bio/bio_xstream.c +++ b/src/bio/bio_xstream.c @@ -958,8 +958,8 @@ init_bio_bdevs(struct bio_xs_context *ctxt) D_ASSERT(!is_server_started()); if (spdk_bdev_first() == NULL) { - D_ERROR("No SPDK bdevs found!"); - rc = -DER_NONEXIST; + D_ERROR("No SPDK bdevs found!\n"); + return -DER_NONEXIST; } for (bdev = spdk_bdev_first(); bdev != NULL; bdev = spdk_bdev_next(bdev)) { diff --git a/src/vos/tests/bio_ut.c b/src/vos/tests/bio_ut.c index 2f89d67ffd8..52c834f50a3 100644 --- a/src/vos/tests/bio_ut.c +++ b/src/vos/tests/bio_ut.c @@ -34,7 +34,9 @@ ut_init(struct bio_ut_args *args) { int rc; - daos_debug_init(DAOS_LOG_DEFAULT); + rc = daos_debug_init(DAOS_LOG_DEFAULT); + if (rc) + return rc; rc = vos_self_init(db_path, false, BIO_STANDALONE_TGT_ID); if (rc) From 64ef8395a64d0388578cc16d22b2d807759c95ab Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Thu, 7 Dec 2023 15:22:37 +0000 Subject: [PATCH 09/23] DAOS-14716 dfuse: Add a inode lock for readdir and remove assert. (#13437) Remove the uses of assert for detecting unexpected kernel behaviour. Add a lock to protect against concurrent readdir calls. Remove check entirely. Signed-off-by: Ashley Pittman --- src/client/dfuse/dfuse.h | 9 +++------ src/client/dfuse/dfuse_core.c | 7 ++++--- src/client/dfuse/ops/readdir.c | 9 ++------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h index f742977763c..ddae9ee0884 100644 --- a/src/client/dfuse/dfuse.h +++ b/src/client/dfuse/dfuse.h @@ -99,9 +99,6 @@ struct dfuse_obj_hdl { ATOMIC uint32_t doh_il_calls; - /** Number of active readdir operations */ - ATOMIC uint32_t doh_readdir_number; - ATOMIC uint64_t doh_write_count; /* Next offset we expect from readdir */ @@ -749,6 +746,9 @@ struct dfuse_inode_entry { struct dfuse_cont *ie_dfs; + /* Lock, used to protect readdir calls */ + pthread_mutex_t ie_lock; + /** Hash table of inodes * All valid inodes are kept in a hash table, using the hash table locking. */ @@ -778,9 +778,6 @@ struct dfuse_inode_entry { /* Readdir handle, if present. May be shared */ struct dfuse_readdir_hdl *ie_rd_hdl; - /** Number of active readdir operations */ - ATOMIC uint32_t ie_readdir_number; - /** file was truncated from 0 to a certain size */ bool ie_truncated; diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c index 8c1889d4ac9..662b79e6c21 100644 --- a/src/client/dfuse/dfuse_core.c +++ b/src/client/dfuse/dfuse_core.c @@ -1111,7 +1111,6 @@ dfuse_open_handle_init(struct dfuse_obj_hdl *oh, struct dfuse_inode_entry *ie) oh->doh_linear_read = true; oh->doh_linear_read_pos = 0; atomic_init(&oh->doh_il_calls, 0); - atomic_init(&oh->doh_readdir_number, 0); atomic_init(&oh->doh_write_count, 0); } @@ -1122,7 +1121,8 @@ dfuse_ie_init(struct dfuse_inode_entry *ie) atomic_init(&ie->ie_open_count, 0); atomic_init(&ie->ie_open_write_count, 0); atomic_init(&ie->ie_il_count, 0); - atomic_init(&ie->ie_readdir_number, 0); + + D_MUTEX_INIT(&ie->ie_lock, NULL); } void @@ -1136,7 +1136,6 @@ dfuse_ie_close(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry ie->ie_stat.st_ino, ref, ie->ie_name, ie->ie_parent); D_ASSERT(ref == 0); - D_ASSERT(atomic_load_relaxed(&ie->ie_readdir_number) == 0); D_ASSERT(atomic_load_relaxed(&ie->ie_il_count) == 0); D_ASSERT(atomic_load_relaxed(&ie->ie_open_count) == 0); @@ -1156,6 +1155,8 @@ dfuse_ie_close(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry d_hash_rec_decref(&dfp->dfp_cont_table, &dfc->dfs_entry); } + D_MUTEX_DESTROY(&ie->ie_lock); + D_FREE(ie); } diff --git a/src/client/dfuse/ops/readdir.c b/src/client/dfuse/ops/readdir.c index d6bca1c7c87..f24f62883b9 100644 --- a/src/client/dfuse/ops/readdir.c +++ b/src/client/dfuse/ops/readdir.c @@ -788,11 +788,7 @@ dfuse_cb_readdir(fuse_req_t req, struct dfuse_obj_hdl *oh, size_t size, off_t of char *reply_buff = NULL; int rc = EIO; - D_ASSERTF(atomic_fetch_add_relaxed(&oh->doh_readdir_number, 1) == 0, - "Multiple readdir per handle"); - - D_ASSERTF(atomic_fetch_add_relaxed(&oh->doh_ie->ie_readdir_number, 1) == 0, - "Multiple readdir per inode"); + D_MUTEX_LOCK(&oh->doh_ie->ie_lock); /* Handle the EOD case, the kernel will keep reading until it receives zero replies so * reply early in this case. @@ -820,8 +816,7 @@ dfuse_cb_readdir(fuse_req_t req, struct dfuse_obj_hdl *oh, size_t size, off_t of rc = dfuse_do_readdir(fs_handle, req, oh, reply_buff, &size, offset, plus); out: - atomic_fetch_sub_relaxed(&oh->doh_readdir_number, 1); - atomic_fetch_sub_relaxed(&oh->doh_ie->ie_readdir_number, 1); + D_MUTEX_UNLOCK(&oh->doh_ie->ie_lock); if (rc) DFUSE_REPLY_ERR_RAW(oh, req, rc); From af57db7eff4fad91ff3bcdd05a083e3af37f3f8a Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 7 Dec 2023 10:28:17 -0500 Subject: [PATCH 10/23] =?UTF-8?q?DAOS-14440=20build:=20Build=20on=20EL=208?= =?UTF-8?q?.6=E2=80=A6=20(#13381)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit But test on EL 8.8. Build and test on Leap 15.5. - update packaging support for Leap 15.5. Update MOFED version so that newer OS versions are supported. During provisioning move OS upgrade to after PR-repos are added so that PR-repos can upgrade packages. Use new repo-files mock repo names during RPM build. Use fedora:latest for mock building. Allow PR repos to replace packages from modules. Move functions to the functions file. Update RPM Test stage. - Add "system-ram-reserved 4" to engine configuration. - Auto-size SCM. - Remove --recreate-superblocks from daos start. - Collect logs and update status at the end of the stage. - Update default provider. - Define MODULEPATH for SUSE. - Remove creation and moutning of tmpfs on /mnt/daos. - Format the storage if needed. - Add a 300s timeout to the server start and format. - Display server stdout if startup fails. - Use export POOL_NVME_SIZE=0. - Archive DAOS logs. Update packaging/. Signed-off-by: Brian J. Murrell --- Jenkinsfile | 172 +++++++++++------- ci/parse_ci_envs.sh | 11 +- .../post_provision_config_common.sh | 2 +- .../post_provision_config_common_functions.sh | 137 +++++++++++--- .../post_provision_config_nodes.sh | 71 -------- ci/rpm/test_daos_node.sh | 107 +++++++---- debian/changelog | 7 + debian/control | 3 +- ftest.sh | 4 +- src/rdb/raft | 2 +- src/tests/ftest/config_file_gen.py | 2 +- .../ftest/daos_test/suite.single_host.yaml | 159 ++++++++++++++++ src/tests/ftest/util/server_utils_params.py | 2 +- utils/rpms/daos.spec | 8 +- utils/rpms/packaging/Dockerfile.centos.7 | 30 ++- utils/rpms/packaging/Dockerfile.mockbuild | 14 +- utils/rpms/packaging/Makefile_distro_vars.mk | 8 + utils/rpms/packaging/Makefile_packaging.mk | 71 +++++--- utils/rpms/packaging/rpm_chrootbuild | 59 +++--- utils/scripts/helpers/repo-helper-leap15.sh | 2 +- 20 files changed, 595 insertions(+), 276 deletions(-) create mode 100644 src/tests/ftest/daos_test/suite.single_host.yaml diff --git a/Jenkinsfile b/Jenkinsfile index 28f8d1f56d3..4536019bc48 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -155,6 +155,21 @@ void fixup_rpmlintrc() { writeFile(file: 'utils/rpms/daos.rpmlintrc', text: content) } +String vm9_label(String distro) { + return cachedCommitPragma(pragma: distro + '-VM9-label', + def_val: cachedCommitPragma(pragma: 'VM9-label', + def_val: params.FUNCTIONAL_VM_LABEL)) +} + +void rpm_test_post(String stage_name, String node) { + sh label: 'Fetch and stage artifacts', + script: 'hostname; ssh -i ci_key jenkins@' + node + ' ls -ltar /tmp; mkdir -p "' + env.STAGE_NAME + '/" && ' + + 'scp -i ci_key jenkins@' + node + ':/tmp/{{suite_dmg,daos_{server_helper,{control,agent}}}.log,daos_server.log.*} "' + + env.STAGE_NAME + '/"' + archiveArtifacts artifacts: env.STAGE_NAME + '/**' + job_status_update() +} + pipeline { agent { label 'lightweight' } @@ -516,7 +531,7 @@ pipeline { filename 'packaging/Dockerfile.mockbuild' dir 'utils/rpms' label 'docker_runner' - additionalBuildArgs dockerBuildArgs() + additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=38' args '--cap-add=SYS_ADMIN' } } @@ -543,7 +558,7 @@ pipeline { } } } - stage('Build RPM on Leap 15.4') { + stage('Build RPM on Leap 15.5') { when { beforeAgent true expression { !skipStage() } @@ -553,7 +568,7 @@ pipeline { filename 'packaging/Dockerfile.mockbuild' dir 'utils/rpms' label 'docker_runner' - additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=37' + additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=38' args '--cap-add=SYS_ADMIN' } } @@ -700,7 +715,7 @@ pipeline { } } } - stage('Build on Leap 15.4 with Intel-C and TARGET_PREFIX') { + stage('Build on Leap 15.5 with Intel-C and TARGET_PREFIX') { when { beforeAgent true expression { !skipStage() } @@ -744,7 +759,7 @@ pipeline { expression { !skipStage() } } parallel { - stage('Unit Test on EL 8') { + stage('Unit Test on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -766,7 +781,7 @@ pipeline { } } } - stage('Unit Test bdev on EL 8') { + stage('Unit Test bdev on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -788,7 +803,7 @@ pipeline { } } } - stage('NLT on EL 8') { + stage('NLT on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -825,7 +840,7 @@ pipeline { } } } - stage('Unit Test Bullseye on EL 8') { + stage('Unit Test Bullseye on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -852,8 +867,8 @@ pipeline { job_status_update() } } - } // stage('Unit test Bullseye on EL 8') - stage('Unit Test with memcheck on EL 8') { + } // stage('Unit test Bullseye on EL 8.8') + stage('Unit Test with memcheck on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -877,8 +892,8 @@ pipeline { job_status_update() } } - } // stage('Unit Test with memcheck on EL 8') - stage('Unit Test bdev with memcheck on EL 8') { + } // stage('Unit Test with memcheck on EL 8.8') + stage('Unit Test bdev with memcheck on EL 8.8') { when { beforeAgent true expression { !skipStage() } @@ -911,7 +926,7 @@ pipeline { expression { !skipStage() } } parallel { - stage('Functional on EL 8 with Valgrind') { + stage('Functional on EL 8.8 with Valgrind') { when { beforeAgent true expression { !skipStage() } @@ -932,14 +947,14 @@ pipeline { job_status_update() } } - } // stage('Functional on EL 8 with Valgrind') - stage('Functional on EL 8') { + } // stage('Functional on EL 8.8 with Valgrind') + stage('Functional on EL 8.8') { when { beforeAgent true expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'EL8-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('EL8') } steps { job_step_update( @@ -954,14 +969,14 @@ pipeline { job_status_update() } } - } // stage('Functional on EL 8') - stage('Functional on Leap 15.4') { + } // stage('Functional on EL 8.8') + stage('Functional on Leap 15.5') { when { beforeAgent true expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'Leap15-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('Leap15') } steps { job_step_update( @@ -976,14 +991,14 @@ pipeline { job_status_update() } } // post - } // stage('Functional on Leap 15.4') + } // stage('Functional on Leap 15.5') stage('Functional on Ubuntu 20.04') { when { beforeAgent true expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'Ubuntu-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('Ubuntu') } steps { job_step_update( @@ -999,44 +1014,7 @@ pipeline { } } // post } // stage('Functional on Ubuntu 20.04') - stage('Scan EL 8 RPMs') { - when { - beforeAgent true - expression { !skipStage() } - } - agent { - dockerfile { - filename 'ci/docker/Dockerfile.maldet.el.8' - label 'docker_runner' - additionalBuildArgs dockerBuildArgs() + - " -t ${sanitized_JOB_NAME}-el8 " + - ' --build-arg REPOS="' + prRepos() + '"' + - ' --build-arg BUILD_URL="' + env.BUILD_URL + '"' - } - } - steps { - job_step_update( - runTest(script: 'export DAOS_PKG_VERSION=' + - daosPackagesVersion(next_version) + '\n' + - 'utils/scripts/helpers/scan_daos_maldet.sh', - junit_files: 'maldetect_el8.xml', - failure_artifacts: env.STAGE_NAME, - ignore_failure: true, - description: env.STAGE_NAME, - context: 'test/' + env.STAGE_NAME)) - } - post { - always { - junit 'maldetect_el8.xml' - archiveArtifacts artifacts: 'maldetect_el8.xml' - job_status_update() - // Force a job failure if anything was found - sh label: 'Check if anything was found.', - script: '! grep "/dev/null | grep ..\*); then + DNF_REPO_ARGS+=" --enablerepo=$repo" + else + local repo_url="${REPOSITORY_URL}${add_repo}" + local repo_name + repo_name=$(url_to_repo "$repo_url") + if ! dnf -y repolist | grep "$repo_name"; then + dnf -y config-manager --add-repo="${repo_url}" >&2 + if ! $gpg_check; then + disable_gpg_check "$add_repo" >&2 + fi + fi + DNF_REPO_ARGS+=" --enablerepo=$repo_name" + fi +} + +add_group_repo() { + local match="$1" + + add_repo "$match" "$DAOS_STACK_GROUP_REPO" + group_repo_post +} + +add_local_repo() { + add_repo 'argobots' "$DAOS_STACK_LOCAL_REPO" false +} + +disable_gpg_check() { + local url="$1" + + repo="$(url_to_repo "$url")" + # bug in EL7 DNF: this needs to be enabled before it can be disabled + dnf -y config-manager --save --setopt="$repo".gpgcheck=1 + dnf -y config-manager --save --setopt="$repo".gpgcheck=0 + # but even that seems to be not enough, so just brute-force it + if [ -d /etc/yum.repos.d ] && + ! grep gpgcheck /etc/yum.repos.d/"$repo".repo; then + echo "gpgcheck=0" >> /etc/yum.repos.d/"$repo".repo + fi +} + +dump_repos() { + for file in "$REPOS_DIR"/*.repo; do + echo "---- $file ----" + cat "$file" + done +} retry_dnf() { local monitor_threshold="$1" shift @@ -259,7 +330,7 @@ post_provision_config_nodes() { if $CONFIG_POWER_ONLY; then rm -f "$REPOS_DIR"/*.hpdd.intel.com_job_daos-stack_job_*_job_*.repo - time dnf -y erase fio fuse ior-hpc mpich-autoload \ + time dnf -y erase fio fuse ior-hpc mpich-autoload \ ompi argobots cart daos daos-client dpdk \ fuse-libs libisa-l libpmemobj mercury mpich \ pmix protobuf-c spdk libfabric libpmem \ @@ -267,7 +338,41 @@ post_provision_config_nodes() { slurm-example-configs slurmctld slurm-slurmmd fi - lsb_release -a + cat /etc/os-release + + if lspci | grep "ConnectX-6" && ! grep MOFED_VERSION /etc/do-release; then + # Remove OPA and install MOFED + install_mofed + fi + + local repos_added=() + local repo + local inst_repos=() + # shellcheck disable=SC2153 + read -ra inst_repos <<< "$INST_REPOS" + for repo in "${inst_repos[@]}"; do + branch="master" + build_number="lastSuccessfulBuild" + if [[ $repo = *@* ]]; then + branch="${repo#*@}" + repo="${repo%@*}" + if [[ \ ${repos_added[*]+${repos_added[*]}}\ = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" + fi + fi + local repo_url="${ARTIFACTS_URL:-${JENKINS_URL}job/}"daos-stack/job/"$repo"/job/"${branch//\//%252F}"/"$build_number"/artifact/artifacts/$DISTRO_NAME/ + dnf -y config-manager --add-repo="${repo_url}" + # Disable module filtering + repo=$(url_to_repo "$repo_url") + dnf config-manager --save --setopt "$repo.module_hotfixes=true" "$repo" + disable_gpg_check "$repo_url" + done # start with everything fully up-to-date # all subsequent package installs beyond this will install the newest packages @@ -283,30 +388,8 @@ post_provision_config_nodes() { return 1 fi - if lspci | grep "ConnectX-6" && ! grep MOFED_VERSION /etc/do-release; then - # Remove OPA and install MOFED - install_mofed - fi - - if [ -n "$INST_REPOS" ]; then - local repo - for repo in $INST_REPOS; do - branch="master" - build_number="lastSuccessfulBuild" - if [[ $repo = *@* ]]; then - branch="${repo#*@}" - repo="${repo%@*}" - if [[ $branch = *:* ]]; then - build_number="${branch#*:}" - branch="${branch%:*}" - fi - fi - local repo_url="${JENKINS_URL}"job/daos-stack/job/"${repo}"/job/"${branch//\//%252F}"/"${build_number}"/artifact/artifacts/$DISTRO_NAME/ - dnf -y config-manager --add-repo="${repo_url}" - disable_gpg_check "$repo_url" - done - fi - inst_rpms=() + local inst_rpms=() + # shellcheck disable=SC2153 if [ -n "$INST_RPMS" ]; then eval "inst_rpms=($INST_RPMS)" time dnf -y erase "${inst_rpms[@]}" @@ -358,7 +441,7 @@ setenv MPI_HOME /usr/mpi/gcc/openmpi-$version EOF printf 'MOFED_VERSION=%s\n' "$MLNX_VER_NUM" >> /etc/do-release - fi + fi distro_custom diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index 1b205946f6b..8d70a0c1a51 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -2,77 +2,6 @@ set -eux -# functions common to more than one distro specific provisioning -url_to_repo() { - local url="$1" - - local repo=${url#*://} - repo="${repo//%/}" - repo="${repo//\//_}" - - echo "$repo" -} - -add_repo() { - local match="$1" - local add_repo="$2" - local gpg_check="${3:-true}" - - if [ -z "$match" ]; then - # we cannot try to add a repo that has no match - return - fi - - local repo - # see if a package we know is in the repo is present - if repo=$(dnf -y repoquery --qf "%{repoid}" "$1" 2>/dev/null | grep ..\*); then - DNF_REPO_ARGS+=" --enablerepo=$repo" - else - local repo_url="${REPOSITORY_URL}${add_repo}" - local repo_name - repo_name=$(url_to_repo "$repo_url") - if ! dnf -y repolist | grep "$repo_name"; then - dnf -y config-manager --add-repo="${repo_url}" >&2 - if ! $gpg_check; then - disable_gpg_check "$add_repo" >&2 - fi - fi - DNF_REPO_ARGS+=" --enablerepo=$repo_name" - fi -} - -add_group_repo() { - local match="$1" - - add_repo "$match" "$DAOS_STACK_GROUP_REPO" - group_repo_post -} - -add_local_repo() { - add_repo 'argobots' "$DAOS_STACK_LOCAL_REPO" false -} - -disable_gpg_check() { - local url="$1" - - repo="$(url_to_repo "$url")" - # bug in EL7 DNF: this needs to be enabled before it can be disabled - dnf -y config-manager --save --setopt="$repo".gpgcheck=1 - dnf -y config-manager --save --setopt="$repo".gpgcheck=0 - # but even that seems to be not enough, so just brute-force it - if [ -d /etc/yum.repos.d ] && - ! grep gpgcheck /etc/yum.repos.d/"$repo".repo; then - echo "gpgcheck=0" >> /etc/yum.repos.d/"$repo".repo - fi -} - -dump_repos() { - for file in "$REPOS_DIR"/*.repo; do - echo "---- $file ----" - cat "$file" - done -} - env > /root/last_run-env.txt if ! grep ":$MY_UID:" /etc/group; then groupadd -g "$MY_UID" jenkins diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index c7b27473593..3e92bb7b703 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -1,25 +1,31 @@ #!/bin/bash +. /etc/os-release + YUM=dnf -id="$(lsb_release -si)" -if [ "$id" = "CentOS" ] || - [ "$id" = "AlmaLinux" ] || - [ "$id" = "Rocky" ] || - [ "$id" = "RedHatEnterpriseServer" ]; then - if [[ $(lsb_release -sr) = 8* ]]; then - OPENMPI_RPM=openmpi - OPENMPI=mpi/openmpi-x86_64 - else +case "$ID_LIKE" in + *rhel*) + if [[ $VERSION_ID = [89].* ]]; then + OPENMPI_RPM=openmpi + OPENMPI=mpi/openmpi-x86_64 + else + OPENMPI_RPM=openmpi3 + OPENMPI=mpi/openmpi3-x86_64 + fi + ;; + *suse*) OPENMPI_RPM=openmpi3 - OPENMPI=mpi/openmpi3-x86_64 - fi -elif [ "$(lsb_release -si)" = "openSUSE" ]; then - OPENMPI_RPM=openmpi3 - OPENMPI=gnu-openmpi + OPENMPI=gnu-openmpi + export MODULEPATH=/usr/share/modules + ;; +esac + +if [ -n "$DAOS_PKG_VERSION" ]; then + DAOS_PKG_VERSION="-${DAOS_PKG_VERSION}" fi set -uex -sudo $YUM -y install daos-client-"${DAOS_PKG_VERSION}" +sudo $YUM -y install daos-client"$DAOS_PKG_VERSION" if rpm -q daos-server; then echo "daos-server RPM should not be installed as a dependency of daos-client" exit 1 @@ -29,9 +35,9 @@ if ! sudo $YUM -y history undo last; then $YUM history exit 1 fi -sudo $YUM -y erase $OPENMPI_RPM -sudo $YUM -y install daos-client-tests-"${DAOS_PKG_VERSION}" -if rpm -q $OPENMPI_RPM; then +sudo $YUM -y erase "$OPENMPI_RPM" +sudo $YUM -y install daos-client-tests"$DAOS_PKG_VERSION" +if rpm -q "$OPENMPI_RPM"; then echo "$OPENMPI_RPM RPM should not be installed as a dependency of daos-client-tests" exit 1 fi @@ -44,8 +50,8 @@ if ! sudo $YUM -y history undo last; then $YUM history exit 1 fi -sudo $YUM -y install daos-server-tests-"${DAOS_PKG_VERSION}" -if rpm -q $OPENMPI_RPM; then +sudo $YUM -y install daos-server-tests"$DAOS_PKG_VERSION" +if rpm -q "$OPENMPI_RPM"; then echo "$OPENMPI_RPM RPM should not be installed as a dependency of daos-server-tests" exit 1 fi @@ -58,7 +64,7 @@ if ! sudo $YUM -y history undo last; then $YUM history exit 1 fi -sudo $YUM -y install --exclude ompi daos-client-tests-openmpi-"${DAOS_PKG_VERSION}" +sudo $YUM -y install --exclude ompi daos-client-tests-openmpi"$DAOS_PKG_VERSION" if ! rpm -q daos-client; then echo "daos-client RPM should be installed as a dependency of daos-client-tests-openmpi" exit 1 @@ -76,13 +82,13 @@ if ! sudo $YUM -y history undo last; then $YUM history exit 1 fi -sudo $YUM -y install daos-server-"${DAOS_PKG_VERSION}" +sudo $YUM -y install daos-server"$DAOS_PKG_VERSION" if rpm -q daos-client; then echo "daos-client RPM should not be installed as a dependency of daos-server" exit 1 fi -sudo $YUM -y install --exclude ompi daos-client-tests-openmpi-"${DAOS_PKG_VERSION}" +sudo $YUM -y install --exclude ompi daos-client-tests-openmpi"$DAOS_PKG_VERSION" me=$(whoami) for dir in server agent; do @@ -93,42 +99,63 @@ done sudo mkdir /tmp/daos_sockets sudo chmod 0755 /tmp/daos_sockets sudo chown "$me:$me" /tmp/daos_sockets -sudo mkdir -p /mnt/daos -sudo mount -t tmpfs -o size=16777216k tmpfs /mnt/daos FTEST=/usr/lib/daos/TESTING/ftest sudo PYTHONPATH="$FTEST/util" \ $FTEST/config_file_gen.py -n "$HOSTNAME" \ -a /etc/daos/daos_agent.yml \ -s /etc/daos/daos_server.yml +sudo bash -c 'echo "system_ram_reserved: 4" >> /etc/daos/daos_server.yml' sudo PYTHONPATH="$FTEST/util" \ $FTEST/config_file_gen.py -n "$HOSTNAME" \ - -d /etc/daos/daos.yml + -d /etc/daos/daos_control.yml cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml -cat /etc/daos/daos.yml -if ! module load $OPENMPI; then +cat /etc/daos/daos_control.yml +if ! module load "$OPENMPI"; then echo "Unable to load OpenMPI module: $OPENMPI" module avail module list exit 1 fi -coproc SERVER { exec daos_server --debug start -t 1 --recreate-superblocks; } 2>&1 +export POOL_NVME_SIZE=0 + +coproc SERVER { exec daos_server --debug start -t 1; } 2>&1 trap 'set -x; kill -INT $SERVER_PID' EXIT line="" -while [[ "$line" != *started\ on\ rank\ 0* ]]; do - if ! read -r -t 60 line <&"${SERVER[0]}"; then - rc=${PIPESTATUS[0]} - if [ "$rc" = "142" ]; then - echo "Timed out waiting for output from the server" - else - echo "Error reading the output from the server: $rc" - fi - exit "$rc" - fi - echo "Server stdout: $line" +stdout=() +deadline=$((SECONDS+300)) +while [[ $line != *started\ on\ rank\ 0* ]] && [ "$SECONDS" -lt "$deadline" ]; do + if ! read -r -t 60 line <&"${SERVER[0]}"; then + rc=${PIPESTATUS[0]} + if [ "$rc" = "142" ]; then + echo "Timed out waiting for output from the server" + else + echo "Error reading the output from the server: $rc" + fi + echo "Server output:" + export IFS=$'\n' + echo "${stdout[*]}" + exit "$rc" + fi + echo "Server stdout: $line" + stdout+=("$line") + if [[ $line == SCM\ format\ required\ on\ instance\ * ]]; then + dmg storage format -l "$HOSTNAME" --force + if ! dmg system query -v; then + sleep 5 + dmg system query -v || true + fi + fi done +if [ "$SECONDS" -ge "$deadline" ]; then + echo "Timed out waiting for server to start" + echo "Server output:" + export IFS=$'\n' + echo "${stdout[*]}" + exit 1 +fi echo "Server started!" coproc AGENT { exec daos_agent --debug; } 2>&1 trap 'set -x; kill -INT $AGENT_PID $SERVER_PID' EXIT diff --git a/debian/changelog b/debian/changelog index 704a3bf07de..7018a806823 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +daos (2.4.0-5) unstable; urgency=medium + [ Brian J. Murrell ] + * Update for EL 8.8 and Leap 15.5 + * Update raft to 0.10.1-2.411.gefa15f4 + + -- Brian J. Murrell Wed, 06 Dec 2023 08:54:56 -0500 + daos (2.4.0-4) unstable; urgency=medium [ Jerome Soumagne ] * Bump mercury min version to 2.3.1 diff --git a/debian/control b/debian/control index 6afa7bf47af..c717114254d 100644 --- a/debian/control +++ b/debian/control @@ -29,7 +29,7 @@ Build-Depends: debhelper (>= 10), libboost-dev, libspdk-dev (>= 22.01.2), libipmctl-dev, - libraft-dev (= 0.10.1-1408.g9524cdb), + libraft-dev (= 0.10.1-1411.gefa15f4), python3-tabulate, liblz4-dev, libcapstone-dev @@ -201,4 +201,3 @@ Description: The Distributed Asynchronous Object Storage (DAOS) is an open-sourc to optimize performance and cost. . This package contains the DAOS administrative tools (e.g. dmg). - diff --git a/ftest.sh b/ftest.sh index 4963cec765a..94661790c5c 100755 --- a/ftest.sh +++ b/ftest.sh @@ -29,9 +29,7 @@ LAUNCH_OPT_ARGS="${3:-}" # Add the missing '--nvme' argument identifier for backwards compatibility with # the 'auto:Optane' optional argument specified without the identifier. -if [[ "${LAUNCH_OPT_ARGS}" == "auto:-3DNAND" ]]; then - LAUNCH_OPT_ARGS="--nvme=${LAUNCH_OPT_ARGS}" -fi +LAUNCH_OPT_ARGS="$(echo "$LAUNCH_OPT_ARGS" | sed -e 's/^/ /' -e 's/ \(auto:-3DNAND\)/--nvme=\1/' -e 's/^ *//')" # For nodes that are only rebooted between CI nodes left over mounts # need to be cleaned up. diff --git a/src/rdb/raft b/src/rdb/raft index c354cd7fb1e..efa15f46360 160000 --- a/src/rdb/raft +++ b/src/rdb/raft @@ -1 +1 @@ -Subproject commit c354cd7fb1e1dbf3fbc9b3d24a6b05d0c5c9d5af +Subproject commit efa15f46360078ff427562c53d23ed6f0e4a06ac diff --git a/src/tests/ftest/config_file_gen.py b/src/tests/ftest/config_file_gen.py index d51e029d48f..58f97f3b902 100755 --- a/src/tests/ftest/config_file_gen.py +++ b/src/tests/ftest/config_file_gen.py @@ -50,7 +50,7 @@ def generate_server_config(args): config = DaosServerYamlParameters(args.server_file, common_cfg) config.engine_params[0].storage.storage_tiers[0].storage_class.value = "ram" config.engine_params[0].storage.storage_tiers[0].scm_mount.value = "/mnt/daos" - config.engine_params[0].storage.storage_tiers[0].scm_size.value = 16 + config.engine_params[0].storage.storage_tiers[0].scm_size.value = 0 # Update the configuration file access points config.other_params.access_points.value = args.node_list.split(",") return create_config(args, config) diff --git a/src/tests/ftest/daos_test/suite.single_host.yaml b/src/tests/ftest/daos_test/suite.single_host.yaml new file mode 100644 index 00000000000..e4fba4a3e12 --- /dev/null +++ b/src/tests/ftest/daos_test/suite.single_host.yaml @@ -0,0 +1,159 @@ +# change host names to your reserved nodes, the +# required quantity is indicated by the placeholders +hosts: + test_servers: 1 +# Note that subtests below can set their own timeout so this +# should be a general average of all tests not including outliers +# (I'm looking at you "rebuild tests") +timeout: 600 +timeouts: + test_daos_degraded_mode: 450 + test_daos_management: 110 + test_daos_pool: 180 + test_daos_container: 450 + test_daos_epoch: 125 + test_daos_verify_consistency: 105 + test_daos_io: 290 + test_daos_ec_io: 450 + test_daos_ec_obj: 600 + test_daos_object_array: 105 + test_daos_array: 106 + test_daos_kv: 105 + test_daos_capability: 104 + test_daos_epoch_recovery: 104 + test_daos_md_replication: 104 + test_daos_rebuild_simple: 1800 + test_daos_drain_simple: 3600 + test_daos_extend_simple: 3600 + test_daos_oid_allocator: 640 + test_daos_checksum: 500 + test_daos_rebuild_ec: 6400 + test_daos_aggregate_ec: 200 + test_daos_degraded_ec: 1900 + test_daos_dedup: 220 + test_daos_upgrade: 300 + test_daos_pipeline: 60 +pool: + size: 16G +server_config: + name: daos_server + engines_per_host: 1 + engines: + 0: + storage: auto + transport_config: + allow_insecure: true + system_ram_reserved: 64 +agent_config: + transport_config: + allow_insecure: true +dmg: + transport_config: + allow_insecure: true +daos_tests: + num_clients: + test_daos_degraded_mode: 1 + test_daos_management: 1 + test_daos_pool: 2 + test_daos_container: 1 + test_daos_epoch: 1 + test_daos_single_rdg_tx: 1 + test_daos_distributed_tx: 1 + test_daos_verify_consistency: 1 + test_daos_io: 1 + test_daos_ec_io: 1 + test_daos_ec_obj: 1 + test_daos_object_array: 1 + test_daos_array: 1 + test_daos_kv: 1 + test_daos_capability: 2 + test_daos_epoch_recovery: 1 + test_daos_md_replication: 2 + test_daos_rebuild_simple: 1 + test_daos_drain_simple: 1 + test_daos_extend_simple: 1 + test_daos_oid_allocator: 1 + test_daos_checksum: 1 + test_daos_rebuild_ec: 1 + test_daos_aggregate_ec: 1 + test_daos_degraded_ec: 1 + test_daos_dedup: 1 + test_daos_upgrade: 1 + test_daos_pipeline: 1 + test_name: + test_daos_degraded_mode: DAOS_Degraded_Mode + test_daos_management: DAOS_Management + test_daos_pool: DAOS_Pool + test_daos_container: DAOS_Container + test_daos_epoch: DAOS_Epoch + test_daos_single_rdg_tx: DAOS_Single_RDG_TX + test_daos_distributed_tx: DAOS_Distributed_TX + test_daos_verify_consistency: DAOS_Verify_Consistency + test_daos_io: DAOS_IO + test_daos_ec_io: DAOS_IO_EC_4P2G1 + test_daos_ec_obj: DAOS_EC + test_daos_object_array: DAOS_Object_Array + test_daos_array: DAOS_Array + test_daos_kv: DAOS_KV + test_daos_capability: DAOS_Capability + test_daos_epoch_recovery: DAOS_Epoch_Recovery + test_daos_md_replication: DAOS_MD_Replication + test_daos_rebuild_simple: DAOS_Rebuild_Simple + test_daos_drain_simple: DAOS_Drain_Simple + test_daos_oid_allocator: DAOS_OID_Allocator + test_daos_checksum: DAOS_Checksum + test_daos_rebuild_ec: DAOS_Rebuild_EC + test_daos_aggregate_ec: DAOS_Aggregate_EC + test_daos_degraded_ec: DAOS_Degraded_EC + test_daos_dedup: DAOS_Dedup + test_daos_extend_simple: DAOS_Extend_Simple + test_daos_upgrade: DAOS_Upgrade + test_daos_pipeline: DAOS_Pipeline + daos_test: + test_daos_degraded_mode: d + test_daos_management: m + test_daos_pool: p + test_daos_container: c + test_daos_epoch: e + test_daos_single_rdg_tx: t + test_daos_distributed_tx: T + test_daos_verify_consistency: V + test_daos_io: i + test_daos_ec_io: i + test_daos_ec_obj: I + test_daos_object_array: A + test_daos_array: D + test_daos_kv: K + test_daos_capability: C + test_daos_epoch_recovery: o + test_daos_md_replication: R + test_daos_rebuild_simple: v + test_daos_drain_simple: b + test_daos_extend_simple: B + test_daos_oid_allocator: O + test_daos_checksum: z + test_daos_rebuild_ec: S + test_daos_aggregate_ec: Z + test_daos_degraded_ec: X + test_daos_dedup: U + test_daos_upgrade: G + test_daos_pipeline: P + args: + test_daos_ec_io: -l"EC_4P2G1" + test_daos_rebuild_ec: -s5 + test_daos_md_replication: -s5 + scalable_endpoint: + test_daos_degraded_mode: true + stopped_ranks: + test_daos_degraded_mode: [5, 6, 7] + test_daos_oid_allocator: [6, 7] + pools_created: + test_daos_management: 4 + test_daos_pool: 9 + test_daos_container: 17 + test_daos_distributed_tx: 5 + test_daos_rebuild_simple: 21 + test_daos_drain_simple: 8 + test_daos_extend_simple: 5 + test_daos_rebuild_ec: 43 + test_daos_degraded_ec: 29 diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index d22b9cc5cb3..a79c750b54d 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -107,7 +107,7 @@ def __init__(self, filename, common_yaml): # is set for the running process. If group look up fails or user # is not member, use uid return from user lookup. # - default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+sockets") + default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm") # All log files should be placed in the same directory on each host to # enable easy log file archiving by launch.py diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 960ab7868ea..d73dfffeb17 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -15,7 +15,7 @@ Name: daos Version: 2.4.0 -Release: 4%{?relval}%{?dist} +Release: 5%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -72,7 +72,7 @@ BuildRequires: libisa-l_crypto-devel BuildRequires: libisal-devel BuildRequires: libisal_crypto-devel %endif -BuildRequires: daos-raft-devel = 0.10.1-1.408.g9524cdb%{?dist} +BuildRequires: daos-raft-devel = 0.10.1-2.411.gefa15f4%{?dist} BuildRequires: openssl-devel BuildRequires: libevent-devel BuildRequires: libyaml-devel @@ -555,6 +555,10 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Wed Dec 06 2023 Brian J. Murrell 2.4.0-3 +- Update for EL 8.8 and Leap 15.5 +- Update raft to 0.10.1-2.411.gefa15f4 + * Tue Nov 28 2023 Jerome Soumagne 2.4.0-4 - Bump mercury min version to 2.3.1 diff --git a/utils/rpms/packaging/Dockerfile.centos.7 b/utils/rpms/packaging/Dockerfile.centos.7 index cdfb7f69784..189ea1e4c61 100644 --- a/utils/rpms/packaging/Dockerfile.centos.7 +++ b/utils/rpms/packaging/Dockerfile.centos.7 @@ -5,9 +5,31 @@ # # Pull base image -FROM centos:7 +FROM centos:centos7 LABEL maintainer="daos@daos.groups.io" +# Use local repo server if present +ARG REPO_FILE_URL +RUN set -e; \ + if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/yum.repos.d/ && \ + curl -k -f -o daos_ci-centos7-artifactory.repo.tmp \ + "$REPO_FILE_URL"daos_ci-centos7-artifactory.repo && \ + for file in *.repo; do \ + true > $file; \ + done; \ + mv daos_ci-centos7-artifactory.repo{.tmp,}; \ + fi; \ + yum -y install dnf; \ + yum clean all; \ + dnf --disablerepo \*epel\* -y install epel-release \ + dnf-plugins-core; \ + if [ -n "$REPO_FILE_URL" ]; then \ + dnf -y --quiet config-manager --disable epel; \ + fi; \ + dnf -y update epel-release; \ + dnf -y clean all + # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -15,9 +37,9 @@ ARG UID=1000 #Nothing to do for CentOS # Install basic tools -RUN yum install -y epel-release -RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ - git python-srpm-macros dnf +RUN dnf install -y epel-release +RUN dnf install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ + git python-srpm-macros dnf && dnf -y clean all # Add build user (to keep rpmbuild happy) ENV USER build diff --git a/utils/rpms/packaging/Dockerfile.mockbuild b/utils/rpms/packaging/Dockerfile.mockbuild index 533c7c5fda5..5c6475db1f5 100644 --- a/utils/rpms/packaging/Dockerfile.mockbuild +++ b/utils/rpms/packaging/Dockerfile.mockbuild @@ -5,7 +5,7 @@ # # Pull base image -ARG FVERSION=38 # this should be changed to 'latest' and some point soon +ARG FVERSION=latest FROM fedora:$FVERSION # Needed for later use of FVERSION ARG FVERSION @@ -15,16 +15,19 @@ LABEL maintainer="daos@daos.groups.io" ARG REPO_FILE_URL RUN if [ -n "$REPO_FILE_URL" ]; then \ cd /etc/yum.repos.d/ && \ - curl -f -o daos_ci-fedora-artifactory.repo.tmp \ + curl -k -f -o daos_ci-fedora-artifactory.repo.tmp \ "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ - rm -f *.repo && \ + for file in *.repo; do \ + true > $file; \ + done; \ mv daos_ci-fedora-artifactory.repo{.tmp,}; \ fi # Install basic tools RUN dnf -y install mock make \ rpm-build createrepo rpmlint redhat-lsb-core git \ - python-srpm-macros rpmdevtools + python-srpm-macros rpmdevtools && \ + dnf -y clean all # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -45,7 +48,8 @@ RUN dnf -y upgrade && \ # https://github.com/rpm-software-management/rpmlint/pull/795 in it # But make sure to patch after dnf upgrade so that an upgraded rpmlint # RPM doesn't wipe out our patch -COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch . +ARG PACKAGINGDIR=packaging +COPY ${PACKAGINGDIR}/rpmlint--ignore-unused-rpmlintrc.patch . RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then \ if ! patch -p1; then \ diff --git a/utils/rpms/packaging/Makefile_distro_vars.mk b/utils/rpms/packaging/Makefile_distro_vars.mk index e9e0784668a..85cd97dab71 100644 --- a/utils/rpms/packaging/Makefile_distro_vars.mk +++ b/utils/rpms/packaging/Makefile_distro_vars.mk @@ -71,6 +71,14 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 15.4 SED_EXPR := 1p endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.5-x86_64) +VERSION_ID := 15.5 +DISTRO_ID := sl15.5 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.5 +SED_EXPR := 1p +endif endif ifeq ($(ID),centos) ID = el diff --git a/utils/rpms/packaging/Makefile_packaging.mk b/utils/rpms/packaging/Makefile_packaging.mk index 7adda97cd9c..d045f42c0e4 100644 --- a/utils/rpms/packaging/Makefile_packaging.mk +++ b/utils/rpms/packaging/Makefile_packaging.mk @@ -53,6 +53,7 @@ RPM_BUILD_OPTIONS := $(BUILD_DEFINES) GIT_DIFF_EXCLUDES := $(PATCH_EXCLUDE_FILES:%=':!%') endif +FVERSION ?= latest COMMON_RPM_ARGS := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) SPEC := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi) VERSION = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION) @@ -64,10 +65,16 @@ RPMS = $(eval RPMS := $(addsuffix .rpm,$(addprefix _topdir/RPMS/x86 DEB_TOP := _topdir/BUILD DEB_BUILD := $(DEB_TOP)/$(NAME)-$(VERSION) DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(VERSION) -SOURCE ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*: *//'))$(SOURCE) -PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) -OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) -SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES)) +REAL_SOURCE ?= $(eval REAL_SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/Source.*: *//'))$(REAL_SOURCE) +ifeq ($(ID_LIKE),debian) +ifneq ($(DEB_SOURCE),) +SOURCE ?= $(DEB_SOURCE) +endif +endif +SOURCE ?= $(REAL_SOURCE) +PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) +OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e '/^Patch.*:/d' -e 's/Source.*: *//' -e 's/.*\///' -e p))$(OTHER_SOURCES) +SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE) $(OTHER_SOURCES)) $(PATCHES)) ifeq ($(ID_LIKE),debian) DEBS := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control)) DEB_PREV_RELEASE := $(shell cd $(TOPDIR) && dpkg-parsechangelog -S version) @@ -154,9 +161,9 @@ ifeq ($(DL_NAME),) DL_NAME = $(NAME) endif -$(notdir $(SOURCE)): $(SPEC) $(CALLING_MAKEFILE) +$(notdir $(SOURCE) $(OTHER_SOURCES) $(REAL_SOURCE)): $(SPEC) $(CALLING_MAKEFILE) # TODO: need to clean up old ones - $(SPECTOOL) -g $(SPEC) + $(SPECTOOL) $(COMMON_RPM_ARGS) -g $(SPEC) $(DEB_TOP)/%: % | $(DEB_TOP)/ @@ -181,27 +188,29 @@ $(DEB_TOP)/.patched: $(PATCHES) check-env deb_detar | \ $(DEB_BUILD)/debian/ mkdir -p ${DEB_BUILD}/debian/patches mkdir -p $(DEB_TOP)/patches - for f in $(PATCHES); do \ - rm -f $(DEB_TOP)/patches/*; \ - if git mailsplit -o$(DEB_TOP)/patches < "$$f" ;then \ - fn=$$(basename "$$f"); \ - for f1 in $(DEB_TOP)/patches/*;do \ - [ -e "$$f1" ] || continue; \ - f1n=$$(basename "$$f1"); \ - echo "$${fn}_$${f1n}" >> $(DEB_BUILD)/debian/patches/series ; \ - mv "$$f1" $(DEB_BUILD)/debian/patches/$${fn}_$${f1n}; \ - done; \ - else \ - fb=$$(basename "$$f"); \ - cp "$$f" $(DEB_BUILD)/debian/patches/ ; \ - echo "$$fb" >> $(DEB_BUILD)/debian/patches/series ; \ - if ! grep -q "^Description:\|^Subject:" "$$f" ;then \ - sed -i '1 iSubject: Auto added patch' \ - "$(DEB_BUILD)/debian/patches/$$fb" ;fi ; \ - if ! grep -q "^Origin:\|^Author:\|^From:" "$$f" ;then \ - sed -i '1 iOrigin: other' \ - "$(DEB_BUILD)/debian/patches/$$fb" ;fi ; \ - fi ; \ + for f in $(PATCHES); do \ + rm -f $(DEB_TOP)/patches/*; \ + if git mailsplit -o$(DEB_TOP)/patches < "$$f"; then \ + fn=$$(basename "$$f"); \ + for f1 in $(DEB_TOP)/patches/*;do \ + [ -e "$$f1" ] || continue; \ + f1n=$$(basename "$$f1"); \ + echo "$${fn}_$${f1n}" >> $(DEB_BUILD)/debian/patches/series; \ + mv "$$f1" $(DEB_BUILD)/debian/patches/$${fn}_$${f1n}; \ + done; \ + else \ + fb=$$(basename "$$f"); \ + cp "$$f" $(DEB_BUILD)/debian/patches/; \ + echo "$$fb" >> $(DEB_BUILD)/debian/patches/series; \ + if ! grep -q "^Description:\|^Subject:" "$$f"; then \ + sed -i '1 iSubject: Auto added patch' \ + "$(DEB_BUILD)/debian/patches/$$fb"; \ + fi; \ + if ! grep -q "^Origin:\|^Author:\|^From:" "$$f"; then \ + sed -i '1 iOrigin: other' \ + "$(DEB_BUILD)/debian/patches/$$fb"; \ + fi; \ + fi; \ done touch $@ @@ -360,6 +369,7 @@ endif podman_chrootbuild: if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + --build-arg FVERSION=$(FVERSION) \ -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \ -f packaging/Dockerfile.mockbuild .; then \ echo "Container build failed"; \ @@ -377,7 +387,9 @@ podman_chrootbuild: exit 1; \ fi; \ rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm | \ - grep -v -e debuginfo -e debugsource -e src.rpm)' + grep -v -e debuginfo -e debugsource -e src.rpm)'; then \ + exit 1; \ + fi docker_chrootbuild: if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ @@ -466,6 +478,9 @@ show_rpms: show_source: @echo '$(SOURCE)' +show_real_source: + @echo '$(REAL_SOURCE)' + show_patches: @echo '$(PATCHES)' diff --git a/utils/rpms/packaging/rpm_chrootbuild b/utils/rpms/packaging/rpm_chrootbuild index 5e79337cae0..9661047727b 100755 --- a/utils/rpms/packaging/rpm_chrootbuild +++ b/utils/rpms/packaging/rpm_chrootbuild @@ -2,17 +2,10 @@ set -uex -mock_config_dir="${WORKSPACE:-$PWD}/mock" -original_cfg_file="/etc/mock/${CHROOT_NAME}.cfg" -cfg_file="$mock_config_dir/${CHROOT_NAME}.cfg" -mkdir -p "$mock_config_dir" -ln -sf /etc/mock/templates "$mock_config_dir/" -ln -sf /etc/mock/logging.ini "$mock_config_dir/" - -cp "$original_cfg_file" "$cfg_file" +cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then - cat <> "$cfg_file" + cat <> mock.cfg config_opts['module_setup_commands'] = [ ('enable', 'javapackages-tools:201801'), ('disable', 'go-toolset') @@ -25,12 +18,8 @@ if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then MOCK_OPTIONS="--dnf --no-bootstrap-chroot${MOCK_OPTIONS:+ }$MOCK_OPTIONS" fi -# disable bootstrap_image -# https://github.com/rpm-software-management/mock/issues/1184 -MOCK_OPTIONS="--no-bootstrap-image${MOCK_OPTIONS:+ }$MOCK_OPTIONS" - # Allow BR: foo-devel < 1.2 to work when foo-devel-1.3 is actually available -cat <> "$cfg_file" +cat <> mock.cfg config_opts['dnf.conf'] += """ [main] best=0 @@ -41,7 +30,7 @@ EOF repo_adds=() repo_dels=() -echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" +echo -e "config_opts['yum.conf'] += \"\"\"\n" >> mock.cfg if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then repo_dels+=("--disablerepo=\*") @@ -64,7 +53,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then REPO_FILE_URL="file://$(readlink -e "$REPO_FILES_PR")/" fi fi - curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-mock-artifactory.repo >> "$cfg_file" + curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg repo_adds+=("--enablerepo *-artifactory") fi fi @@ -85,7 +74,7 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do name=$repo:$branch:$build_number\n\ baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ -gpgcheck=False\n" >> "$cfg_file" +gpgcheck=False\n" >> mock.cfg done for repo in $JOB_REPOS; do repo_name=${repo##*://} @@ -93,19 +82,35 @@ for repo in $JOB_REPOS; do repo_adds+=("--enablerepo $repo_name") echo -e "[${repo_name//@/_}]\n\ name=${repo_name}\n\ -baseurl=${repo}\n\ -enabled=1\n" >> "$cfg_file" +baseurl=${repo//\//%2F}\n\ +enabled=1\n" >> mock.cfg done -echo "\"\"\"" >> "$cfg_file" +echo "\"\"\"" >> mock.cfg if [ -n "$DISTRO_VERSION" ]; then releasever_opt=("--config-opts=releasever=$DISTRO_VERSION") fi -# shellcheck disable=SC2086 -if ! eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ - ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ - "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET"; then - # Debug information for filing bugs on mock - rpm -q mock - mock --debug-config + +bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap +if ls -l /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then + mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap" + flock "$bs_dir" -c "cp -a $bs_dir/root_cache /var/cache/mock/${CHROOT_NAME}-bootstrap" +fi + +rc=0 +# shellcheck disable=SC2086,SC2048 +if ! eval mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \ + --disablerepo=\*-debug* ${releasever_opt[*]} $MOCK_OPTIONS \ + $RPM_BUILD_OPTIONS "$TARGET"; then + rc=${PIPESTATUS[0]} fi + +if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz && + [ -d /scratch/ ]; then + mkdir -p /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/ + if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then + flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/" + fi +fi + +exit "$rc" diff --git a/utils/scripts/helpers/repo-helper-leap15.sh b/utils/scripts/helpers/repo-helper-leap15.sh index 4d46d8db540..af9c9948956 100755 --- a/utils/scripts/helpers/repo-helper-leap15.sh +++ b/utils/scripts/helpers/repo-helper-leap15.sh @@ -7,7 +7,7 @@ set -uex # for custom packages if present. : "${REPO_FILE_URL:=}" -: "${BASE_DISTRO:=opensuse/leap:15.3}" +: "${BASE_DISTRO:=opensuse/leap:15.5}" : "${JENKINS_URL:=}" : "${REPOS:=}" From 79b5bd65badd1fdda2267ad57fa1f7023ca52f03 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Fri, 8 Dec 2023 08:36:42 -0800 Subject: [PATCH 11/23] DAOS-13433 test: increase timeout for dfuse/caching_check (#13355) (#13426) Increase timeout for dfuse/caching_check.py and add more logging. Signed-off-by: Dalton Bohning --- src/tests/ftest/dfuse/caching_check.py | 18 +++++++++--------- src/tests/ftest/dfuse/caching_check.yaml | 3 +-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/tests/ftest/dfuse/caching_check.py b/src/tests/ftest/dfuse/caching_check.py index bcd2db544e0..852a24f0dfd 100644 --- a/src/tests/ftest/dfuse/caching_check.py +++ b/src/tests/ftest/dfuse/caching_check.py @@ -43,34 +43,34 @@ def test_dfuse_caching_check(self): # update flag self.ior_cmd.update_params(flags=flags[0]) - # run ior to write to the dfuse mount point + self.log_step('Write to the dfuse mount point') self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False) - # update ior flag to read + self.log_step('Get baseline read performance from dfuse with caching disabled') self.ior_cmd.update_params(flags=flags[1]) - # run ior to read and store the read performance base_read_arr = [] out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False) base_read_arr.append(IorCommand.get_ior_metrics(out)) - # run ior again to read with caching disabled and store performance out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False) base_read_arr.append(IorCommand.get_ior_metrics(out)) # the index of max_mib max_mib = IorMetrics.MAX_MIB - # unmount dfuse and mount again with caching enabled + self.log_step('Re-mount dfuse with caching enabled') self.dfuse.unmount(tries=1) self.dfuse.update_params(disable_caching=False) self.dfuse.run() - # run ior to obtain first read performance after mount + + self.log_step('Get first read performance with caching enabled') out = self.run_ior_with_pool(fail_on_warning=False, stop_dfuse=False) base_read_arr.append(IorCommand.get_ior_metrics(out)) - # run ior again to obtain second read performance with caching enabled - # second read should be multiple times greater than first read + + self.log_step('Get cached read performance') out = self.run_ior_with_pool(fail_on_warning=False) with_caching = IorCommand.get_ior_metrics(out) - # verify cached read performance is multiple times greater than without caching + + self.log_step('Verify cached read performance is greater than first read') # Log all the values first, then do the assert so that failures can be checked easily. for base_read in base_read_arr: actual_change = percent_change(base_read[0][max_mib], with_caching[0][max_mib]) diff --git a/src/tests/ftest/dfuse/caching_check.yaml b/src/tests/ftest/dfuse/caching_check.yaml index 66928e7de04..304bc0ae77e 100644 --- a/src/tests/ftest/dfuse/caching_check.yaml +++ b/src/tests/ftest/dfuse/caching_check.yaml @@ -1,7 +1,7 @@ hosts: test_servers: 3 test_clients: 1 -timeout: 240 +timeout: 300 server_config: name: daos_server engines_per_host: 1 @@ -11,7 +11,6 @@ server_config: storage: auto pool: size: 50% - control_method: dmg container: type: POSIX control_method: daos From c962d2abc2245e6f208df368a827e9219901cfab Mon Sep 17 00:00:00 2001 From: Liu Xuezhao Date: Sat, 9 Dec 2023 00:38:17 +0800 Subject: [PATCH 12/23] DAOS-14655 tests: change no_cap.yaml to 1 target (#13389) (#13428) Signed-off-by: Xuezhao Liu --- src/tests/ftest/rebuild/no_cap.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tests/ftest/rebuild/no_cap.yaml b/src/tests/ftest/rebuild/no_cap.yaml index 25aca5676b1..f55793eebf9 100644 --- a/src/tests/ftest/rebuild/no_cap.yaml +++ b/src/tests/ftest/rebuild/no_cap.yaml @@ -12,7 +12,7 @@ server_config: engines_per_host: 2 engines: 0: - targets: 8 + targets: 1 pinned_numa_node: 0 nr_xs_helpers: 1 fabric_iface: ib0 @@ -20,7 +20,7 @@ server_config: log_file: daos_server0.log storage: auto 1: - targets: 8 + targets: 1 pinned_numa_node: 1 nr_xs_helpers: 1 fabric_iface: ib1 @@ -36,6 +36,6 @@ pool: pool_query_timeout: 30 pool_query_interval: 1 test_data_list: [1048576] - oclass: "OC_RP_2G1" + oclass: "OC_RP_4G1" rebuild: rank_to_kill: 1 From ddd9c5c014717128a09068877f20dda01a96ac1d Mon Sep 17 00:00:00 2001 From: Nasf-Fan Date: Sat, 9 Dec 2023 01:04:52 +0800 Subject: [PATCH 13/23] DAOS-14714 vos: return fetch handle only when vos_fetch_begin succeed - b24 (#13431) The old logic in vos_fetch_begin() may assign io_context handle to the output parameter too early, but some subsequent logic may destroy such handle after that because of some failure. That can cause the returned handle to be invalid as to related memory to be double freed. Signed-off-by: Fan Yong --- src/vos/vos_io.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/vos/vos_io.c b/src/vos/vos_io.c index 644ffa771dc..e4ba73826fc 100644 --- a/src/vos/vos_io.c +++ b/src/vos/vos_io.c @@ -1492,28 +1492,24 @@ vos_fetch_begin(daos_handle_t coh, daos_unit_oid_t oid, daos_epoch_t epoch, &rc, false)) { if (rc == 0) { if (ioc->ic_read_ts_only) - goto set_ioc; + goto out; + if (ioc->ic_obj != NULL && has_uncertainty(ioc, &ioc->ic_obj->obj_ilog_info)) goto fetch_dkey; for (i = 0; i < iod_nr; i++) iod_empty_sgl(ioc, i); - goto set_ioc; } goto out; } fetch_dkey: if (dkey == NULL || dkey->iov_len == 0) { - if (ioc->ic_read_ts_only) - goto set_ioc; - D_GOTO(out, rc = -DER_INVAL); + if (!ioc->ic_read_ts_only) + rc = -DER_INVAL; + } else { + rc = dkey_fetch(ioc, dkey); } - rc = dkey_fetch(ioc, dkey); - if (rc != 0) - goto out; -set_ioc: - *ioh = vos_ioc2ioh(ioc); out: vos_dth_set(NULL, ioc->ic_cont->vc_pool->vp_sysdb); @@ -1532,9 +1528,13 @@ vos_fetch_begin(daos_handle_t coh, daos_unit_oid_t oid, daos_epoch_t epoch, if (rc != 0) { daos_recx_ep_list_free(ioc->ic_recx_lists, ioc->ic_iod_nr); ioc->ic_recx_lists = NULL; - return vos_fetch_end(vos_ioc2ioh(ioc), NULL, rc); + rc = vos_fetch_end(vos_ioc2ioh(ioc), NULL, rc); + *ioh = DAOS_HDL_INVAL; + } else { + *ioh = vos_ioc2ioh(ioc); } - return 0; + + return rc; } static umem_off_t From 4e4dcb90e6f038741f2bab433495d956f67cacd0 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Fri, 8 Dec 2023 17:23:27 +0000 Subject: [PATCH 14/23] DAOS-14450 rebuild: Only print target buffers if there are any (#13127) (#13415) Only show tgts_buf if there are real targets in task structure. Signed-off-by: Di Wang Co-authored-by: wangdi --- src/rebuild/srv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rebuild/srv.c b/src/rebuild/srv.c index fca94bf4105..1cb174e8d18 100644 --- a/src/rebuild/srv.c +++ b/src/rebuild/srv.c @@ -1093,7 +1093,7 @@ rebuild_debug_print_queue() * This only accumulates the targets in a single task, so it doesn't * need to be very big. 200 bytes is enough for ~30 5-digit target ids */ - char tgts_buf[200]; + char tgts_buf[200] = { 0 }; int i; /* Position in stack buffer where str data should be written next */ size_t tgts_pos; @@ -1121,7 +1121,7 @@ rebuild_debug_print_queue() } D_DEBUG(DB_REBUILD, DF_UUID" op=%s ver=%u tgts=%s\n", DP_UUID(task->dst_pool_uuid), RB_OP_STR(task->dst_rebuild_op), - task->dst_map_ver, tgts_buf); + task->dst_map_ver, task->dst_tgts.pti_number > 0 ? tgts_buf : "None"); } } From 72f614993645478f72da5d8e3fea7007028ccbba Mon Sep 17 00:00:00 2001 From: Ravindran Padmanabhan Date: Fri, 8 Dec 2023 12:35:27 -0500 Subject: [PATCH 15/23] DAOS-14048 test: Avoid dmg_command objects race condition. (#12750) (#13458) Summary: - Race condition happens when dmg_objects from different threads updates the BasicParameter object. This can cause problem. - Avoid invoking dmg_command from the IOR thread under osa_utils.py - Presently, ior thread is checking the pool space (which uses a dmg command object). Signed-off-by: Padmanabhan --- src/tests/ftest/util/osa_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tests/ftest/util/osa_utils.py b/src/tests/ftest/util/osa_utils.py index cc5d119bf09..0b9c9002809 100644 --- a/src/tests/ftest/util/osa_utils.py +++ b/src/tests/ftest/util/osa_utils.py @@ -456,8 +456,11 @@ def ior_thread(self, pool, oclass, test, flags, single_cont_read=True, fail_on_w "Detected container redundancy factor: %s", self.container.properties.value) self.ior_cmd.dfs_oclass.update(None, "ior.dfs_oclass") self.ior_cmd.dfs_dir_oclass.update(None, "ior.dfs_dir_oclass") + # Run run_ior_with_pool without invoking the pool query method for + # displaying pool space information (display_space=False) self.run_ior_with_pool(create_pool=False, create_cont=False, fail_on_warning=fail_on_warning, + display_space=False, out_queue=self.out_queue) if fail_on_warning and not self.out_queue.empty(): self.assert_on_exception() From 205ca128611e7de70a040ba8a30376200be3d0f3 Mon Sep 17 00:00:00 2001 From: Ryon Jensen Date: Fri, 8 Dec 2023 11:15:51 -0700 Subject: [PATCH 16/23] DAOS-623 build: Allow deps repo and PR-repo... (#13460) To update modular RPMs. Fix Launchable analysis workaround. Signed-off-by: Brian J. Murrell --- ci/functional/launchable_analysis | 19 +++++++++++++------ .../post_provision_config_common_functions.sh | 8 ++++---- utils/scripts/helpers/repo-helper-el8.sh | 5 +++-- utils/scripts/helpers/repo-helper-el9.sh | 11 ++++++++--- utils/scripts/helpers/repo-helper-leap15.sh | 7 ++++--- 5 files changed, 32 insertions(+), 18 deletions(-) diff --git a/ci/functional/launchable_analysis b/ci/functional/launchable_analysis index a5485ce87c2..ee076207e25 100755 --- a/ci/functional/launchable_analysis +++ b/ci/functional/launchable_analysis @@ -5,14 +5,21 @@ set -eux dir="$1/$STAGE_NAME/" if ! mkdir -p "$dir"; then echo "Failed to create \"$dir\":" - while [ -n "$dir" ]; do - ls -ld "$dir" || true - dir=${dir%/*} - if [ -f "$dir" ]; then - cat "$dir" + _dir="$dir" + while [ -n "$_dir" ]; do + ls -ld "$_dir" || true + _dir=${_dir%/*} + if [ -f "$_dir" ]; then + cat "$_dir" fi done - exit 1 + if [ -f "$1" ]; then + mv "$1"{,."$(date +%s)"} + if ! mkdir -p "$dir"; then + echo "Failed to create \"$dir\"" + exit 1 + fi + fi fi notify=false diff --git a/ci/provisioning/post_provision_config_common_functions.sh b/ci/provisioning/post_provision_config_common_functions.sh index 7a97272347c..59bae6d5236 100755 --- a/ci/provisioning/post_provision_config_common_functions.sh +++ b/ci/provisioning/post_provision_config_common_functions.sh @@ -367,10 +367,10 @@ post_provision_config_nodes() { fi fi local repo_url="${ARTIFACTS_URL:-${JENKINS_URL}job/}"daos-stack/job/"$repo"/job/"${branch//\//%252F}"/"$build_number"/artifact/artifacts/$DISTRO_NAME/ - dnf -y config-manager --add-repo="${repo_url}" - # Disable module filtering - repo=$(url_to_repo "$repo_url") - dnf config-manager --save --setopt "$repo.module_hotfixes=true" "$repo" + dnf -y config-manager --add-repo="$repo_url" + repo="$(url_to_repo "$repo_url")" + # PR-repos: should always be able to upgrade modular packages + dnf -y config-manager --save --setopt "$repo.module_hotfixes=true" "$repo" disable_gpg_check "$repo_url" done diff --git a/utils/scripts/helpers/repo-helper-el8.sh b/utils/scripts/helpers/repo-helper-el8.sh index f1a3459d64b..2b512e3189f 100755 --- a/utils/scripts/helpers/repo-helper-el8.sh +++ b/utils/scripts/helpers/repo-helper-el8.sh @@ -97,8 +97,9 @@ for repo in $REPOS; do name=$repo:$branch:$build_number\n\ baseurl=${JENKINS_URL}$daos_base$repo/job/$branch/$build_number$artifacts\n\ enabled=1\n\ -gpgcheck=False\n" >> /etc/yum.repos.d/$repo:$branch:$build_number.repo - cat /etc/yum.repos.d/$repo:$branch:$build_number.repo +gpgcheck=False\n +module_hotfixes=true\n" >> /etc/yum.repos.d/"$repo:$branch:$build_number".repo + cat /etc/yum.repos.d/"$repo:$branch:$build_number".repo save_repos+=("$repo:$branch:$build_number") done diff --git a/utils/scripts/helpers/repo-helper-el9.sh b/utils/scripts/helpers/repo-helper-el9.sh index 60e1314bea2..fe1959589ef 100644 --- a/utils/scripts/helpers/repo-helper-el9.sh +++ b/utils/scripts/helpers/repo-helper-el9.sh @@ -75,7 +75,7 @@ if [ ! -f /etc/fedora-release ]; then PT_REPO=crb fi dnf install epel-release - dnf config-manager --enable $PT_REPO + dnf config-manager --enable "$PT_REPO" fi dnf clean all @@ -83,6 +83,10 @@ daos_base="job/daos-stack/job/" artifacts="/artifact/artifacts/el$MAJOR_VER/" save_repos=() for repo in $REPOS; do + # don't install daos@ repos since we are building daos + if [[ $repo = daos@* ]]; then + continue + fi branch="master" build_number="lastSuccessfulBuild" if [[ $repo = *@* ]]; then @@ -97,8 +101,9 @@ for repo in $REPOS; do name=$repo:$branch:$build_number\n\ baseurl=${JENKINS_URL}$daos_base$repo/job/$branch/$build_number$artifacts\n\ enabled=1\n\ -gpgcheck=False\n" >> /etc/yum.repos.d/$repo:$branch:$build_number.repo - cat /etc/yum.repos.d/$repo:$branch:$build_number.repo +gpgcheck=False\n +module_hotfixes=true\n" >> /etc/yum.repos.d/"$repo:$branch:$build_number".repo + cat /etc/yum.repos.d/"$repo:$branch:$build_number".repo save_repos+=("$repo:$branch:$build_number") done diff --git a/utils/scripts/helpers/repo-helper-leap15.sh b/utils/scripts/helpers/repo-helper-leap15.sh index af9c9948956..6afb27d0c3c 100755 --- a/utils/scripts/helpers/repo-helper-leap15.sh +++ b/utils/scripts/helpers/repo-helper-leap15.sh @@ -135,12 +135,13 @@ for repo in $REPOS; do name=$repo:$branch:$build_number\n\ baseurl=${JENKINS_URL}$daos_base$repo/job/$branch/$build_number$artifacts\n\ enabled=1\n\ -gpgcheck=False\n" >> $repos_dir$repo:$branch:$build_number.repo - cat $repos_dir$repo:$branch:$build_number.repo +gpgcheck=False\n +module_hotfixes=true\n" >> "$repos_dir$repo:$branch:$build_number".repo + cat "$repos_dir$repo:$branch:$build_number".repo save_repos+=("$repo:$branch:$build_number") done -disable_repos $repos_dir "${save_repos[@]}" +disable_repos "$repos_dir" "${save_repos[@]}" if [ -e /etc/profile.d/lmod.sh ]; then if ! grep "MODULEPATH=.*/usr/share/modules" /etc/profile.d/lmod.sh; then From c979f7325b69c83d8c0c4f9c6ba997aee78aad40 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 8 Dec 2023 18:03:31 -0500 Subject: [PATCH 17/23] DAOS-14737 build: Tag 2.4.1 rc1 (#13468) Tag first release candidate for 2.4.1. Signed-off-by: Phil Henderson --- TAG | 2 +- VERSION | 2 +- debian/changelog | 7 +++++++ utils/rpms/daos.spec | 7 +++++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/TAG b/TAG index 738a757a4a3..8fe19aac487 100644 --- a/TAG +++ b/TAG @@ -1 +1 @@ -2.4.0-rc2 +2.4.1-rc1 diff --git a/VERSION b/VERSION index 197c4d5c2d7..005119baaa0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0 +2.4.1 diff --git a/debian/changelog b/debian/changelog index 7018a806823..d9d5e80c087 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +daos (2.4.1-1) unstable; urgency=medium + + [ Phillip Henderson ] + * First release candidate for 2.4.1 + + -- Phillip Henderson Fri, 08 Dec 2023 16:12:00 -0500 + daos (2.4.0-5) unstable; urgency=medium [ Brian J. Murrell ] * Update for EL 8.8 and Leap 15.5 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index d73dfffeb17..3fc06773726 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -14,8 +14,8 @@ %endif Name: daos -Version: 2.4.0 -Release: 5%{?relval}%{?dist} +Version: 2.4.1 +Release: 1%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -555,6 +555,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Fri Dec 08 2023 Phillip Henderson 2.4.1-1 +- First release candidate for 2.4.1 + * Wed Dec 06 2023 Brian J. Murrell 2.4.0-3 - Update for EL 8.8 and Leap 15.5 - Update raft to 0.10.1-2.411.gefa15f4 From c83baea18b6e21ac17c380fcea640579843b5a21 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Tue, 19 Dec 2023 12:06:18 -0600 Subject: [PATCH 18/23] DAOS-14590 tests: increase test_daos_ec_obj timeout (#13376) (#13474) now test_daos_ec_obj with 17 "wait for 30 seconds for EC aggregation", that need 510S, so the original timeout of 600 Seconds possibly hit if the system is busy. Increase to 700 S. Signed-off-by: Xuezhao Liu --- src/tests/ftest/daos_test/suite.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/daos_test/suite.yaml b/src/tests/ftest/daos_test/suite.yaml index 279950ca2a2..4da1d5e2a64 100644 --- a/src/tests/ftest/daos_test/suite.yaml +++ b/src/tests/ftest/daos_test/suite.yaml @@ -15,7 +15,7 @@ timeouts: test_daos_verify_consistency: 105 test_daos_io: 290 test_daos_ec_io: 450 - test_daos_ec_obj: 600 + test_daos_ec_obj: 700 test_daos_object_array: 105 test_daos_array: 106 test_daos_kv: 105 From 3b346aca572fb1e19e26e300898fcfd6e49bcd8d Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 20 Dec 2023 07:33:10 -0600 Subject: [PATCH 19/23] DAOS-14784 dfs: do not use query_max_epoch if server is < 2.4 (#13521) DAOS 2.2.0 and previous (OBJ_VERSION <= 8) do not support querying just the max epoch of an object. So when a 2.4.x client is talking to a 2.2.x server, avoid issuing that operation to prevent an error on stat() operations. What that means is that mtime on directory objects is not always accurate and will use the initial entry mtime in that scenario. Signed-off-by: Mohamad Chaarawi --- src/client/dfs/dfs.c | 24 ++++++++++++++++-------- src/include/daos/object.h | 8 ++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/client/dfs/dfs.c b/src/client/dfs/dfs.c index 7e40b618dac..27005c01ade 100644 --- a/src/client/dfs/dfs.c +++ b/src/client/dfs/dfs.c @@ -1026,10 +1026,14 @@ entry_stat(dfs_t *dfs, daos_handle_t th, daos_handle_t oh, const char *name, siz return daos_der2errno(rc); } - rc = daos_obj_query_max_epoch(dir_oh, th, &ep, NULL); - if (rc) { - daos_obj_close(dir_oh, NULL); - return daos_der2errno(rc); + if (dc_supports_epoch_query()) { + rc = daos_obj_query_max_epoch(dir_oh, th, &ep, NULL); + if (rc) { + daos_obj_close(dir_oh, NULL); + return daos_der2errno(rc); + } + } else { + ep = 0; } rc = daos_obj_close(dir_oh, NULL); @@ -4130,10 +4134,14 @@ dfs_lookup_rel_int(dfs_t *dfs, dfs_obj_t *parent, const char *name, int flags, if (stbuf) { daos_epoch_t ep; - rc = daos_obj_query_max_epoch(obj->oh, DAOS_TX_NONE, &ep, NULL); - if (rc) { - daos_obj_close(obj->oh, NULL); - D_GOTO(err_obj, rc = daos_der2errno(rc)); + if (dc_supports_epoch_query()) { + rc = daos_obj_query_max_epoch(obj->oh, DAOS_TX_NONE, &ep, NULL); + if (rc) { + daos_obj_close(obj->oh, NULL); + D_GOTO(err_obj, rc = daos_der2errno(rc)); + } + } else { + ep = 0; } rc = update_stbuf_times(entry, ep, stbuf, NULL); diff --git a/src/include/daos/object.h b/src/include/daos/object.h index 3692c545f4d..3a8d3758858 100644 --- a/src/include/daos/object.h +++ b/src/include/daos/object.h @@ -830,4 +830,12 @@ int dc_obj_enum_unpack(daos_unit_oid_t oid, daos_key_desc_t *kds, int kds_num, d_sg_list_t *sgl, d_iov_t *csum, dc_obj_enum_unpack_cb_t cb, void *cb_arg); + +/** Check if server supports daos_obj_query_max_epoch */ +extern int dc_obj_proto_version; +static inline bool +dc_supports_epoch_query() +{ + return (dc_obj_proto_version >= 9); +} #endif /* __DD_OBJ_H__ */ From a770a78f27d256180a3cbbd6ad87928687c4f2b2 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Thu, 21 Dec 2023 08:21:01 -0700 Subject: [PATCH 20/23] DAOS-14707 control: Allow multiple agents simultaneously (#13465) (#13527) This allows for the case where multiple agents are used to allow a client connections to multiple systems. The agent log prints a NOTICE-level message to indicate the presence of other daos_agent processes on startup. Signed-off-by: Kris Jacque --- src/control/cmd/daos_agent/start.go | 5 +- src/control/drpc/drpc_server.go | 39 +++++-- src/control/drpc/drpc_server_test.go | 155 +++++++++++++++++++-------- src/control/drpc/fault.go | 27 +++++ src/control/fault/code/codes.go | 1 + utils/test_memcheck.supp | 6 ++ 6 files changed, 179 insertions(+), 54 deletions(-) create mode 100644 src/control/drpc/fault.go diff --git a/src/control/cmd/daos_agent/start.go b/src/control/cmd/daos_agent/start.go index 5814c3f5250..791af47ba63 100644 --- a/src/control/cmd/daos_agent/start.go +++ b/src/control/cmd/daos_agent/start.go @@ -48,7 +48,7 @@ type startCmd struct { func (cmd *startCmd) Execute(_ []string) error { if err := common.CheckDupeProcess(); err != nil { - return err + cmd.Notice(err.Error()) } cmd.Infof("Starting %s (pid %d)", versionString(), os.Getpid()) @@ -123,8 +123,7 @@ func (cmd *startCmd) Execute(_ []string) error { drpcSrvStart := time.Now() err = drpcServer.Start(hwlocCtx) if err != nil { - cmd.Errorf("Unable to start socket server on %s: %v", sockPath, err) - return err + return errors.Wrap(err, "unable to start dRPC server") } cmd.Debugf("dRPC socket server started: %s", time.Since(drpcSrvStart)) diff --git a/src/control/drpc/drpc_server.go b/src/control/drpc/drpc_server.go index 6627e4b6f47..35f85ef5758 100644 --- a/src/control/drpc/drpc_server.go +++ b/src/control/drpc/drpc_server.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2018-2022 Intel Corporation. +// (C) Copyright 2018-2023 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -87,28 +87,51 @@ func (d *DomainSocketServer) Listen(ctx context.Context) { // Start sets up the dRPC server socket and kicks off the listener goroutine. func (d *DomainSocketServer) Start(ctx context.Context) error { - // Just in case an old socket file is still lying around - if err := syscall.Unlink(d.sockFile); err != nil && !os.IsNotExist(err) { - return errors.Wrapf(err, "Unable to unlink %s", d.sockFile) + if d == nil { + return errors.New("DomainSocketServer is nil") } addr := &net.UnixAddr{Name: d.sockFile, Net: "unixpacket"} + if err := d.checkExistingSocket(ctx, addr); err != nil { + return err + } + lis, err := net.ListenUnix("unixpacket", addr) if err != nil { - return errors.Wrapf(err, "Unable to listen on unix socket %s", d.sockFile) + return errors.Wrapf(err, "unable to listen on unix socket %s", d.sockFile) } d.listener = lis - // The only writer should be the I/O Engines which should be running as the same user as - // daos_server process. if err := os.Chmod(d.sockFile, d.sockFileMode); err != nil { - return errors.Wrapf(err, "Unable to set permissions on %s", d.sockFile) + return errors.Wrapf(err, "unable to set permissions on %s", d.sockFile) } go d.Listen(ctx) return nil } +func (d *DomainSocketServer) checkExistingSocket(ctx context.Context, addr *net.UnixAddr) error { + conn, err := net.DialUnix("unixpacket", nil, addr) + if err == nil { + _ = conn.Close() + return FaultSocketFileInUse(d.sockFile) + } + + if errors.Is(err, syscall.ENOENT) { + return nil + } + + if errors.Is(err, syscall.ECONNREFUSED) { + // File exists but no one is listening - it's safe to delete. + if err := syscall.Unlink(addr.Name); err != nil && !os.IsNotExist(err) { + return errors.Wrap(err, "unlink old socket file") + } + return nil + } + + return err +} + // RegisterRPCModule takes a Module and associates it with the given // DomainSocketServer so it can be used to process incoming dRPC calls. func (d *DomainSocketServer) RegisterRPCModule(mod Module) { diff --git a/src/control/drpc/drpc_server_test.go b/src/control/drpc/drpc_server_test.go index a63175f95a9..1713a6e8b43 100644 --- a/src/control/drpc/drpc_server_test.go +++ b/src/control/drpc/drpc_server_test.go @@ -17,6 +17,8 @@ import ( "google.golang.org/protobuf/proto" "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/fault" + "github.com/daos-stack/daos/src/control/fault/code" "github.com/daos-stack/daos/src/control/logging" ) @@ -144,52 +146,119 @@ func TestNewDomainSocketServer(t *testing.T) { test.AssertEqual(t, dss.sockFile, expectedSock, "wrong sockfile") } -func TestServer_Start_CantUnlinkSocket(t *testing.T) { - log, buf := logging.NewTestLogger(t.Name()) - defer test.ShowBufferOnFailure(t, buf) - - tmpDir, tmpCleanup := test.CreateTestDir(t) - defer tmpCleanup() - - path := filepath.Join(tmpDir, "test.sock") - - // Forbid searching the directory - if err := os.Chmod(tmpDir, 0000); err != nil { - t.Fatalf("Couldn't change permissions on dir: %v", err) +func TestDrpc_DomainSocketServer_Start(t *testing.T) { + sockPath := func(dir string) string { + return filepath.Join(dir, "test.sock") } - defer func() { - _ = os.Chmod(tmpDir, 0700) - }() - - dss, _ := NewDomainSocketServer(log, path, testFileMode) - - err := dss.Start(test.Context(t)) - - test.CmpErr(t, errors.New("unlink"), err) -} -func TestServer_Start_CantListen(t *testing.T) { - log, buf := logging.NewTestLogger(t.Name()) - defer test.ShowBufferOnFailure(t, buf) - - tmpDir, tmpCleanup := test.CreateTestDir(t) - defer tmpCleanup() - - path := filepath.Join(tmpDir, "test.sock") - - // Forbid writing the directory - if err := os.Chmod(tmpDir, 0500); err != nil { - t.Fatalf("Couldn't change permissions on dir: %v", err) + for name, tc := range map[string]struct { + nilServer bool + setup func(t *testing.T, dir string) func() + expErr error + }{ + "nil": { + nilServer: true, + expErr: errors.New("nil"), + }, + "unused existing socket file": { + setup: func(t *testing.T, dir string) func() { + t.Helper() + + f, err := os.Create(sockPath(dir)) + if err != nil { + t.Fatal(err) + } + _ = f.Close() + return func() {} + }, + }, + "can't unlink old socket file": { + setup: func(t *testing.T, dir string) func() { + t.Helper() + + sockFile := sockPath(dir) + f, err := os.Create(sockFile) + if err != nil { + t.Fatal(err) + } + _ = f.Close() + + if err := os.Chmod(dir, 0500); err != nil { + t.Fatalf("Couldn't change permissions on dir: %v", err) + } + return func() { + _ = os.Chmod(dir, 0700) + } + }, + expErr: errors.New("unlink"), + }, + "socket file in use": { + setup: func(t *testing.T, dir string) func() { + t.Helper() + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + other, err := NewDomainSocketServer(log, sockPath(dir), testFileMode) + if err != nil { + t.Fatalf("can't create first server: %s", err.Error()) + } + + err = other.Start(test.Context(t)) + if err != nil { + t.Fatalf("can't start up first server: %s", err.Error()) + } + + // NB: The started server is shut down when the test context is canceled. + return func() {} + }, + expErr: FaultSocketFileInUse(""), + }, + "listen fails": { + setup: func(t *testing.T, dir string) func() { + t.Helper() + + if err := os.Chmod(dir, 0500); err != nil { + t.Fatalf("Couldn't change permissions on dir: %v", err) + } + return func() { + _ = os.Chmod(dir, 0700) + } + }, + expErr: errors.New("listen"), + }, + "success": {}, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + tmpDir, tmpCleanup := test.CreateTestDir(t) + defer tmpCleanup() + + if tc.setup != nil { + teardown := tc.setup(t, tmpDir) + defer teardown() + } + + // Test hack - make sure the right path is included in the fault message for comparison + if fault.IsFaultCode(tc.expErr, code.SocketFileInUse) { + tc.expErr = FaultSocketFileInUse(sockPath(tmpDir)) + } + + var err error + var dss *DomainSocketServer + if !tc.nilServer { + dss, err = NewDomainSocketServer(log, sockPath(tmpDir), testFileMode) + if err != nil { + t.Fatal(err) + } + } + + err = dss.Start(test.Context(t)) + + test.CmpErr(t, tc.expErr, err) + }) } - defer func() { - _ = os.Chmod(tmpDir, 0700) - }() - - dss, _ := NewDomainSocketServer(log, path, testFileMode) - - err := dss.Start(test.Context(t)) - - test.CmpErr(t, errors.New("listen"), err) } func TestServer_RegisterModule(t *testing.T) { diff --git a/src/control/drpc/fault.go b/src/control/drpc/fault.go new file mode 100644 index 00000000000..f41a7882a98 --- /dev/null +++ b/src/control/drpc/fault.go @@ -0,0 +1,27 @@ +// +// (C) Copyright 2023 Intel Corporation. +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package drpc + +import ( + "fmt" + + "github.com/daos-stack/daos/src/control/fault" + "github.com/daos-stack/daos/src/control/fault/code" +) + +// FaultSocketFileInUse indicates that the dRPC socket file was already in use when we tried +// to start the dRPC server. +func FaultSocketFileInUse(path string) *fault.Fault { + return &fault.Fault{ + Domain: "drpc", + Code: code.SocketFileInUse, + Description: fmt.Sprintf("Configured dRPC socket file '%s' is already in use.", path), + Reason: "dRPC socket file already in use", + Resolution: "If another process is using the socket file, configure a different socket directory. " + + "Otherwise, delete the existing socket file and try again.", + } +} diff --git a/src/control/fault/code/codes.go b/src/control/fault/code/codes.go index 89bfb32bed0..4a368aeb5e3 100644 --- a/src/control/fault/code/codes.go +++ b/src/control/fault/code/codes.go @@ -50,6 +50,7 @@ const ( PrivilegedHelperNotPrivileged PrivilegedHelperNotAvailable PrivilegedHelperRequestFailed + SocketFileInUse ) // generic storage fault codes diff --git a/utils/test_memcheck.supp b/utils/test_memcheck.supp index 4f0d88fb28d..d1260ecf572 100644 --- a/utils/test_memcheck.supp +++ b/utils/test_memcheck.supp @@ -383,3 +383,9 @@ ... fun:mdb_txn_commit } +{ + go_runtime_syscall_param + Memcheck:Param + write(buf) + fun:runtime/internal/syscall.Syscall6 +} From 542d6e1426e627ab03bb353ee67e74644663b6d6 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 21 Dec 2023 10:32:11 -0500 Subject: [PATCH 21/23] DAOS-14591 test: Avoid including xml CDATA in the job.log (#13310) (#13514) TO prevent xml parsing errors do not include CDATA sections of other xml files in the launch.py job.log by only logging the diff of xml file changes. Signed-off-by: Phil Henderson --- src/tests/ftest/util/collection_utils.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index b8e996cfc25..f160af869c9 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -9,6 +9,7 @@ import re import sys from collections import OrderedDict +from difflib import unified_diff from ClusterShell.NodeSet import NodeSet from process_core_files import CoreFileException, CoreFileProcessing @@ -870,12 +871,6 @@ def replace_xml(logger, xml_file, pattern, replacement, xml_data, test_result): str: the updated xml_data; None if an error was detected """ logger.debug("Replacing '%s' with '%s' in %s", pattern, replacement, xml_file) - - logger.debug(" Contents of %s before replacement", xml_file) - for line in xml_data.splitlines(): - logger.debug(" %s", line) - logger.debug("") - try: with open(xml_file, "w", encoding="utf-8") as xml_buffer: xml_buffer.write(re.sub(pattern, replacement, xml_data)) @@ -886,8 +881,9 @@ def replace_xml(logger, xml_file, pattern, replacement, xml_data, test_result): new_xml_data = get_xml_data(logger, xml_file, test_result) if new_xml_data is not None: - logger.debug(" Contents of %s after replacement", xml_file) - for line in new_xml_data.splitlines(): + logger.debug(" Diff of %s after replacement", xml_file) + for line in unified_diff(xml_data.splitlines(), new_xml_data.splitlines(), + fromfile=xml_file, tofile=xml_file, n=0, lineterm=""): logger.debug(" %s", line) logger.debug("") From b65ab3c01e130d4e6c3368d8c5e2d8a6de79ff08 Mon Sep 17 00:00:00 2001 From: Nasf-Fan Date: Fri, 22 Dec 2023 00:31:50 +0800 Subject: [PATCH 22/23] DAOS-13129 vos: refresh lru entry key with DTX epoch renew - b24 (#13384) The transaction epoch maybe renewed after DTX entry allocated. Under such case, related lru entry for the DTX entry needs to be refreshed for its key. Otherwise, we cannot locate such lru entry with the new DTX epoch. Signed-off-by: Fan Yong --- src/dtx/dtx_common.c | 9 ++-- src/include/daos_srv/dtx_srv.h | 1 + src/include/daos_srv/vos.h | 8 +++ src/object/srv_obj.c | 68 ++++++++++++++++++++++--- src/tests/ftest/util/telemetry_utils.py | 2 +- src/vos/lru_array.h | 18 +++++-- src/vos/tests/vts_ilog.c | 2 +- src/vos/vos_dtx.c | 26 ++++++++-- 8 files changed, 113 insertions(+), 21 deletions(-) diff --git a/src/dtx/dtx_common.c b/src/dtx/dtx_common.c index d2af1b0c06e..5a7f65474dd 100644 --- a/src/dtx/dtx_common.c +++ b/src/dtx/dtx_common.c @@ -1006,6 +1006,7 @@ dtx_renew_epoch(struct dtx_epoch *epoch, struct dtx_handle *dth) { dth->dth_epoch = epoch->oe_value; dth->dth_epoch_bound = dtx_epoch_bound(epoch); + vos_dtx_renew_epoch(dth); } /** @@ -1147,9 +1148,9 @@ dtx_leader_begin(daos_handle_t coh, struct dtx_id *dti, if (rc == 0 && sub_modification_cnt > 0) rc = vos_dtx_attach(dth, false, (flags & DTX_PREPARED) ? true : false); - D_DEBUG(DB_IO, "Start DTX "DF_DTI" sub modification %d, ver %u, leader " + D_DEBUG(DB_IO, "Start DTX "DF_DTI" sub modification %d, ver %u, epoch "DF_X64", leader " DF_UOID", dti_cos_cnt %d, tgt_cnt %d, flags %x: "DF_RC"\n", - DP_DTI(dti), sub_modification_cnt, dth->dth_ver, + DP_DTI(dti), sub_modification_cnt, dth->dth_ver, epoch->oe_value, DP_UOID(*leader_oid), dti_cos_cnt, tgt_cnt, flags, DP_RC(rc)); if (rc != 0) { @@ -1483,9 +1484,9 @@ dtx_begin(daos_handle_t coh, struct dtx_id *dti, rc = vos_dtx_attach(dth, false, false); D_DEBUG(DB_IO, "Start DTX "DF_DTI" sub modification %d, ver %u, " - "dti_cos_cnt %d, flags %x: "DF_RC"\n", + "epoch "DF_X64", dti_cos_cnt %d, flags %x: "DF_RC"\n", DP_DTI(dti), sub_modification_cnt, - dth->dth_ver, dti_cos_cnt, flags, DP_RC(rc)); + dth->dth_ver, epoch->oe_value, dti_cos_cnt, flags, DP_RC(rc)); if (rc != 0) D_FREE(dth); diff --git a/src/include/daos_srv/dtx_srv.h b/src/include/daos_srv/dtx_srv.h index 864f48b820a..6d0936a5b55 100644 --- a/src/include/daos_srv/dtx_srv.h +++ b/src/include/daos_srv/dtx_srv.h @@ -123,6 +123,7 @@ struct dtx_handle { struct dtx_rsrvd_uint dth_rsrvd_inline; struct dtx_rsrvd_uint *dth_rsrvds; void **dth_deferred; + void *dth_local_stub; /* NVME extents to release */ d_list_t dth_deferred_nvme; /* Committed or comittable DTX list */ diff --git a/src/include/daos_srv/vos.h b/src/include/daos_srv/vos.h index 8a749eae1b0..c26cf02f96b 100644 --- a/src/include/daos_srv/vos.h +++ b/src/include/daos_srv/vos.h @@ -852,6 +852,14 @@ vos_update_end(daos_handle_t ioh, uint32_t pm_ver, daos_key_t *dkey, int err, void vos_update_renew_epoch(daos_handle_t ioh, struct dtx_handle *dth); +/** + * Renew the epoch for the DTX entry. + * + * \param dth [IN] Pointer to the DTX handle. + */ +void +vos_dtx_renew_epoch(struct dtx_handle *dth); + /** * Get the recx/epoch list. * diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c index 5b35647dc54..334dae8b4da 100644 --- a/src/object/srv_obj.c +++ b/src/object/srv_obj.c @@ -1785,7 +1785,7 @@ obj_local_rw_internal(crt_rpc_t *rpc, struct obj_io_context *ioc, daos_iod_t *io vos_update_renew_epoch(ioh, dth); D_DEBUG(DB_IO, - "update rpc %p renew epoch "DF_U64" => "DF_U64" for "DF_DTI"\n", + "update rpc %p renew epoch "DF_X64" => "DF_X64" for "DF_DTI"\n", rpc, orw->orw_epoch, dth->dth_epoch, DP_DTI(&orw->orw_dti)); orw->orw_epoch = dth->dth_epoch; @@ -3483,8 +3483,9 @@ static int obj_local_punch(struct obj_punch_in *opi, crt_opcode_t opc, struct obj_io_context *ioc, struct dtx_handle *dth) { - struct ds_cont_child *cont = ioc->ioc_coc; - int rc = 0; + struct ds_cont_child *cont = ioc->ioc_coc; + uint64_t sched_seq; + int rc = 0; if (daos_is_zero_dti(&opi->opi_dti)) { D_DEBUG(DB_TRACE, "disable dtx\n"); @@ -3496,6 +3497,8 @@ obj_local_punch(struct obj_punch_in *opi, crt_opcode_t opc, if (rc != 0) goto out; + sched_seq = sched_cur_seq(); + switch (opc) { case DAOS_OBJ_RPC_PUNCH: case DAOS_OBJ_RPC_TGT_PUNCH: @@ -3528,8 +3531,60 @@ obj_local_punch(struct obj_punch_in *opi, crt_opcode_t opc, if (dth != NULL && obj_dtx_need_refresh(dth, rc)) { rc = dtx_refresh(dth, ioc->ioc_coc); - if (rc == -DER_AGAIN) + if (rc != -DER_AGAIN) + goto out; + + if (unlikely(sched_cur_seq() == sched_seq)) goto again; + + /* + * There is CPU yield after DTX start, and the resent RPC may be handled + * during that. Let's check resent again before further process. + */ + + if (dth->dth_need_validation) { + daos_epoch_t epoch = 0; + int rc1; + + rc1 = dtx_handle_resend(ioc->ioc_vos_coh, &opi->opi_dti, &epoch, NULL); + switch (rc1) { + case 0: + opi->opi_epoch = epoch; + /* Fall through */ + case -DER_ALREADY: + rc = -DER_ALREADY; + break; + case -DER_NONEXIST: + case -DER_EP_OLD: + break; + default: + rc = rc1; + break; + } + } + + /* + * For solo punch, it will be handled via one-phase transaction. If there is CPU + * yield after its epoch generated, we will renew the epoch, then we can use the + * epoch to sort related solo DTXs based on their epochs. + */ + if (rc == -DER_AGAIN && dth->dth_solo) { + struct dtx_epoch epoch; + + epoch.oe_value = d_hlc_get(); + epoch.oe_first = epoch.oe_value; + epoch.oe_flags = orf_to_dtx_epoch_flags(opi->opi_flags); + + dtx_renew_epoch(&epoch, dth); + + D_DEBUG(DB_IO, + "punch rpc %u renew epoch "DF_X64" => "DF_X64" for "DF_DTI"\n", + opc, opi->opi_epoch, dth->dth_epoch, DP_DTI(&opi->opi_dti)); + + opi->opi_epoch = dth->dth_epoch; + } + + goto again; } out: @@ -4463,10 +4518,11 @@ ds_cpd_handle_one(crt_rpc_t *rpc, struct daos_cpd_sub_head *dcsh, dcsh->dcsh_epoch.oe_value = d_hlc_get(); dtx_renew_epoch(&dcsh->dcsh_epoch, dth); - vos_update_renew_epoch(iohs[0], dth); + if (daos_handle_is_valid(iohs[0])) + vos_update_renew_epoch(iohs[0], dth); D_DEBUG(DB_IO, - "CPD rpc %p renew epoch "DF_U64" => "DF_U64" for "DF_DTI"\n", + "CPD rpc %p renew epoch "DF_X64" => "DF_X64" for "DF_DTI"\n", rpc, epoch, dcsh->dcsh_epoch.oe_value, DP_DTI(&dcsh->dcsh_xid)); } } diff --git a/src/tests/ftest/util/telemetry_utils.py b/src/tests/ftest/util/telemetry_utils.py index 021ba517b1b..6656baa2df3 100644 --- a/src/tests/ftest/util/telemetry_utils.py +++ b/src/tests/ftest/util/telemetry_utils.py @@ -562,7 +562,7 @@ class TelemetryUtils(): "engine_mem_vos_dtx_cmt_ent_48", "engine_mem_vos_vos_obj_360", "engine_mem_vos_vos_lru_size", - "engine_mem_dtx_dtx_leader_handle_336", + "engine_mem_dtx_dtx_leader_handle_344", "engine_mem_dtx_dtx_entry_40"] ENGINE_MEM_TOTAL_USAGE_METRICS = [ "engine_mem_total_mem"] diff --git a/src/vos/lru_array.h b/src/vos/lru_array.h index 193e9cf4260..e1aeeddfd0d 100644 --- a/src/vos/lru_array.h +++ b/src/vos/lru_array.h @@ -334,11 +334,11 @@ lrua_peek_(struct lru_array *array, const uint32_t *idx, void **entryp) * -DER_BUSY Entries need to be evicted to free up * entries in the table */ -#define lrua_allocx(array, idx, key, entryp) \ - lrua_allocx_(array, idx, key, (void **)(entryp)) +#define lrua_allocx(array, idx, key, entryp, stub) \ + lrua_allocx_(array, idx, key, (void **)(entryp), (void **)(stub)) static inline int lrua_allocx_(struct lru_array *array, uint32_t *idx, uint64_t key, - void **entryp) + void **entryp, void **stub) { struct lru_entry *new_entry; int rc; @@ -354,6 +354,8 @@ lrua_allocx_(struct lru_array *array, uint32_t *idx, uint64_t key, return rc; *entryp = new_entry->le_payload; + if (stub != NULL) + *stub = new_entry; return 0; } @@ -378,7 +380,7 @@ lrua_allocx_(struct lru_array *array, uint32_t *idx, uint64_t key, static inline int lrua_alloc_(struct lru_array *array, uint32_t *idx, void **entryp) { - return lrua_allocx_(array, idx, (uint64_t)idx, entryp); + return lrua_allocx_(array, idx, (uint64_t)idx, entryp, NULL); } /** Allocate an entry in place. Used for recreating an old array. @@ -497,4 +499,12 @@ lrua_array_free(struct lru_array *array); void lrua_array_aggregate(struct lru_array *array); +static inline void +lrua_refresh_key(struct lru_entry *entry, uint64_t key) +{ + D_ASSERT(entry != NULL); + + entry->le_key = key; +} + #endif /* __LRU_ARRAY__ */ diff --git a/src/vos/tests/vts_ilog.c b/src/vos/tests/vts_ilog.c index 6d4b0a5c88c..1b85d2cb683 100644 --- a/src/vos/tests/vts_ilog.c +++ b/src/vos/tests/vts_ilog.c @@ -158,7 +158,7 @@ fake_tx_log_add(struct umem_instance *umm, umem_off_t offset, uint32_t *tx_id, uint32_t idx; int rc; - rc = lrua_allocx(array, &idx, epoch, &entry); + rc = lrua_allocx(array, &idx, epoch, &entry, NULL); assert_rc_equal(rc, 0); assert_non_null(entry); diff --git a/src/vos/vos_dtx.c b/src/vos/vos_dtx.c index ccebf569082..3a0ad9cc938 100644 --- a/src/vos/vos_dtx.c +++ b/src/vos/vos_dtx.c @@ -40,8 +40,8 @@ enum { D_ASSERT(dae->dae_dth->dth_ent == dae); \ dae->dae_dth->dth_ent = NULL; \ } \ - D_DEBUG(DB_TRACE, "Evicting lid "DF_DTI": lid=%d\n", \ - DP_DTI(&DAE_XID(dae)), DAE_LID(dae)); \ + D_DEBUG(DB_TRACE, "Evicting lid "DF_DTI": lid=%lx\n", \ + DP_DTI(&DAE_XID(dae)), DAE_LID(dae) & DTX_LID_SOLO_MASK); \ d_list_del_init(&dae->dae_link); \ lrua_evictx(cont->vc_dtx_array, \ (DAE_LID(dae) & DTX_LID_SOLO_MASK) - DTX_LID_RESERVED, \ @@ -963,7 +963,7 @@ vos_dtx_alloc(struct vos_dtx_blob_df *dbd, struct dtx_handle *dth) cont = vos_hdl2cont(dth->dth_coh); D_ASSERT(cont != NULL); - rc = lrua_allocx(cont->vc_dtx_array, &idx, dth->dth_epoch, &dae); + rc = lrua_allocx(cont->vc_dtx_array, &idx, dth->dth_epoch, &dae, &dth->dth_local_stub); if (rc != 0) { /* The array is full, need to commit some transactions first */ if (rc == -DER_BUSY) @@ -1007,8 +1007,8 @@ vos_dtx_alloc(struct vos_dtx_blob_df *dbd, struct dtx_handle *dth) dae->dae_dbd = dbd; dae->dae_dth = dth; - D_DEBUG(DB_IO, "Allocated new lid DTX: "DF_DTI" lid=%d dae=%p" - " dae_dbd=%p\n", DP_DTI(&dth->dth_xid), DAE_LID(dae), dae, dbd); + D_DEBUG(DB_IO, "Allocated new lid DTX: "DF_DTI" lid=%lx, dae=%p, dae_dbd=%p\n", + DP_DTI(&dth->dth_xid), DAE_LID(dae) & DTX_LID_SOLO_MASK, dae, dbd); d_iov_set(&kiov, &DAE_XID(dae), sizeof(DAE_XID(dae))); d_iov_set(&riov, dae, sizeof(*dae)); @@ -1155,6 +1155,10 @@ vos_dtx_check_availability(daos_handle_t coh, uint32_t entry, found = lrua_lookupx(cont->vc_dtx_array, (entry & DTX_LID_SOLO_MASK) - DTX_LID_RESERVED, epoch, &dae); if (!found) { + D_ASSERTF(!(entry & DTX_LID_SOLO_FLAG), + "non-committed solo entry %lu must be there, epoch "DF_X64", boundary " + DF_X64"\n", entry & DTX_LID_SOLO_MASK, epoch, cont->vc_solo_dtx_epoch); + D_DEBUG(DB_TRACE, "Entry %d "DF_U64" not in lru array, it must be committed\n", entry, epoch); @@ -3238,3 +3242,15 @@ vos_dtx_cache_reset(daos_handle_t coh, bool force) return 0; } + +void +vos_dtx_renew_epoch(struct dtx_handle *dth) +{ + struct vos_dtx_act_ent *dae = dth->dth_ent; + + if (dae != NULL) + DAE_EPOCH(dae) = dth->dth_epoch; + + if (dth->dth_local_stub != NULL) + lrua_refresh_key(dth->dth_local_stub, dth->dth_epoch); +} From ebf7ed0fd6eb87f25f22ff1a05734ac651eebd27 Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Fri, 1 Dec 2023 07:23:29 -0800 Subject: [PATCH 23/23] DAOS-14650 test: replace mpirun bind-to and map-by with args (#13392) Skip-test: true Skip-unit-test: true Skip-unit-tests: true - Replace --bind-to and --map-by with generic args to support flexible options. - Update mpirun np AND ppn to be compatible with more implementations Required-githooks: true Signed-off-by: Dalton Bohning --- src/tests/ftest/deployment/agent_failure.py | 3 +-- src/tests/ftest/deployment/target_failure.py | 3 +-- src/tests/ftest/performance/ior_easy.yaml | 3 +-- src/tests/ftest/performance/ior_hard.yaml | 3 +-- src/tests/ftest/performance/mdtest_easy.yaml | 3 +-- src/tests/ftest/performance/mdtest_hard.yaml | 3 +-- .../ftest/telemetry/pool_space_metrics.yaml | 2 +- src/tests/ftest/util/data_mover_utils.py | 6 +---- src/tests/ftest/util/ior_test_base.py | 8 +++---- src/tests/ftest/util/ior_utils.py | 8 +++---- src/tests/ftest/util/job_manager_utils.py | 24 +++++++++++-------- src/tests/ftest/util/mdtest_test_base.py | 8 +++---- 12 files changed, 34 insertions(+), 40 deletions(-) diff --git a/src/tests/ftest/deployment/agent_failure.py b/src/tests/ftest/deployment/agent_failure.py index 3bac851f4ef..181b43b689c 100644 --- a/src/tests/ftest/deployment/agent_failure.py +++ b/src/tests/ftest/deployment/agent_failure.py @@ -45,8 +45,7 @@ def run_ior_collect_error(self, results, job_num, file_name, clients): mpi_type="mpich") manager.assign_hosts(clients_nodeset, self.workdir, self.hostfile_clients_slots) ppn = self.params.get("ppn", '/run/ior/client_processes/*') - manager.ppn.update(ppn, 'mpirun.ppn') - manager.processes.update(None, 'mpirun.np') + manager.assign_processes(ppn=ppn) try: ior_output = manager.run() diff --git a/src/tests/ftest/deployment/target_failure.py b/src/tests/ftest/deployment/target_failure.py index 1751148f073..2ddeebfccc9 100644 --- a/src/tests/ftest/deployment/target_failure.py +++ b/src/tests/ftest/deployment/target_failure.py @@ -48,8 +48,7 @@ def run_ior_report_error(self, results, job_num, file_name, pool, container, nam manager.assign_hosts( self.hostlist_clients, self.workdir, self.hostfile_clients_slots) ppn = self.params.get("ppn", '/run/ior/client_processes/*') - manager.ppn.update(ppn, 'mpirun.ppn') - manager.processes.update(None, 'mpirun.np') + manager.assign_processes(ppn=ppn) # Run the command. try: diff --git a/src/tests/ftest/performance/ior_easy.yaml b/src/tests/ftest/performance/ior_easy.yaml index d5c38dcc482..52767bc4d7b 100644 --- a/src/tests/ftest/performance/ior_easy.yaml +++ b/src/tests/ftest/performance/ior_easy.yaml @@ -93,5 +93,4 @@ client: env_vars: - D_LOG_MASK=INFO mpirun: - bind_to: hwthread - map_by: socket + args: "--bind-to hwthread --map-by socket" diff --git a/src/tests/ftest/performance/ior_hard.yaml b/src/tests/ftest/performance/ior_hard.yaml index c18517de947..db1d225a92c 100644 --- a/src/tests/ftest/performance/ior_hard.yaml +++ b/src/tests/ftest/performance/ior_hard.yaml @@ -72,5 +72,4 @@ client: env_vars: - D_LOG_MASK=INFO mpirun: - bind_to: hwthread - map_by: socket + args: "--bind-to hwthread --map-by socket" diff --git a/src/tests/ftest/performance/mdtest_easy.yaml b/src/tests/ftest/performance/mdtest_easy.yaml index cf0f2d6460f..78d64b4aa1e 100644 --- a/src/tests/ftest/performance/mdtest_easy.yaml +++ b/src/tests/ftest/performance/mdtest_easy.yaml @@ -76,5 +76,4 @@ client: env_vars: - D_LOG_MASK=INFO mpirun: - bind_to: hwthread - map_by: socket + args: "--bind-to hwthread --map-by socket" diff --git a/src/tests/ftest/performance/mdtest_hard.yaml b/src/tests/ftest/performance/mdtest_hard.yaml index fa0b2bd230c..8e2e974625e 100644 --- a/src/tests/ftest/performance/mdtest_hard.yaml +++ b/src/tests/ftest/performance/mdtest_hard.yaml @@ -72,5 +72,4 @@ client: env_vars: - D_LOG_MASK=INFO mpirun: - bind_to: hwthread - map_by: socket + args: "--bind-to hwthread --map-by socket" diff --git a/src/tests/ftest/telemetry/pool_space_metrics.yaml b/src/tests/ftest/telemetry/pool_space_metrics.yaml index 459e2a1954d..9a2e261f6b2 100644 --- a/src/tests/ftest/telemetry/pool_space_metrics.yaml +++ b/src/tests/ftest/telemetry/pool_space_metrics.yaml @@ -44,7 +44,7 @@ ior: env_vars: - D_LOG_MASK=INFO mpirun: - bind_to: socket + args: "--bind-to socket" scm_metric_thresholds: # Maximal metadata size is empirically adjusted to 8MiB diff --git a/src/tests/ftest/util/data_mover_utils.py b/src/tests/ftest/util/data_mover_utils.py index 15c89b6651b..a0b715faa16 100644 --- a/src/tests/ftest/util/data_mover_utils.py +++ b/src/tests/ftest/util/data_mover_utils.py @@ -101,11 +101,7 @@ def run(self, processes, job_manager, ppn=None, env=None): # Get job manager cmd job_manager = Mpirun(self, mpi_type="mpich") job_manager.assign_hosts(self.hosts, self.tmp) - if ppn is None: - job_manager.assign_processes(processes) - else: - job_manager.ppn.update(ppn, 'mpirun.ppn') - job_manager.processes.update(None, 'mpirun.np') + job_manager.assign_processes(processes, ppn) job_manager.exit_status_exception = self.exit_status_exception job_manager.assign_environment(env or {}, True) diff --git a/src/tests/ftest/util/ior_test_base.py b/src/tests/ftest/util/ior_test_base.py index bc593dd7a17..08c65a41754 100644 --- a/src/tests/ftest/util/ior_test_base.py +++ b/src/tests/ftest/util/ior_test_base.py @@ -225,11 +225,11 @@ def run_ior(self, manager, processes, intercept=None, display_space=True, manager.working_dir.value = self.dfuse.mount_dir.value manager.assign_hosts( self.hostlist_clients, self.workdir, self.hostfile_clients_slots) - if self.ppn is None: - manager.assign_processes(processes) + # Pass only processes or ppn to be compatible with previous behavior + if self.ppn is not None: + manager.assign_processes(ppn=self.ppn) else: - manager.ppn.update(self.ppn, 'mpirun.ppn') - manager.processes.update(None, 'mpirun.np') + manager.assign_processes(processes=processes) manager.assign_environment(env) diff --git a/src/tests/ftest/util/ior_utils.py b/src/tests/ftest/util/ior_utils.py index fb56993a260..cd54b0e19af 100644 --- a/src/tests/ftest/util/ior_utils.py +++ b/src/tests/ftest/util/ior_utils.py @@ -571,11 +571,11 @@ def run(self, group, pool, container, processes, ppn=None, intercept=None, plugi self.manager.job.test_file.update( os.path.join(os.sep, self.label_generator.get_label("testfile"))) - if ppn is None: - self.manager.assign_processes(processes) + # Pass only processes or ppn to be compatible with previous behavior + if ppn is not None: + self.manager.assign_processes(ppn=ppn) else: - self.manager.ppn.update(ppn, ".".join([self.manager.command, "ppn"])) - self.manager.processes.update(None, ".".join([self.manager.command, "np"])) + self.manager.assign_processes(processes=processes) self.manager.assign_environment(self.env) diff --git a/src/tests/ftest/util/job_manager_utils.py b/src/tests/ftest/util/job_manager_utils.py index afb9b3e4bdd..adbf004162c 100644 --- a/src/tests/ftest/util/job_manager_utils.py +++ b/src/tests/ftest/util/job_manager_utils.py @@ -11,7 +11,7 @@ from ClusterShell.NodeSet import NodeSet from command_utils import ExecutableCommand, SystemctlCommand -from command_utils_base import EnvironmentVariables, FormattedParameter +from command_utils_base import BasicParameter, EnvironmentVariables, FormattedParameter from env_modules import load_mpi from exception_utils import CommandFailure, MPILoadError from general_utils import (get_job_manager_class, get_journalctl_command, journalctl_time, pcmd, @@ -162,12 +162,12 @@ def assign_hosts(self, hosts, path=None, slots=None, hostfile=True): """ def assign_processes(self, processes): - """Assign the number of processes per node. + """Assign the number of processes. Set the appropriate command line parameter with the specified value. Args: - processes (int): number of processes per node + processes (int): number of processes """ def assign_environment(self, env_vars, append=False): @@ -336,10 +336,10 @@ def assign_hosts(self, hosts, path=None, slots=None, hostfile=True): self.hostfile.value = write_host_file(**kwargs) def assign_processes(self, processes): - """Assign the number of processes per node (-np). + """Assign the number of processes (-np). Args: - processes (int): number of processes per node + processes (int): number of processes """ self.processes.value = processes @@ -432,8 +432,7 @@ def __init__(self, job, subprocess=False, mpi_type="openmpi"): self.mca = FormattedParameter("--mca {}", mca_default) self.working_dir = FormattedParameter("-wdir {}", None) self.tmpdir_base = FormattedParameter("--mca orte_tmpdir_base {}", None) - self.bind_to = FormattedParameter("--bind-to {}", None) - self.map_by = FormattedParameter("--map-by {}", None) + self.args = BasicParameter(None, None) self.mpi_type = mpi_type def assign_hosts(self, hosts, path=None, slots=None, hostfile=True): @@ -455,13 +454,18 @@ def assign_hosts(self, hosts, path=None, slots=None, hostfile=True): kwargs["path"] = path self.hostfile.value = write_host_file(**kwargs) - def assign_processes(self, processes): - """Assign the number of processes per node (-np). + def assign_processes(self, processes=None, ppn=None): + """Assign the number of processes (-np) and processes per node (-ppn). Args: - processes (int): number of processes per node + processes (int, optional): number of processes. Defaults to None. + if not specified, auto-calculated from ppn. + ppn (int, optional): number of processes per node. Defaults to None. """ + if ppn is not None and processes is None: + processes = ppn * len(self._hosts) self.processes.update(processes, "mpirun.np") + self.ppn.update(ppn, "mpirun.ppn") def assign_environment(self, env_vars, append=False): """Assign or add environment variables to the command. diff --git a/src/tests/ftest/util/mdtest_test_base.py b/src/tests/ftest/util/mdtest_test_base.py index 04aa44e5814..8646d4ef0ac 100644 --- a/src/tests/ftest/util/mdtest_test_base.py +++ b/src/tests/ftest/util/mdtest_test_base.py @@ -124,11 +124,11 @@ def run_mdtest(self, manager, processes, display_space=True, pool=None, out_queu """ env = self.mdtest_cmd.get_default_env(str(manager), self.client_log) manager.assign_hosts(self.hostlist_clients, self.workdir, self.hostfile_clients_slots) - if self.ppn is None: - manager.assign_processes(processes) + # Pass only processes or ppn to be compatible with previous behavior + if self.ppn is not None: + manager.assign_processes(ppn=self.ppn) else: - manager.ppn.update(self.ppn, 'mpirun.ppn') - manager.processes.update(None, 'mpirun.np') + manager.assign_processes(processes=processes) manager.assign_environment(env)