From 5f50c0e7a72671e06605c05be37224d139807911 Mon Sep 17 00:00:00 2001 From: Fan Yong Date: Tue, 17 Mar 2026 13:35:24 +0800 Subject: [PATCH] DAOS-18674 chk: CHK engine talks with local control plane CHK engine will report interaction and repair result to the local control plane on its own node via dRPC (upcall) instead of to CHK leader. Correspondingly, local control plane will downcall to CHK engine to trigger CHK repair. Cleanup related useless logic. Test-tag: recovery Signed-off-by: Fan Yong --- src/chk/chk_common.c | 185 ++++++++++++++++++---- src/chk/chk_engine.c | 178 +-------------------- src/chk/chk_internal.h | 68 ++++---- src/chk/chk_leader.c | 271 ++------------------------------ src/chk/chk_rpc.c | 139 ---------------- src/chk/chk_srv.c | 34 +--- src/include/daos_srv/daos_chk.h | 2 +- src/mgmt/srv_chk.c | 2 +- 8 files changed, 212 insertions(+), 667 deletions(-) diff --git a/src/chk/chk_common.c b/src/chk/chk_common.c index cb5317fafe6..d3be5ab3ef8 100644 --- a/src/chk/chk_common.c +++ b/src/chk/chk_common.c @@ -185,7 +185,6 @@ btr_ops_t chk_pool_ops = { struct chk_pending_bundle { struct chk_instance *cpb_ins; d_list_t *cpb_pool_head; - d_list_t *cpb_rank_head; uuid_t cpb_uuid; d_rank_t cpb_rank; uint32_t cpb_class; @@ -247,11 +246,6 @@ chk_pending_alloc(struct btr_instance *tins, d_iov_t *key_iov, d_iov_t *val_iov, d_list_add_tail(&cpr->cpr_ins_link, &cpb->cpb_ins->ci_pending_list); - if (cpb->cpb_rank_head != NULL) - d_list_add_tail(&cpr->cpr_rank_link, cpb->cpb_rank_head); - else - D_INIT_LIST_HEAD(&cpr->cpr_rank_link); - rec->rec_off = umem_ptr2off(&tins->ti_umm, cpr); d_list_add_tail(&cpr->cpr_pool_link, cpb->cpb_pool_head); @@ -279,7 +273,6 @@ chk_pending_free(struct btr_instance *tins, struct btr_record *rec, void *args) rec->rec_off = UMOFF_NULL; d_list_del_init(&cpr->cpr_pool_link); - d_list_del_init(&cpr->cpr_rank_link); d_list_del_init(&cpr->cpr_ins_link); if (val_iov != NULL) { @@ -931,30 +924,9 @@ chk_pool_shard_cleanup(struct chk_instance *ins) } } -int -chk_pending_lookup(struct chk_instance *ins, uint64_t seq, struct chk_pending_rec **cpr) -{ - d_iov_t kiov; - d_iov_t riov; - int rc; - - d_iov_set(&riov, NULL, 0); - d_iov_set(&kiov, &seq, sizeof(seq)); - - ABT_rwlock_rdlock(ins->ci_abt_lock); - rc = dbtree_lookup(ins->ci_pending_hdl, &kiov, &riov); - ABT_rwlock_unlock(ins->ci_abt_lock); - if (rc == 0) - *cpr = (struct chk_pending_rec *)riov.iov_buf; - else - *cpr = NULL; - - return rc; -} - -int -chk_pending_add(struct chk_instance *ins, d_list_t *pool_head, d_list_t *rank_head, uuid_t uuid, - uint64_t seq, uint32_t rank, uint32_t cla, uint32_t option_nr, uint32_t *options, +static int +chk_pending_add(struct chk_instance *ins, d_list_t *pool_head, uuid_t uuid, uint64_t seq, + uint32_t rank, uint32_t cla, uint32_t option_nr, uint32_t *options, struct chk_pending_rec **cpr) { struct chk_pending_bundle rbund; @@ -967,7 +939,6 @@ chk_pending_add(struct chk_instance *ins, d_list_t *pool_head, d_list_t *rank_he uuid_copy(rbund.cpb_uuid, uuid); rbund.cpb_pool_head = pool_head; - rbund.cpb_rank_head = rank_head; rbund.cpb_ins = ins; rbund.cpb_seq = seq; rbund.cpb_rank = rank; @@ -1059,6 +1030,156 @@ chk_pending_wakeup(struct chk_instance *ins, struct chk_pending_rec *cpr) return rc; } +int +chk_report(struct chk_instance *ins, struct chk_report_unit *cru, uint64_t *seq, int *decision) +{ + struct chk_pending_rec *cpr = NULL; + struct chk_pool_rec *pool = NULL; + d_iov_t kiov; + d_iov_t riov; + int rc; + + CHK_IS_READY(ins); + + if (cru->cru_result == 0 && ins->ci_prop.cp_flags & CHK__CHECK_FLAG__CF_DRYRUN) + cru->cru_result = CHK__CHECK_RESULT__DRY_RUN; + + if (*seq == 0) { +new_seq: + *seq = chk_report_seq_gen(ins); + } + + D_INFO("Report on %u (%s) with seq " DF_X64 " class %u, action %u, %s, result %d\n", + cru->cru_rank, ins->ci_is_leader ? "leader" : "engine", *seq, cru->cru_cla, + cru->cru_act, cru->cru_msg, cru->cru_result); + + if (cru->cru_act == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT) { + if (cru->cru_pool == NULL) + D_GOTO(log, rc = -DER_INVAL); + + d_iov_set(&riov, NULL, 0); + d_iov_set(&kiov, cru->cru_pool, sizeof(uuid_t)); + rc = dbtree_lookup(ins->ci_pool_hdl, &kiov, &riov); + if (rc != 0) + goto log; + + pool = (struct chk_pool_rec *)riov.iov_buf; + + rc = chk_pending_add(ins, &pool->cpr_pending_list, *cru->cru_pool, *seq, + cru->cru_rank, cru->cru_cla, cru->cru_option_nr, + cru->cru_options, &cpr); + if (unlikely(rc == -DER_AGAIN)) + goto new_seq; + + if (rc != 0) + goto log; + } + + rc = chk_report_upcall(cru->cru_gen, *seq, cru->cru_cla, cru->cru_act, cru->cru_result, + cru->cru_rank, cru->cru_target, cru->cru_pool, cru->cru_pool_label, + cru->cru_cont, cru->cru_cont_label, cru->cru_obj, cru->cru_dkey, + cru->cru_akey, cru->cru_msg, cru->cru_option_nr, cru->cru_options, + cru->cru_detail_nr, cru->cru_details); + /* Check cpr->cpr_action for the case of "dmg check repair" by race. */ + if (rc == 0 && pool != NULL && + likely(cpr->cpr_action == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT)) + pool->cpr_bk.cb_pool_status = CHK__CHECK_POOL_STATUS__CPS_PENDING; + +log: + if (rc != 0) { + D_ERROR("Failed to handle report from rank %u (%s) with seq " DF_X64 ", class %u, " + "action %u, handle_rc %d, report_rc %d\n", + cru->cru_rank, ins->ci_is_leader ? "leader" : "engine", *seq, cru->cru_cla, + cru->cru_act, cru->cru_result, rc); + goto out; + } + + if (decision == NULL || cpr == NULL) + goto out; + + D_ASSERT(cpr->cpr_busy); + + D_INFO("Need interaction for class %u with seq " DF_X64 "\n", cru->cru_cla, *seq); + + ABT_mutex_lock(cpr->cpr_mutex); + +again: + if (cpr->cpr_action != CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT) { + *decision = cpr->cpr_action; + ABT_mutex_unlock(cpr->cpr_mutex); + goto out; + } + + if (!ins->ci_sched_running || ins->ci_sched_exiting || cpr->cpr_exiting) { + rc = 1; + ABT_mutex_unlock(cpr->cpr_mutex); + goto out; + } + + ABT_cond_wait(cpr->cpr_cond, cpr->cpr_mutex); + + goto again; + +out: + if ((rc != 0 || decision != NULL) && cpr != NULL) + chk_pending_destroy(ins, cpr); + + if (pool != NULL && pool->cpr_bk.cb_pool_status == CHK__CHECK_POOL_STATUS__CPS_PENDING && + d_list_empty(&pool->cpr_pending_list)) + pool->cpr_bk.cb_pool_status = CHK__CHECK_POOL_STATUS__CPS_CHECKING; + + return rc; +} + +int +chk_act_internal(struct chk_instance *ins, uint64_t seq, uint32_t act) +{ + struct chk_pending_rec *cpr = NULL; + int rc; + + CHK_IS_READY(ins); + + rc = chk_pending_del(ins, seq, &cpr); + if (rc == 0) { + /* The cpr will be destroyed by the waiter via chk_engine_report(). */ + D_ASSERT(cpr->cpr_busy); + + ABT_mutex_lock(cpr->cpr_mutex); + /* + * It is the control plane's duty to guarantee that the decision is a valid + * action from the report options. Otherwise, related inconsistency will be + * ignored. + */ + cpr->cpr_action = act; + ABT_cond_broadcast(cpr->cpr_cond); + ABT_mutex_unlock(cpr->cpr_mutex); + } + + return rc; +} + +int +chk_act(uint64_t seq, uint32_t act) +{ + int rc = -DER_INVAL; + + if (likely(act != CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT)) { + if (chk_report_seq_leader(seq)) + rc = chk_leader_act(seq, act); + else + rc = chk_engine_act(seq, act); + } + + D_CDEBUG(rc != 0 && rc != -DER_NONEXIST && rc != -DER_NO_HDL, DLOG_ERR, DLOG_INFO, + "CHK repair on rank %u, act %u, seq " DF_X64 ": " DF_RC "\n", dss_self_rank(), act, + seq, DP_RC(rc)); + + if (rc == -DER_NONEXIST || rc == -DER_NO_HDL) + rc = 0; + + return rc; +} + int chk_policy_refresh(uint32_t policy_nr, struct chk_policy *policies, struct chk_property *prop) { diff --git a/src/chk/chk_engine.c b/src/chk/chk_engine.c index f4c0be91d58..127e6a6bad5 100644 --- a/src/chk/chk_engine.c +++ b/src/chk/chk_engine.c @@ -87,8 +87,6 @@ enum chk_pm_status { CPS_TGT_DOWN, }; -static int chk_engine_report(struct chk_report_unit *cru, uint64_t *seq, int *decision); - static int chk_cont_hkey_size(void) { @@ -408,7 +406,7 @@ chk_engine_pm_orphan_ult(void *args) cru.cru_details = details; cru.cru_result = result; - rc = chk_engine_report(&cru, &seq, &decision); + rc = chk_report(ins, &cru, &seq, &decision); D_CDEBUG(result != 0 || rc != 0, DLOG_ERR, DLOG_INFO, DF_ENGINE @@ -634,7 +632,7 @@ chk_engine_pm_dangling_ult(void *args) cru.cru_details = details; cru.cru_result = result; - rc = chk_engine_report(&cru, &seq, &decision); + rc = chk_report(ins, &cru, &seq, &decision); D_CDEBUG(result != 0 || rc != 0, DLOG_ERR, DLOG_INFO, DF_ENGINE" detects dangling %s entry in pool map for pool " @@ -794,7 +792,7 @@ chk_engine_handle_unknown_ult(void *args) cru.cru_msg = msg; cru.cru_result = 0; - rc = chk_engine_report(&cru, &seq, NULL); + rc = chk_report(ins, &cru, &seq, NULL); if (ccr != NULL) D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, @@ -1102,7 +1100,7 @@ chk_engine_bad_pool_label(struct chk_pool_rec *cpr, struct ds_pool_svc *svc) cru.cru_msg = msg; cru.cru_result = result; - rc = chk_engine_report(&cru, &seq, NULL); + rc = chk_report(ins, &cru, &seq, NULL); D_CDEBUG(result != 0 || rc != 0, DLOG_ERR, DLOG_INFO, DF_ENGINE" detects corrupted label %s (MS) vs %s (PS) for pool " @@ -1332,7 +1330,7 @@ chk_engine_cont_orphan_ult(void *args) cru.cru_details = details; cru.cru_result = result; - rc = chk_engine_report(&cru, &seq, &decision); + rc = chk_report(ins, &cru, &seq, &decision); D_CDEBUG(result != 0 || rc != 0, DLOG_ERR, DLOG_INFO, DF_ENGINE" detects orphan container " @@ -1684,7 +1682,7 @@ chk_engine_cont_set_label_ult(void *args) cru.cru_details = details; cru.cru_result = result; - rc = chk_engine_report(&cru, &seq, &decision); + rc = chk_report(ins, &cru, &seq, &decision); D_CDEBUG(result != 0 || rc != 0, DLOG_ERR, DLOG_INFO, DF_ENGINE" detects inconsistent container label for "DF_UUIDF"/"DF_UUIDF @@ -2935,62 +2933,10 @@ chk_engine_mark_rank_dead(uint64_t gen, d_rank_t rank, uint32_t version) return rc; } -static int -chk_engine_act_internal(struct chk_instance *ins, uint64_t seq, uint32_t act) -{ - struct chk_pending_rec *cpr = NULL; - int rc; - - rc = chk_pending_del(ins, seq, &cpr); - if (rc == 0) { - /* The cpr will be destroyed by the waiter via chk_engine_report(). */ - D_ASSERT(cpr->cpr_busy); - - ABT_mutex_lock(cpr->cpr_mutex); - /* - * It is the control plane's duty to guarantee that the decision is a valid - * action from the report options. Otherwise, related inconsistency will be - * ignored. - */ - cpr->cpr_action = act; - ABT_cond_broadcast(cpr->cpr_cond); - ABT_mutex_unlock(cpr->cpr_mutex); - } - - D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, - DF_ENGINE" on rank %u takes action for seq "DF_X64" with action %u: %d\n", - DP_ENGINE(ins), dss_self_rank(), seq, act, rc); - - return rc; -} - int -chk_engine_act(uint64_t gen, uint64_t seq, uint32_t act) +chk_engine_act(uint64_t seq, uint32_t act) { - struct chk_instance *ins = chk_engine; - int rc; - - CHK_IS_READY(ins); - - if (ins->ci_bk.cb_gen != gen) - D_GOTO(out, rc = -DER_NOTAPPLICABLE); - - /* The admin may input the wrong option, not acceptable. */ - if (unlikely(act == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT)) { - D_ERROR("%u is not acceptable for interaction decision.\n", act); - D_GOTO(out, rc = -DER_INVAL); - } - - rc = chk_engine_act_internal(ins, seq, act); - if (rc == -DER_NONEXIST || rc == -DER_NO_HDL) - rc = 0; - -out: - D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, - DF_ENGINE " on rank %u takes act %u, seq " DF_X64 ", gen " DF_X64 ": " DF_RC "\n", - DP_ENGINE(ins), dss_self_rank(), act, seq, gen, DP_RC(rc)); - - return rc; + return chk_act_internal(chk_engine, seq, act); } static int @@ -3409,114 +3355,6 @@ chk_engine_set_policy(uint64_t gen, uint32_t policy_nr, struct chk_policy *polic return rc == -DER_NOTAPPLICABLE ? 0 : rc; } -/* - * \return Positive value if interaction is interrupted, such as check stop. - * Zero on success. - * Negative value if error. - */ -static int -chk_engine_report(struct chk_report_unit *cru, uint64_t *seq, int *decision) -{ - struct chk_instance *ins = chk_engine; - struct chk_pending_rec *cpr = NULL; - struct chk_pool_rec *pool = NULL; - d_iov_t kiov; - d_iov_t riov; - int rc; - - D_ASSERT(cru->cru_pool != NULL); - - if (*seq == 0) { - -new_seq: - *seq = chk_report_seq_gen(ins); - } - - if (cru->cru_act == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT) { - d_iov_set(&riov, NULL, 0); - d_iov_set(&kiov, cru->cru_pool, sizeof(uuid_t)); - rc = dbtree_lookup(ins->ci_pool_hdl, &kiov, &riov); - if (rc != 0) - goto log; - - pool = (struct chk_pool_rec *)riov.iov_buf; - - rc = chk_pending_add(ins, &pool->cpr_pending_list, NULL, *cru->cru_pool, *seq, - cru->cru_rank, cru->cru_cla, cru->cru_option_nr, - cru->cru_options, &cpr); - if (unlikely(rc == -DER_AGAIN)) - goto new_seq; - - if (rc != 0) - goto log; - } - - rc = chk_report_remote(ins->ci_prop.cp_leader, ins->ci_bk.cb_gen, cru->cru_cla, - cru->cru_act, cru->cru_result, cru->cru_rank, cru->cru_target, - cru->cru_pool, cru->cru_pool_label, cru->cru_cont, - cru->cru_cont_label, cru->cru_obj, cru->cru_dkey, - cru->cru_akey, cru->cru_msg, cru->cru_option_nr, cru->cru_options, - cru->cru_detail_nr, cru->cru_details, *seq); - if (unlikely(rc == -DER_AGAIN)) { - D_ASSERT(cru->cru_act == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT); - D_ASSERT(cpr != NULL); - - chk_pending_destroy(ins, cpr); - cpr = NULL; - - goto new_seq; - } - - /* Check cpr->cpr_action for the case of "dmg check repair" by race. */ - if (rc == 0 && pool != NULL && - likely(cpr->cpr_action == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT)) - pool->cpr_bk.cb_pool_status = CHK__CHECK_POOL_STATUS__CPS_PENDING; - -log: - DL_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, rc, - DF_ENGINE " on rank %u report with class %u, action %u, seq " DF_X64 ", %s, " - "handle_rc %d, report_rc %d", - DP_ENGINE(ins), cru->cru_rank, cru->cru_cla, cru->cru_act, *seq, cru->cru_msg, - cru->cru_result, rc); - - if (rc != 0 || cpr == NULL) - goto out; - - D_ASSERT(cpr->cpr_busy); - - D_INFO(DF_ENGINE" on rank %u need interaction for class %u\n", - DP_ENGINE(ins), cru->cru_rank, cru->cru_cla); - - ABT_mutex_lock(cpr->cpr_mutex); - -again: - if (cpr->cpr_action != CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT) { - *decision = cpr->cpr_action; - ABT_mutex_unlock(cpr->cpr_mutex); - goto out; - } - - if (!ins->ci_sched_running || ins->ci_sched_exiting || cpr->cpr_exiting) { - rc = 1; - ABT_mutex_unlock(cpr->cpr_mutex); - goto out; - } - - ABT_cond_wait(cpr->cpr_cond, cpr->cpr_mutex); - - goto again; - -out: - if (cpr != NULL) - chk_pending_destroy(ins, cpr); - - if (pool != NULL && pool->cpr_bk.cb_pool_status == CHK__CHECK_POOL_STATUS__CPS_PENDING && - d_list_empty(&pool->cpr_pending_list)) - pool->cpr_bk.cb_pool_status = CHK__CHECK_POOL_STATUS__CPS_CHECKING; - - return rc; -} - int chk_engine_notify(struct chk_iv *iv) { diff --git a/src/chk/chk_internal.h b/src/chk/chk_internal.h index debb3db3127..8c8eae0d6e5 100644 --- a/src/chk/chk_internal.h +++ b/src/chk/chk_internal.h @@ -45,7 +45,7 @@ X(CHK_MARK, \ 0, &CQF_chk_mark, ds_chk_mark_hdlr, &chk_mark_co_ops), \ X(CHK_ACT, \ - 0, &CQF_chk_act, ds_chk_act_hdlr, &chk_act_co_ops), \ + 0, &CQF_chk_act, ds_chk_act_hdlr, NULL), \ X(CHK_CONT_LIST, \ 0, &CQF_chk_cont_list, ds_chk_cont_list_hdlr, &chk_cont_list_co_ops), \ X(CHK_POOL_START, \ @@ -156,7 +156,7 @@ CRT_RPC_DECLARE(chk_query, DAOS_ISEQ_CHK_QUERY, DAOS_OSEQ_CHK_QUERY); CRT_RPC_DECLARE(chk_mark, DAOS_ISEQ_CHK_MARK, DAOS_OSEQ_CHK_MARK); /* - * CHK_ACT: + * CHK_ACT: obsolete - DAOS-18674. * From check leader to check engine to execute the admin specified repair action for former * reported inconsistency under interaction mode. */ @@ -228,7 +228,7 @@ CRT_RPC_DECLARE(chk_pool_start, DAOS_ISEQ_CHK_POOL_START, DAOS_OSEQ_CHK_POOL_STA CRT_RPC_DECLARE(chk_pool_mbs, DAOS_ISEQ_CHK_POOL_MBS, DAOS_OSEQ_CHK_POOL_MBS); /* - * CHK_REPORT: + * CHK_REPORT: obsolete - DAOS-18674. * From check engine to check leader to report the inconsistency and related repair action * and result. It can require to interact with the admin to make decision for how to handle * the inconsistency. @@ -318,15 +318,11 @@ CRT_RPC_DECLARE(chk_set_policy, DAOS_ISEQ_CHK_SET_POLICY, DAOS_OSEQ_CHK_SET_POLI #define CHK_LEADER_RANK (uint32_t)(-1) /* - * Keep the lowest 20-bits of DAOS engine rank in the check report sequence. - * If the count of DAOS engines exceeds 2 ^ 20, then different check engines - * may generate the same sequence for different check reports. Such conflict - * is not fatal for non-interaction report. As for interaction report, check - * leader will detect such report sequqnce conflict and ask related engine(s) - * to generate new sequence(s). + * The highest bit in CHK report sequence is reserved, the next 23 bits are for DAOS engine rank. + * The others for detailed sequence number. */ -#define CHK_REPORT_RANK_BIT 40 -#define CHK_REPORT_SEQ_MASK ((1ULL << CHK_REPORT_RANK_BIT) - 1) +#define CHK_REPORT_SEQ_BIT 40 +#define CHK_REPORT_SEQ_MASK ((1ULL << CHK_REPORT_SEQ_BIT) - 1) #define CHK_BTREE_ORDER 16 @@ -645,8 +641,6 @@ struct chk_pool_rec { struct chk_pending_rec { /* Link into chk_pool_rec::cpr_pending_list. */ d_list_t cpr_pool_link; - /* Link into chk_rank_rec::crr_pending_list. */ - d_list_t cpr_rank_link; /* Link into chk_instance::ci_pending_list or chk_instance::ci_interaction_filter_list. */ d_list_t cpr_ins_link; uuid_t cpr_uuid; @@ -654,7 +648,7 @@ struct chk_pending_rec { d_rank_t cpr_rank; uint32_t cpr_class; uint32_t cpr_action; - uint32_t cpr_busy : 1, cpr_exiting : 1, cpr_on_leader : 1; + uint32_t cpr_busy : 1, cpr_exiting : 1; uint32_t cpr_option_nr; /* Currently, support at most three options, can be extended if necessary in future. */ uint32_t cpr_options[CHK_INTERACT_OPTION_MAX]; @@ -699,13 +693,12 @@ struct chk_dead_rank { extern struct crt_proto_format chk_proto_fmt; -extern struct crt_corpc_ops chk_start_co_ops; -extern struct crt_corpc_ops chk_stop_co_ops; -extern struct crt_corpc_ops chk_query_co_ops; -extern struct crt_corpc_ops chk_mark_co_ops; -extern struct crt_corpc_ops chk_act_co_ops; -extern struct crt_corpc_ops chk_cont_list_co_ops; -extern struct crt_corpc_ops chk_pool_start_co_ops; +extern struct crt_corpc_ops chk_start_co_ops; +extern struct crt_corpc_ops chk_stop_co_ops; +extern struct crt_corpc_ops chk_query_co_ops; +extern struct crt_corpc_ops chk_mark_co_ops; +extern struct crt_corpc_ops chk_cont_list_co_ops; +extern struct crt_corpc_ops chk_pool_start_co_ops; extern struct crt_corpc_ops chk_set_policy_co_ops; extern btr_ops_t chk_pool_ops; @@ -752,16 +745,14 @@ int chk_pool_add_shard(daos_handle_t hdl, d_list_t *head, uuid_t uuid, d_rank_t void chk_pool_shard_cleanup(struct chk_instance *ins); -int chk_pending_lookup(struct chk_instance *ins, uint64_t seq, struct chk_pending_rec **cpr); - -int chk_pending_add(struct chk_instance *ins, d_list_t *pool_head, d_list_t *rank_head, uuid_t uuid, - uint64_t seq, uint32_t rank, uint32_t cla, uint32_t option_nr, - uint32_t *options, struct chk_pending_rec **cpr); - int chk_pending_del(struct chk_instance *ins, uint64_t seq, struct chk_pending_rec **cpr); int chk_pending_wakeup(struct chk_instance *ins, struct chk_pending_rec *cpr); +int chk_act_internal(struct chk_instance *ins, uint64_t seq, uint32_t act); + +int chk_report(struct chk_instance *ins, struct chk_report_unit *cru, uint64_t *seq, int *decision); + int chk_policy_refresh(uint32_t policy_nr, struct chk_policy *policies, struct chk_property *prop); int chk_prop_prepare(d_rank_t leader, uint32_t flags, uint32_t policy_nr, @@ -793,7 +784,7 @@ int chk_engine_query(uint64_t gen, int pool_nr, uuid_t pools[], uint32_t *ins_st int chk_engine_mark_rank_dead(uint64_t gen, d_rank_t rank, uint32_t version); -int chk_engine_act(uint64_t gen, uint64_t seq, uint32_t act); +int chk_engine_act(uint64_t seq, uint32_t act); int chk_engine_cont_list(uint64_t gen, uuid_t pool_uuid, uuid_t **conts, uint32_t *count); @@ -838,7 +829,7 @@ bool chk_is_on_leader(uint64_t gen, d_rank_t leader, bool known_leader); struct ds_iv_ns *chk_leader_get_iv_ns(void); -int chk_leader_report(struct chk_report_unit *cru, uint64_t *seq, int *decision); +int chk_leader_act(uint64_t seq, uint32_t act); int chk_leader_notify(struct chk_iv *iv); @@ -868,9 +859,6 @@ int chk_query_remote(d_rank_list_t *rank_list, uint64_t gen, int pool_nr, uuid_t int chk_mark_remote(d_rank_list_t *rank_list, uint64_t gen, d_rank_t rank, uint32_t version); -int chk_act_remote(d_rank_list_t *rank_list, uint64_t gen, uint64_t seq, uint32_t cla, - uint32_t act, d_rank_t rank); - int chk_cont_list_remote(struct ds_pool *pool, uint64_t gen, chk_co_rpc_cb_t list_cb, void *args); int chk_pool_start_remote(d_rank_list_t *rank_list, uint64_t gen, uuid_t uuid, uint32_t phase, @@ -880,12 +868,6 @@ int chk_pool_mbs_remote(d_rank_t rank, uint32_t phase, uint64_t gen, uuid_t uuid uint64_t seq, uint32_t flags, uint32_t mbs_nr, struct chk_pool_mbs *mbs_array, int *svc_rc, struct rsvc_hint *svc_hint); -int chk_report_remote(d_rank_t leader, uint64_t gen, uint32_t cla, uint32_t act, int result, - d_rank_t rank, uint32_t target, uuid_t *pool, char *pool_label, - uuid_t *cont, char *cont_label, daos_unit_oid_t *obj, daos_key_t *dkey, - daos_key_t *akey, char *msg, uint32_t option_nr, uint32_t *options, - uint32_t detail_nr, d_sg_list_t *details, uint64_t seq); - int chk_rejoin_remote(d_rank_t leader, uint64_t gen, d_rank_t rank, uuid_t iv_uuid, uint32_t *flags, uint32_t *ns_ver, uint32_t *pool_nr, uuid_t **pools, d_rank_list_t **ranks); @@ -1000,7 +982,6 @@ static inline void chk_pending_destroy(struct chk_instance *ins, struct chk_pending_rec *cpr) { if (d_list_empty(&cpr->cpr_pool_link)) { - D_ASSERT(d_list_empty(&cpr->cpr_rank_link)); D_ASSERT(d_list_empty(&cpr->cpr_ins_link)); if (cpr->cpr_cond != ABT_COND_NULL) @@ -1269,7 +1250,7 @@ chk_report_seq_init(struct chk_instance *ins) else myrank = dss_self_rank(); - ins->ci_seq = (myrank << CHK_REPORT_RANK_BIT) | (d_hlc_get() >> (64 - CHK_REPORT_RANK_BIT)); + ins->ci_seq = (myrank << CHK_REPORT_SEQ_BIT) | (d_hlc_get() & CHK_REPORT_SEQ_MASK); /* Clear the highest bit. */ ins->ci_seq &= ~(1ULL << 63); @@ -1287,6 +1268,13 @@ chk_report_seq_gen(struct chk_instance *ins) return ins->ci_seq; } +static inline bool +chk_report_seq_leader(uint64_t seq) +{ + return (seq & ~CHK_REPORT_SEQ_MASK) == + (((uint64_t)CHK_LEADER_RANK << CHK_REPORT_SEQ_BIT) & ~(1ULL << 63)); +} + static inline void chk_uuid_unparse(struct chk_instance *ins, const uuid_t uuid, char *uuid_str) { diff --git a/src/chk/chk_leader.c b/src/chk/chk_leader.c index 18be52d0ac0..3a43f486b44 100644 --- a/src/chk/chk_leader.c +++ b/src/chk/chk_leader.c @@ -39,12 +39,10 @@ struct chk_query_args { struct chk_rank_rec { /* Link into chk_instance::ci_rank_list. */ - d_list_t crr_link; - /* The list of chk_pending_rec. */ - d_list_t crr_pending_list; - d_rank_t crr_rank; - uint32_t crr_phase; - struct chk_instance *crr_ins; + d_list_t crr_link; + d_rank_t crr_rank; + uint32_t crr_phase; + struct chk_instance *crr_ins; }; struct chk_rank_bundle { @@ -81,7 +79,6 @@ chk_rank_alloc(struct btr_instance *tins, d_iov_t *key_iov, d_iov_t *val_iov, if (crr == NULL) D_GOTO(out, rc = -DER_NOMEM); - D_INIT_LIST_HEAD(&crr->crr_pending_list); crr->crr_rank = crb->crb_rank; crr->crr_phase = crb->crb_phase; crr->crr_ins = crb->crb_ins; @@ -103,16 +100,10 @@ chk_rank_free(struct btr_instance *tins, struct btr_record *rec, void *args) rec->rec_off = UMOFF_NULL; d_list_del_init(&crr->crr_link); - if (val_iov != NULL) { + if (val_iov != NULL) d_iov_set(val_iov, crr, sizeof(*crr)); - } else { - /* - * This only happens when destroy the rank tree. At that time, - * the pending records tree has already been destroyed. - */ - D_ASSERT(d_list_empty(&crr->crr_pending_list)); + else D_FREE(crr); - } return 0; } @@ -179,34 +170,16 @@ chk_leader_get_iv_ns(void) static int chk_rank_del(struct chk_instance *ins, d_rank_t rank) { - struct chk_rank_rec *crr; - struct chk_pending_rec *cpr; - d_iov_t riov; - d_iov_t kiov; - int rc; - int rc1; + d_iov_t riov; + d_iov_t kiov; + int rc; d_iov_set(&riov, NULL, 0); d_iov_set(&kiov, &rank, sizeof(rank)); rc = dbtree_delete(ins->ci_rank_hdl, BTR_PROBE_EQ, &kiov, &riov); - if (rc != 0) - D_GOTO(out, rc = ((rc == -DER_NONEXIST || rc == -DER_NO_HDL) ? 0 : rc)); - - crr = (struct chk_rank_rec *)riov.iov_buf; - if (d_list_empty(&crr->crr_pending_list)) - goto out; - - /* Cleanup all pending records belong to this rank. */ - ABT_rwlock_wrlock(ins->ci_abt_lock); - while ((cpr = d_list_pop_entry(&crr->crr_pending_list, struct chk_pending_rec, - cpr_rank_link)) != NULL) { - rc1 = chk_pending_wakeup(ins, cpr); - if (rc1 != 0 && rc == 0) - rc = rc1; - } - ABT_rwlock_unlock(ins->ci_abt_lock); + if (rc == -DER_NONEXIST || rc == -DER_NO_HDL) + rc = 0; -out: return rc; } @@ -416,7 +389,7 @@ chk_leader_fail_pool(struct chk_pool_rec *cpr, int result) cru.cru_msg = "Some engine failed to report information for pool.\n"; cru.cru_result = result; - rc = chk_leader_report(&cru, &seq, NULL); + rc = chk_report(ins, &cru, &seq, NULL); D_WARN(DF_LEADER" some engine failed to report information for pool " DF_UUIDF", action %u, seq "DF_X64", remote_rc %d, report_rc %d\n", @@ -651,7 +624,7 @@ chk_leader_dangling_pool(struct chk_pool_rec *cpr) cru.cru_details = details; cru.cru_result = result; - rc = chk_leader_report(&cru, &seq, &decision); + rc = chk_report(ins, &cru, &seq, &decision); D_CDEBUG(result != 0 || rc < 0, DLOG_ERR, DLOG_INFO, DF_LEADER" detects dangling pool "DF_UUIDF", action %u (%s), seq " @@ -851,7 +824,7 @@ chk_leader_orphan_pool(struct chk_pool_rec *cpr) cru.cru_details = details; cru.cru_result = result; - rc = chk_leader_report(&cru, &seq, &decision); + rc = chk_report(ins, &cru, &seq, &decision); D_CDEBUG(result != 0 || rc < 0, DLOG_ERR, DLOG_INFO, DF_LEADER" detects orphan pool "DF_UUIDF", action %u (%s), seq " @@ -1155,7 +1128,7 @@ chk_leader_no_quorum_pool(struct chk_pool_rec *cpr) cru.cru_details = details; cru.cru_result = result; - rc = chk_leader_report(&cru, &seq, &decision); + rc = chk_report(ins, &cru, &seq, &decision); D_CDEBUG(result != 0 || rc < 0, DLOG_ERR, DLOG_INFO, DF_LEADER" detects corrupted pool "DF_UUIDF", action %u (%s), seq " @@ -1567,7 +1540,7 @@ chk_leader_handle_pool_label(struct chk_pool_rec *cpr, struct ds_pool_clue *clue cru.cru_details = details; cru.cru_result = result; - rc = chk_leader_report(&cru, &seq, &decision); + rc = chk_report(ins, &cru, &seq, &decision); D_CDEBUG(result != 0 || rc < 0, DLOG_ERR, DLOG_INFO, DF_LEADER" detects corrupted label for pool "DF_UUIDF", action %u (%s), seq " @@ -3460,86 +3433,10 @@ chk_leader_prop(chk_prop_cb_t prop_cb, void *buf) return prop_cb(buf, prop->cp_policies, CHK_POLICY_MAX - 1, prop->cp_flags); } -static int -chk_leader_act_internal(struct chk_instance *ins, uint64_t seq, uint32_t act) -{ - struct chk_pending_rec *pending = NULL; - struct chk_pool_rec *pool = NULL; - d_iov_t kiov; - d_iov_t riov; - int rc; - - rc = chk_pending_lookup(ins, seq, &pending); - if (rc != 0) - goto out; - - if (pending->cpr_on_leader) { - ABT_mutex_lock(pending->cpr_mutex); - /* - * It is the control plane's duty to guarantee that the decision is a valid - * action from the report options. Otherwise, related inconsistency will be ignored. - */ - pending->cpr_action = act; - ABT_cond_broadcast(pending->cpr_cond); - ABT_mutex_unlock(pending->cpr_mutex); - chk_pending_del(ins, seq, &pending); - } else { - d_iov_set(&riov, NULL, 0); - d_iov_set(&kiov, pending->cpr_uuid, sizeof(uuid_t)); - rc = dbtree_lookup(ins->ci_pool_hdl, &kiov, &riov); - if (rc == 0) - pool = (struct chk_pool_rec *)riov.iov_buf; - - rc = chk_act_remote(ins->ci_ranks, ins->ci_bk.cb_gen, seq, pending->cpr_class, act, - pending->cpr_rank); - if (rc == 0) { - chk_pending_destroy(ins, pending); - - if (pool != NULL && - pool->cpr_bk.cb_pool_status == CHK__CHECK_POOL_STATUS__CPS_PENDING && - d_list_empty(&pool->cpr_pending_list)) - pool->cpr_bk.cb_pool_status = CHK__CHECK_POOL_STATUS__CPS_CHECKING; - } - } - -out: - D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, - DF_LEADER" takes action for report with seq "DF_X64", action %u: "DF_RC"\n", - DP_LEADER(ins), seq, act, DP_RC(rc)); - - return rc; -} - int chk_leader_act(uint64_t seq, uint32_t act) { - struct chk_instance *ins = chk_leader; - struct chk_bookmark *cbk = &ins->ci_bk; - int rc; - - CHK_IS_READY(ins); - - if (cbk->cb_magic != CHK_BK_MAGIC_LEADER) - D_GOTO(out, rc = -DER_NOTLEADER); - - /* Tell control plane that no check instance is running via "-DER_NOTAPPLICABLE". */ - if (cbk->cb_ins_status != CHK__CHECK_INST_STATUS__CIS_RUNNING) - D_GOTO(out, rc = -DER_NOTAPPLICABLE); - - /* The admin may input the wrong option, not acceptable. */ - if (unlikely(act == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT)) { - D_ERROR("%u is not acceptable for interaction decision.\n", act); - D_GOTO(out, rc = -DER_INVAL); - } - - rc = chk_leader_act_internal(ins, seq, act); - -out: - D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, - DF_LEADER " takes action for report with seq " DF_X64 ", action %u: %d\n", - DP_LEADER(ins), seq, act, rc); - - return rc; + return chk_act_internal(chk_leader, seq, act); } int @@ -3585,140 +3482,6 @@ chk_leader_set_policy(uint32_t policy_nr, struct chk_policy *policies) return rc == -DER_NOTAPPLICABLE ? 0 : rc; } -/* - * \return Positive value if interaction is interrupted, such as check stop. - * Zero on success. - * Negative value if error. - */ -int -chk_leader_report(struct chk_report_unit *cru, uint64_t *seq, int *decision) -{ - struct chk_instance *ins = chk_leader; - struct chk_bookmark *cbk = &ins->ci_bk; - struct chk_pending_rec *cpr = NULL; - struct chk_pool_rec *pool = NULL; - struct chk_rank_rec *crr = NULL; - d_iov_t kiov; - d_iov_t riov; - int rc; - - CHK_IS_READY(ins); - - if (cbk->cb_magic != CHK_BK_MAGIC_LEADER) - D_GOTO(out, rc = -DER_NOTLEADER); - - /* Tell check engine that check leader is not running via "-DER_NOTAPPLICABLE". */ - if (cbk->cb_ins_status != CHK__CHECK_INST_STATUS__CIS_RUNNING) - D_GOTO(out, rc = -DER_NOTAPPLICABLE); - - if (cru->cru_result == 0 && ins->ci_prop.cp_flags & CHK__CHECK_FLAG__CF_DRYRUN) - cru->cru_result = CHK__CHECK_RESULT__DRY_RUN; - - if (*seq == 0) { - -new_seq: - *seq = chk_report_seq_gen(ins); - } - - D_INFO(DF_LEADER " handle %s report from rank %u with seq " DF_X64 " class %u, action %u, " - "%s, result %d\n", - DP_LEADER(ins), decision != NULL ? "local" : "remote", cru->cru_rank, *seq, - cru->cru_cla, cru->cru_act, cru->cru_msg, cru->cru_result); - - if (cru->cru_act == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT) { - if (cru->cru_pool == NULL) - D_GOTO(log, rc = -DER_INVAL); - - d_iov_set(&riov, NULL, 0); - d_iov_set(&kiov, cru->cru_pool, sizeof(uuid_t)); - rc = dbtree_lookup(ins->ci_pool_hdl, &kiov, &riov); - if (rc != 0) - goto log; - - pool = (struct chk_pool_rec *)riov.iov_buf; - - if (decision == NULL) { - d_iov_set(&riov, NULL, 0); - d_iov_set(&kiov, &cru->cru_rank, sizeof(cru->cru_rank)); - rc = dbtree_lookup(ins->ci_rank_hdl, &kiov, &riov); - if (rc != 0) - goto log; - - crr = (struct chk_rank_rec *)riov.iov_buf; - } - - rc = chk_pending_add(ins, &pool->cpr_pending_list, - crr != NULL ? &crr->crr_pending_list : NULL, *cru->cru_pool, - *seq, cru->cru_rank, cru->cru_cla, cru->cru_option_nr, - cru->cru_options, &cpr); - if (decision != NULL) { - if (unlikely(rc == -DER_AGAIN)) - goto new_seq; - - cpr->cpr_on_leader = 1; - } - - if (rc != 0) - goto log; - } - - rc = chk_report_upcall(cru->cru_gen, *seq, cru->cru_cla, cru->cru_act, cru->cru_result, - cru->cru_rank, cru->cru_target, cru->cru_pool, cru->cru_pool_label, - cru->cru_cont, cru->cru_cont_label, cru->cru_obj, cru->cru_dkey, - cru->cru_akey, cru->cru_msg, cru->cru_option_nr, cru->cru_options, - cru->cru_detail_nr, cru->cru_details); - /* Check cpr->cpr_action for the case of "dmg check repair" by race. */ - if (rc == 0 && pool != NULL && - likely(cpr->cpr_action == CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT)) - pool->cpr_bk.cb_pool_status = CHK__CHECK_POOL_STATUS__CPS_PENDING; - -log: - if (rc != 0) { - D_ERROR(DF_LEADER" failed to handle %s report from rank %u with seq " - DF_X64", class %u, action %u, handle_rc %d, report_rc %d\n", - DP_LEADER(ins), decision != NULL ? "local" : "remote", cru->cru_rank, *seq, - cru->cru_cla, cru->cru_act, cru->cru_result, rc); - goto out; - } - - if (decision == NULL || cpr == NULL) - goto out; - - D_ASSERT(cpr->cpr_busy); - - D_INFO(DF_LEADER" need interaction for class %u with seq "DF_X64"\n", - DP_LEADER(ins), cru->cru_cla, *seq); - - ABT_mutex_lock(cpr->cpr_mutex); - -again: - if (cpr->cpr_action != CHK__CHECK_INCONSIST_ACTION__CIA_INTERACT) { - *decision = cpr->cpr_action; - ABT_mutex_unlock(cpr->cpr_mutex); - goto out; - } - - if (!ins->ci_sched_running || ins->ci_sched_exiting || cpr->cpr_exiting) { - rc = 1; - ABT_mutex_unlock(cpr->cpr_mutex); - goto out; - } - - ABT_cond_wait(cpr->cpr_cond, cpr->cpr_mutex); - - goto again; - -out: - if ((rc != 0 || decision != NULL) && cpr != NULL) - chk_pending_destroy(ins, cpr); - - if (pool != NULL && pool->cpr_bk.cb_pool_status == CHK__CHECK_POOL_STATUS__CPS_PENDING && - d_list_empty(&pool->cpr_pending_list)) - pool->cpr_bk.cb_pool_status = CHK__CHECK_POOL_STATUS__CPS_CHECKING; - - return rc; -} - int chk_leader_notify(struct chk_iv *iv) { diff --git a/src/chk/chk_rpc.c b/src/chk/chk_rpc.c index e250936dfc2..47a574f60bf 100644 --- a/src/chk/chk_rpc.c +++ b/src/chk/chk_rpc.c @@ -324,24 +324,6 @@ chk_mark_aggregator(crt_rpc_t *source, crt_rpc_t *result, void *priv) return 0; } -static int -chk_act_aggregator(crt_rpc_t *source, crt_rpc_t *result, void *priv) -{ - struct chk_act_in *in_source = crt_req_get(source); - struct chk_act_out *out_source = crt_reply_get(source); - struct chk_act_out *out_result = crt_reply_get(result); - - if (out_source->cao_status != 0) { - D_ERROR("Failed to check act with gen "DF_X64": "DF_RC"\n", - in_source->cai_gen, DP_RC(out_source->cao_status)); - - if (out_result->cao_status == 0) - out_result->cao_status = out_source->cao_status; - } - - return 0; -} - static int chk_cont_list_aggregator(crt_rpc_t *source, crt_rpc_t *result, void *priv) { @@ -466,11 +448,6 @@ struct crt_corpc_ops chk_mark_co_ops = { .co_pre_forward = NULL, }; -struct crt_corpc_ops chk_act_co_ops = { - .co_aggregate = chk_act_aggregator, - .co_pre_forward = NULL, -}; - struct crt_corpc_ops chk_cont_list_co_ops = { .co_aggregate = chk_cont_list_aggregator, .co_pre_forward = NULL, @@ -762,44 +739,6 @@ chk_mark_remote(d_rank_list_t *rank_list, uint64_t gen, d_rank_t rank, uint32_t return rc; } -int -chk_act_remote(d_rank_list_t *rank_list, uint64_t gen, uint64_t seq, uint32_t cla, uint32_t act, - d_rank_t rank) -{ - crt_rpc_t *req = NULL; - struct chk_act_in *cai; - struct chk_act_out *cao; - int rc; - - rc = chk_sg_rpc_prepare(rank, CHK_ACT, &req); - if (rc != 0) - goto out; - - cai = crt_req_get(req); - cai->cai_gen = gen; - cai->cai_seq = seq; - cai->cai_cla = cla; - cai->cai_act = act; - cai->cai_flags = 0; - - rc = dss_rpc_send(req); - if (rc != 0) - goto out; - - cao = crt_reply_get(req); - rc = cao->cao_status; - -out: - if (req != NULL) - crt_req_decref(req); - - D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, - "Rank %u take action for DAOS check with gen "DF_X64", seq "DF_X64": "DF_RC"\n", - rank, gen, seq, DP_RC(rc)); - - return rc; -} - int chk_cont_list_remote(struct ds_pool *pool, uint64_t gen, chk_co_rpc_cb_t list_cb, void *args) { @@ -939,84 +878,6 @@ chk_pool_mbs_remote(d_rank_t rank, uint32_t phase, uint64_t gen, uuid_t uuid, ch return rc; } -int chk_report_remote(d_rank_t leader, uint64_t gen, uint32_t cla, uint32_t act, int result, - d_rank_t rank, uint32_t target, uuid_t *pool, char *pool_label, uuid_t *cont, - char *cont_label, daos_unit_oid_t *obj, daos_key_t *dkey, daos_key_t *akey, - char *msg, uint32_t option_nr, uint32_t *options, uint32_t detail_nr, - d_sg_list_t *details, uint64_t seq) -{ - crt_rpc_t *req = NULL; - struct chk_report_in *cri; - struct chk_report_out *cro; - int rc; - - rc = chk_sg_rpc_prepare(leader, CHK_REPORT, &req); - if (rc != 0) - goto out; - - cri = crt_req_get(req); - cri->cri_gen = gen; - cri->cri_ics_class = cla; - cri->cri_ics_action = act; - cri->cri_ics_result = result; - cri->cri_rank = rank; - cri->cri_target = target; - cri->cri_seq = seq; - - if (pool != NULL) - uuid_copy(cri->cri_pool, *pool); - else - memset(cri->cri_pool, 0, sizeof(uuid_t)); - - cri->cri_pool_label = pool_label; - - if (cont != NULL) - uuid_copy(cri->cri_cont, *cont); - else - memset(cri->cri_cont, 0, sizeof(uuid_t)); - - cri->cri_cont_label = cont_label; - - if (obj != NULL) - cri->cri_obj = *obj; - else - memset(&cri->cri_obj, 0, sizeof(cri->cri_obj)); - - if (dkey != NULL) - cri->cri_dkey = *dkey; - else - memset(&cri->cri_dkey, 0, sizeof(cri->cri_dkey)); - - if (akey != NULL) - cri->cri_akey = *akey; - else - memset(&cri->cri_akey, 0, sizeof(cri->cri_akey)); - - cri->cri_msg = msg; - cri->cri_options.ca_count = option_nr; - cri->cri_options.ca_arrays = options; - cri->cri_details.ca_count = detail_nr; - cri->cri_details.ca_arrays = details; - - rc = dss_rpc_send(req); - if (rc != 0) - goto out; - - cro = crt_reply_get(req); - rc = cro->cro_status; - -out: - if (req != NULL) - crt_req_decref(req); - - D_CDEBUG(rc != 0, DLOG_ERR, DLOG_INFO, - "Rank %u report DAOS check to leader %u, gen "DF_X64", class %u, action %u, " - "result %d, "DF_UUIDF"/"DF_UUIDF", seq "DF_X64": "DF_RC"\n", rank, leader, - gen, cla, act, result, DP_UUID(pool), DP_UUID(cont), seq, DP_RC(rc)); - - return rc; -} - int chk_rejoin_remote(d_rank_t leader, uint64_t gen, d_rank_t rank, uuid_t iv_uuid, uint32_t *flags, uint32_t *ns_ver, uint32_t *pool_nr, uuid_t **pools, d_rank_list_t **ranks) diff --git a/src/chk/chk_srv.c b/src/chk/chk_srv.c index d50e3b59657..fba39295045 100644 --- a/src/chk/chk_srv.c +++ b/src/chk/chk_srv.c @@ -141,11 +141,8 @@ ds_chk_mark_hdlr(crt_rpc_t *rpc) static void ds_chk_act_hdlr(crt_rpc_t *rpc) { - struct chk_act_in *cai = crt_req_get(rpc); - struct chk_act_out *cao = crt_reply_get(rpc); - int rc; - - rc = chk_engine_act(cai->cai_gen, cai->cai_seq, cai->cai_act); + struct chk_act_out *cao = crt_reply_get(rpc); + int rc = -DER_NOTSUPPORTED; cao->cao_status = rc; rc = crt_reply_send(rpc); @@ -214,31 +211,8 @@ ds_chk_pool_mbs_hdlr(crt_rpc_t *rpc) static void ds_chk_report_hdlr(crt_rpc_t *rpc) { - struct chk_report_in *cri = crt_req_get(rpc); - struct chk_report_out *cro = crt_reply_get(rpc); - struct chk_report_unit cru; - int rc; - - cru.cru_gen = cri->cri_gen; - cru.cru_cla = cri->cri_ics_class; - cru.cru_act = cri->cri_ics_action; - cru.cru_target = cri->cri_target; - cru.cru_rank = cri->cri_rank; - cru.cru_option_nr = cri->cri_options.ca_count; - cru.cru_detail_nr = cri->cri_details.ca_count; - cru.cru_pool = &cri->cri_pool; - cru.cru_pool_label = cri->cri_pool_label; - cru.cru_cont = &cri->cri_cont; - cru.cru_cont_label = cri->cri_cont_label; - cru.cru_obj = &cri->cri_obj; - cru.cru_dkey = &cri->cri_dkey; - cru.cru_akey = &cri->cri_akey; - cru.cru_msg = cri->cri_msg; - cru.cru_options = cri->cri_options.ca_arrays; - cru.cru_details = cri->cri_details.ca_arrays; - cru.cru_result = cri->cri_ics_result; - - rc = chk_leader_report(&cru, &cri->cri_seq, NULL); + struct chk_report_out *cro = crt_reply_get(rpc); + int rc = -DER_NOTSUPPORTED; cro->cro_status = rc; rc = crt_reply_send(rpc); diff --git a/src/include/daos_srv/daos_chk.h b/src/include/daos_srv/daos_chk.h index 9c363c86c9c..e29681bce18 100644 --- a/src/include/daos_srv/daos_chk.h +++ b/src/include/daos_srv/daos_chk.h @@ -86,7 +86,7 @@ int chk_leader_query(int pool_nr, uuid_t pools[], chk_query_head_cb_t head_cb, int chk_leader_prop(chk_prop_cb_t prop_cb, void *buf); int - chk_leader_act(uint64_t seq, uint32_t act); +chk_act(uint64_t seq, uint32_t act); int chk_leader_set_policy(uint32_t policy_nr, struct chk_policy *policies); diff --git a/src/mgmt/srv_chk.c b/src/mgmt/srv_chk.c index 3dd937d9bff..4ffc316facf 100644 --- a/src/mgmt/srv_chk.c +++ b/src/mgmt/srv_chk.c @@ -119,7 +119,7 @@ ds_mgmt_check_prop(chk_prop_cb_t prop_cb, void *buf) int ds_mgmt_check_act(uint64_t seq, uint32_t act) { - return chk_leader_act(seq, act); + return chk_act(seq, act); } int