DAOS-16702 rebuild: restart rebuild for a massive failure case (#15406)

In special massive failure case -
1. some engines down and triggered rebuild.
2. one engine participated the rebuild, not finished yet, it down again,
   the #failures exceeds pool RF and will not change pool map.
3. That engine restarted by administrator.

In that case should recover the rebuild task on the engine, to simplify it now just
abort and retry the global rebuild task.
No such issue by the typical recover approach that restart the whole
system including the PS leader.

another backport commit -
947c76d DAOS-16175 container: fix a case for cont_iv_hdl_fetch (#15395)

Signed-off-by: Xuezhao Liu <xuezhao.liu@intel.com>

src/container/container_iv.c

@@ -1018,6 +1018,7 @@ cont_iv_hdl_fetch(uuid_t cont_hdl_uuid, uuid_t pool_uuid,
 	D_DEBUG(DB_TRACE, "Can not find "DF_UUID" hdl\n",
 		DP_UUID(cont_hdl_uuid));
 
+invalidate_retry:
 	/* Fetch the capability from the leader. To avoid extra locks,
 	 * all metadatas are maintained by xstream 0, so let's create
 	 * an ULT on xstream 0 to let xstream 0 to handle capa fetch
@@ -1046,6 +1047,19 @@ cont_iv_hdl_fetch(uuid_t cont_hdl_uuid, uuid_t pool_uuid,
 	if (*cont_hdl == NULL) {
 		D_DEBUG(DB_TRACE, "Can not find "DF_UUID" hdl\n",
 			DP_UUID(cont_hdl_uuid));
+		/* In reintegrate with case that the IC_CONT_CAPA cache is valid locally
+		 * but cont open handle invalid (not in dt_cont_hdl_hash). For this case
+		 * invalidate local IV cache first and retry again, to avoid in-flight
+		 * UPDATE's failure. (IV locally valid then the IV fetch will not trigger
+		 * cont_iv_ent_update() callback).
+		 */
+		if (!invalidate_current) {
+			invalidate_current = true;
+			ABT_eventual_free(&eventual);
+			D_DEBUG(DB_TRACE, DF_UUID" invalidate_current and retry\n",
+				DP_UUID(cont_hdl_uuid));
+			goto invalidate_retry;
+		}
 		D_GOTO(out_eventual, rc = -DER_NONEXIST);
 	}
 

src/include/daos_srv/rebuild.h

@@ -35,10 +35,32 @@ typedef enum {
 			  (rb_op) == RB_OP_NONE ? "None" : \
 			  "Unknown")
 
+/* Common rebuild identifying information for INFO/DEBUG logging:
+ * rb=<pool_uuid>/<rebuild_ver>/<rebuild_gen>/<opstring>
+ */
+#define DF_RB  "rb=" DF_UUID "/%u/%u/%s"
+
+/* Full rebuild identifying information includes <leader_rank>/<term>
+ * Instead of this, use DF_RB most of the time (use this for leader change scenarios, etc.)
+ */
+#define DF_RBF DF_RB " ld=%u/" DF_U64
+
+/* arguments for log rebuild identifier given a struct rebuild_global_pool_tracker * */
+#define DP_RB_RGT(rgt)                                                                             \
+	DP_UUID((rgt)->rgt_pool_uuid), (rgt)->rgt_rebuild_ver, (rgt)->rgt_rebuild_gen,             \
+	    RB_OP_STR((rgt)->rgt_opc)
+
+/* arguments for log rebuild identifier given a struct rebuild_tgt_pool_tracker *rpt */
+#define DP_RB_RPT(rpt)                                                                             \
+	DP_UUID((rpt)->rt_pool_uuid), (rpt)->rt_rebuild_ver, (rpt)->rt_rebuild_gen,                \
+	    RB_OP_STR((rpt)->rt_rebuild_op)
+#define DP_RBF_RPT(rpt) DP_RB_RPT(rpt), (rpt)->rt_leader_rank, (rpt)->rt_leader_term
+
 int ds_rebuild_schedule(struct ds_pool *pool, uint32_t map_ver,
 			daos_epoch_t stable_eph, uint32_t layout_version,
 			struct pool_target_id_list *tgts,
 			daos_rebuild_opc_t rebuild_op, uint64_t delay_sec);
+void ds_rebuild_restart_if_rank_wip(uuid_t pool_uuid, d_rank_t rank);
 int ds_rebuild_query(uuid_t pool_uuid,
 		     struct daos_rebuild_status *status);
 void ds_rebuild_running_query(uuid_t pool_uuid, uint32_t opc, uint32_t *rebuild_ver,

src/object/cli_obj.c

@@ -6319,7 +6319,9 @@ obj_ec_get_parity_or_alldata_shard(struct obj_auxi_args *obj_auxi, unsigned int
 			shard_idx = grp_start + i;
 			if (obj_shard_is_invalid(obj, shard_idx, DAOS_OBJ_RPC_ENUMERATE)) {
 				if (++fail_cnt > obj_ec_parity_tgt_nr(oca)) {
-					D_ERROR(DF_OID" reach max failure "DF_RC"\n",
+					D_ERROR(DF_CONT", obj "DF_OID" reach max failure "DF_RC"\n",
+						DP_CONT(obj->cob_pool->dp_pool,
+							obj->cob_co->dc_uuid),
 						DP_OID(obj->cob_md.omd_id), DP_RC(-DER_DATA_LOSS));
 					D_GOTO(out, shard = -DER_DATA_LOSS);
 				}
@@ -6466,7 +6468,8 @@ obj_list_shards_get(struct obj_auxi_args *obj_auxi, unsigned int map_ver,
 	}
 
 	if (rc < 0) {
-		D_ERROR(DF_OID" Can not find shard grp %d: "DF_RC"\n",
+		D_ERROR(DF_CONT", obj "DF_OID" Can not find shard grp %d: "DF_RC"\n",
+			DP_CONT(obj->cob_pool->dp_pool, obj->cob_co->dc_uuid),
 			DP_OID(obj->cob_md.omd_id), grp_idx, DP_RC(rc));
 		D_GOTO(out, rc);
 	}

src/pool/srv_pool.c

@@ -1458,6 +1458,43 @@ resume_event_handling(struct pool_svc *svc)
 	ABT_mutex_unlock(events->pse_mutex);
 }
 
+/*
+ * Restart rebuild if the rank is UPIN in pool map and is in rebuilding.
+ *
+ * This function only used when PS leader gets CRT_EVT_ALIVE event of engine \a rank,
+ * if that rank is UPIN in pool map and with unfinished rebuilding should be massive
+ * failure case -
+ * 1. some engines down and triggered rebuild.
+ * 2. the engine \a rank participated the rebuild, not finished yet, it became down again,
+ *    the #failures exceeds pool RF and will not change pool map.
+ * 3. That engine restarted by administrator.
+ *
+ * In that case should recover the rebuild task on engine \a rank, to simplify it now just
+ * abort and retry the global rebuild task.
+ */
+static void
+pool_restart_rebuild_if_rank_wip(struct ds_pool *pool, d_rank_t rank)
+{
+	struct pool_domain	*dom;
+
+	dom = pool_map_find_dom_by_rank(pool->sp_map, rank);
+	if (dom == NULL) {
+		D_DEBUG(DB_MD, DF_UUID": rank %d non-exist on pool map.\n",
+			DP_UUID(pool->sp_uuid), rank);
+		return;
+	}
+
+	if (dom->do_comp.co_status != PO_COMP_ST_UPIN) {
+		D_INFO(DF_UUID": rank %d status %d in pool map, got CRT_EVT_ALIVE.\n",
+		       DP_UUID(pool->sp_uuid), rank, dom->do_comp.co_status);
+		return;
+	}
+
+	ds_rebuild_restart_if_rank_wip(pool->sp_uuid, rank);
+
+	return;
+}
+
 static int pool_svc_exclude_ranks(struct pool_svc *svc, struct pool_svc_event_set *event_set);
 
 static int
@@ -1487,8 +1524,13 @@ handle_event(struct pool_svc *svc, struct pool_svc_event_set *event_set)
 	for (i = 0; i < event_set->pss_len; i++) {
 		struct pool_svc_event *event = &event_set->pss_buf[i];
 
-		if (event->psv_src != CRT_EVS_SWIM || event->psv_type != CRT_EVT_ALIVE)
+		if (event->psv_type != CRT_EVT_ALIVE)
 			continue;
+
+		D_DEBUG(DB_MD, DF_UUID ": got CRT_EVT_ALIVE event, psv_src %d, psv_rank %d\n",
+		       DP_UUID(svc->ps_uuid), event->psv_src, event->psv_rank);
+		pool_restart_rebuild_if_rank_wip(svc->ps_pool, event->psv_rank);
+
 		if (ds_pool_map_rank_up(svc->ps_pool->sp_map, event->psv_rank)) {
 			/*
 			 * The rank is up in the pool map. Request a pool map

src/rebuild/scan.c

@@ -1169,6 +1169,24 @@ rebuild_tgt_scan_handler(crt_rpc_t *rpc)
 			  "rsi_rebuild_ver %d != rt_rebuild_ver %d\n",
 			  rsi->rsi_rebuild_ver, rpt->rt_rebuild_ver);
 
+		/* The same PS leader request rebuild with higher rsi_rebuild_gen.
+		 * Is the case of massive failure case, see pool_restart_rebuild_if_rank_wip().
+		 */
+		if (rpt->rt_leader_rank == rsi->rsi_master_rank &&
+		    rpt->rt_leader_term == rsi->rsi_leader_term &&
+		    rpt->rt_rebuild_gen < rsi->rsi_rebuild_gen) {
+			/* rebuild_leader_status_notify(LAZY rebuild_iv_update),
+			 * it will set rpt->rt_global_done to abort rpt.
+			 * set rt_abort here just for safe.
+			 */
+			rpt->rt_abort = 1;
+			D_INFO(DF_RBF ", start new rebuild, gen %d -> %d.\n",
+			       DP_RBF_RPT(rpt), rpt->rt_rebuild_gen, rsi->rsi_rebuild_gen);
+			rpt_put(rpt);
+			rpt = NULL;
+			goto tls_lookup;
+		}
+
 		D_DEBUG(DB_REBUILD, DF_UUID" already started, req "DF_U64" master %u/"DF_U64"\n",
 			DP_UUID(rsi->rsi_pool_uuid), rsi->rsi_leader_term, rsi->rsi_master_rank,
 			rpt->rt_leader_term);
@@ -1203,6 +1221,7 @@ rebuild_tgt_scan_handler(crt_rpc_t *rpc)
 		rpt = NULL;
 	}
 
+tls_lookup:
 	tls = rebuild_pool_tls_lookup(rsi->rsi_pool_uuid, rsi->rsi_rebuild_ver,
 				      rsi->rsi_rebuild_gen);
 	if (tls != NULL) {

src/rebuild/srv.c

@@ -490,6 +490,45 @@ ds_rebuild_running_query(uuid_t pool_uuid, uint32_t opc, uint32_t *upper_ver,
 		rpt_put(rpt);
 }
 
+/*
+ * Restart rebuild if \a rank's rebuild not finished.
+ * Only used for massive failure recovery case, see pool_restart_rebuild_if_rank_wip().
+ */
+void
+ds_rebuild_restart_if_rank_wip(uuid_t pool_uuid, d_rank_t rank)
+{
+	struct rebuild_global_pool_tracker	*rgt;
+	int					 i;
+
+	rgt = rebuild_global_pool_tracker_lookup(pool_uuid, -1, -1);
+	if (rgt == NULL)
+		return;
+
+	if (rgt->rgt_status.rs_state != DRS_IN_PROGRESS) {
+		rgt_put(rgt);
+		return;
+	}
+
+	for (i = 0; i < rgt->rgt_servers_number; i++) {
+		if (rgt->rgt_servers[i].rank == rank) {
+			if (!rgt->rgt_servers[i].pull_done) {
+				rgt->rgt_status.rs_errno = -DER_STALE;
+				rgt->rgt_abort = 1;
+				rgt->rgt_status.rs_fail_rank = rank;
+				D_INFO(DF_RB ": abort rebuild because rank %d WIP\n",
+				       DP_RB_RGT(rgt), rank);
+			}
+			rgt_put(rgt);
+			return;
+		}
+	}
+
+	D_INFO(DF_RB ": rank %d not in rgt_servers,  rgt_servers_number %d\n",
+	       DP_RB_RGT(rgt), rank, rgt->rgt_servers_number);
+	rgt_put(rgt);
+	return;
+}
+
 /* TODO: Add something about what the current operation is for output status */
 int
 ds_rebuild_query(uuid_t pool_uuid, struct daos_rebuild_status *status)