Add Fenix_Process_detect_failures

sandialabs · Aug 27, 2024 · e2cc897 · e2cc897
1 parent 09f2885
commit e2cc897
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 0 deletions.
diff --git a/include/fenix.h b/include/fenix.h
@@ -231,6 +231,8 @@ int Fenix_Process_fail_list(int** fail_list);
 
 int Fenix_check_cancelled(MPI_Request *request, MPI_Status *status);
 
+int Fenix_Process_detect_failures(int do_recovery);
+
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

diff --git a/include/fenix_ext.h b/include/fenix_ext.h
@@ -96,6 +96,9 @@ typedef struct {
     //Manage state of the comms. Necessary when failures happen rapidly, mussing up state
     int new_world_exists, user_world_exists;
 
+    int dummy_recv_buffer;
+    MPI_Request check_failures_req;
+
 
     MPI_Op   agree_op;              // This is reserved for the global agreement call for Fenix data recovery API
 

diff --git a/include/fenix_process_recovery.h b/include/fenix_process_recovery.h
@@ -118,6 +118,8 @@ void __fenix_set_rank_role(int FenixRankRole);
 
 void __fenix_postinit(int *);
 
+int __fenix_detect_failures(int do_recovery);
+
 void __fenix_finalize();
 
 void __fenix_finalize_spare();

diff --git a/src/fenix.c b/src/fenix.c
@@ -209,3 +209,7 @@ int Fenix_check_cancelled(MPI_Request *request, MPI_Status *status){
     //Request was (potentially) cancelled if ret is MPI_ERR_PROC_FAILED
     return ret == MPI_ERR_PROC_FAILED || ret == MPI_ERR_REVOKED;
 }
+
+int Fenix_Process_detect_failures(int do_recovery){
+    return __fenix_detect_failures(do_recovery); 
+}
diff --git a/src/fenix_process_recovery.c b/src/fenix_process_recovery.c
@@ -686,6 +686,11 @@ void __fenix_postinit(int *error)
     //                fenix.role);
         //}
 
+    if(fenix.new_world_exists){
+        //Set up dummy irecv to use for checking for failures.
+        MPI_Irecv(&fenix.dummy_recv_buffer, 1, MPI_INT, MPI_ANY_SOURCE,
+                  34095347, fenix.new_world, &fenix.check_failures_req);
+    }
 
     if (fenix.repair_result != 0) {
         *error = fenix.repair_result;
@@ -707,6 +712,21 @@ void __fenix_postinit(int *error)
     }
 }
 
+int __fenix_detect_failures(int do_recovery){
+    if(!fenix.new_world_exists) return FENIX_ERROR_UNINITIALIZED;
+
+    int old_ignore_errs = fenix.ignore_errs;
+    fenix.ignore_errs = !do_recovery;
+
+    int req_completed;
+    int ret = MPI_Test(&fenix.check_failures_req, &req_completed, MPI_STATUS_IGNORE);
+
+    if(req_completed) ret = FENIX_ERROR_INTERN;
+
+    fenix.ignore_errs = old_ignore_errs;
+    return ret;
+}
+
 void __fenix_finalize()
 {
     int location = FENIX_FINALIZE_LOC;