Travis fixes (sandialabs#55)

Fix some travis/testing issues.

Travis now pulls from ULFM master branch when it needs to rebuild ULFM.
Travis has an environment variable enabling oversubscription during the tests, instead of having that on all platforms when running make test
Tests that involve failure have their timeouts individually set to 1, so tests don't take 10+ seconds each w/ the default timeout of 10s
Simplified travis scripts (no more .travis_helpers directory)

.travis.yml

@@ -9,24 +9,54 @@ addons:
       - valgrind
 cache:
    directories:
-      - .travis_helpers/ulfm-install
+      - ulfm-install
 before_install:
-   - cd .travis_helpers
-   - source ./fetchULFMmpi.sh
-   - cd ../ #Always end back at the root directory
+   - echo "Configuring ULFM"
+   - if [ -f ulfm-install/lib/libmpi.so ]; then
+       echo "libmpi.so found -- nothing to build.";
+       cd ulfm-install;
+     else
+       ROOT=`pwd`;
+       mkdir ulfm-install;
+       echo "Downloading ULFM from repo";
+       git clone --recursive https://bitbucket.org/icldistcomp/ulfm2.git ulfm-src/;
+       echo " - Configuring and building ULFM.";
+       cd ulfm-src;
+       echo " - Running autogen.pl";
+       ./autogen.pl >../ulfm-install/ulfm_build_output.txt 2>&1;
+       echo " - Running configure";
+       ./configure --prefix=$ROOT/ulfm-install >>../ulfm-install/ulfm_build_output.txt 2>&1;
+       echo " - Running make";
+       make -j4 >>../ulfm-install/ulfm_build_output.txt 2>&1;
+       echo " - Running make install";
+       make install >>../ulfm-install/ulfm_build_output.txt 2>&1;
+       echo " - Finished installing ULFM";
+       cd ../ulfm-install/;
+     fi
+
+   #Expect that any changes to the above still puts me in the install's home dir
+   - export MPI_HOME=`pwd`
+   - export PATH=$MPI_HOME/bin/:$PATH
+   - export LD_LIBRARY_PATH=$MPI_HOME/lib:$LD_LIBRARY_PATH
+   - export DYLD_LIBRARY_PATH=$MPI_HOME/lib:$DYLD_LIBRARY_PATH
+   - export MANPATH=$MPI_HOME/share/man:$MANPATH
+
+   - export MPICC="`which mpicc`"
+   - export MPICXX="`which mpic++`"
+
+   #Allow oversubscription for tests, since we're potentially single core
+   - export OMPI_MCA_rmaps_base_oversubscribe=1 
+
+   - tail -n50 ./ulfm_build_output.txt
+   - cd ../ #End back at root
+install:
+   - mkdir build && cd build
+   - cmake ../ -DBUILD_TESTING=ON && make -j4 VERBOSE=1
 script:
-   - cd .travis_helpers
-   - source fetchULFMmpi.sh #Just updates path if ULFM was built properly in before_install
-   - cd ../
-   - mkdir build
-   - cd build
-   - cmake ../ -DBUILD_TESTING=ON
-   - make -j4 VERBOSE=1
    - make test
-   - cd ../ #Always end back at the root directory.
+after_success:
+   - echo "Success, printing run logs:"
+   - cat Testing/Temporary/LastTest.log
 after_failure:
    - echo "Failure occured, printing run logs:"
-   - pwd
-   - cat build/Testing/Temporary/LastTest.log
-   - echo "Printing ULFM build log tail. If no output, ULFM was built before this test run"
-   - tail -n100 .travis_helpers/build_output.txt
+   - cat Testing/Temporary/LastTest.log

CMakeLists.txt

@@ -25,8 +25,8 @@ set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS)
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 
-set(CMAKE_BUILD_TYPE Release)
-#set(CMAKE_BUILD_TYPE Debug)
+#set(CMAKE_BUILD_TYPE Release)
+set(CMAKE_BUILD_TYPE Debug)
 #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O0 -ggdb")
 
 #ENABLE_TESTING
@@ -109,4 +109,6 @@ if(BUILD_TESTING)
     add_subdirectory(test/subset_internal)
     add_subdirectory(test/subset_merging)
     add_subdirectory(test/request_tracking)
+    add_subdirectory(test/request_cancelled)
+    add_subdirectory(test/no_jump)
 endif()

examples/01_hello_world/fenix/CMakeLists.txt

@@ -16,5 +16,5 @@ if(BUILD_TESTING)
    add_executable(fenix_hello_world-debug fenix_hello_world.c)
    target_link_libraries(fenix_hello_world-debug fenix ${MPI_C_LIBRARIES})
    add_test(NAME hello_world 
-      COMMAND mpirun --oversubscribe -np 3 fenix_hello_world-debug "1")
+      COMMAND mpirun -mca mpi_ft_detector_timeout 1 -np 3 fenix_hello_world-debug "1")
 endif()

examples/01_hello_world/fenix/fenix_hello_world.c

@@ -108,6 +108,19 @@ int main(int argc, char **argv) {
 
   printf("hello world: %s, old rank (MPI_COMM_WORLD): %d, new rank: %d, active ranks: %d, ranks before process failure: %d\n",
          processor_name, old_rank, new_rank, new_world_size, old_world_size);
+
+  int *fails, num_fails;
+  num_fails = Fenix_Process_fail_list(&fails);
+
+  char fails_str[100];
+  sprintf(fails_str, "Rank %d sees failed processes [", new_rank);
+  for(int i = 0; i < num_fails; i++){
+    sprintf(fails_str, "%s%s%d", fails_str, (i==0 ? "" : ", "), fails[i]);
+  }
+  sprintf(fails_str, "%s]\n", fails_str);
+  printf(fails_str);
+
+
 
   Fenix_Finalize();
   MPI_Finalize();

examples/02_send_recv/fenix/CMakeLists.txt

@@ -16,7 +16,7 @@ if(BUILD_TESTING)
    add_executable(fenix_ring-debug fenix_ring.c)
    target_link_libraries(fenix_ring-debug fenix ${MPI_C_LIBRARIES})
    add_test(NAME ring 
-      COMMAND mpirun --oversubscribe -np 5 fenix_ring-debug 1 2)
+      COMMAND mpirun -mca mpi_ft_detector_timeout 1 -np 5 fenix_ring-debug 1 2)
    set_tests_properties(ring PROPERTIES
       FAIL_REGULAR_EXPRESSION "FAILURE") 
 endif()

examples/05_subset_create/CMakeLists.txt

@@ -16,7 +16,7 @@ if(BUILD_TESTING)
    add_executable(fenix_subset_create-debug subset_create.c)
    target_link_libraries(fenix_subset_create-debug fenix ${MPI_C_LIBRARIES})
    add_test(NAME subset_create 
-      COMMAND mpirun -np 5 --oversubscribe fenix_subset_create-debug 1)
+      COMMAND mpirun -mca mpi_ft_detector_timeout 1 -np 5 fenix_subset_create-debug 1)
    set_tests_properties(subset_create PROPERTIES
       FAIL_REGULAR_EXPRESSION "FAILURE") 
 endif()

examples/06_subset_createv/CMakeLists.txt

@@ -16,7 +16,7 @@ if(BUILD_TESTING)
    add_executable(fenix_subset_createv-debug subset_createv.c)
    target_link_libraries(fenix_subset_createv-debug fenix ${MPI_C_LIBRARIES})
    add_test(NAME subset_createv 
-      COMMAND mpirun -np 5 --oversubscribe fenix_subset_createv-debug 1)
+      COMMAND mpirun -mca mpi_ft_detector_timeout 1 -np 5 fenix_subset_createv-debug 1)
    set_tests_properties(subset_createv PROPERTIES
       FAIL_REGULAR_EXPRESSION "FAILURE") 
 endif()

include/fenix.h

@@ -88,6 +88,7 @@ extern "C" {
 #define FENIX_ERROR_SUBSET_STRIDE           -25
 #define FENIX_ERROR_NODATA_FOUND            -30
 #define FENIX_ERROR_INTERN                  -40
+#define FENIX_ERROR_CANCELLED               -50
 #define FENIX_WARNING_SPARE_RANKS_DEPLETED  100
 #define FENIX_WARNING_PARTIAL_RESTORE       101
 
@@ -216,6 +217,8 @@ int Fenix_Data_group_delete(int group_id);
 
 int Fenix_Data_member_delete(int group_id, int member_id);
 
+int Fenix_Process_fail_list(int** fail_list);
+
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

include/fenix_ext.h

@@ -81,6 +81,13 @@ typedef struct {
     int role;    // Role of rank: initial, survivor or repair
     int fenix_init_flag;
 
+    int fail_world_size;
+    int* fail_world;
+
+    //Save the pointer to role and error of Fenix_Init
+    int *ret_role;
+    int *ret_error;
+
     fenix_request_store_t request_store;
 
     fenix_callback_list_t* callback_list;  // singly linked list for user-defined Fenix callback functions

src/fenix.c

@@ -181,3 +181,8 @@ int Fenix_Data_group_delete(int group_id) {
 int Fenix_Data_member_delete(int group_id, int member_id) {
     return __fenix_member_delete(group_id, member_id);
 }
+
+int Fenix_Process_fail_list(int** fail_list){
+  *fail_list = fenix.fail_world;
+  return fenix.fail_world_size;
+}

src/fenix_data_recovery.c

@@ -103,7 +103,7 @@ int __fenix_group_create( int groupid, MPI_Comm comm, int timestart, int depth,
     /* If so, recover the data and set the recovery         */
     /* for member recovery.                                 */
 
-    int i, group_position;
+    int i;
     int remote_need_recovery;
     fenix_group_t *group;
     MPI_Status status;
@@ -149,7 +149,7 @@ int __fenix_group_create( int groupid, MPI_Comm comm, int timestart, int depth,
 
     } else { /* Already created. Renew the MPI communicator  */
 
-      group = ( data_recovery->group[group_position] );
+      group = ( data_recovery->group[group_index] );
       group->comm = comm; /* Renew communicator */
       MPI_Comm_rank(comm, &(group->current_rank));
 

src/fenix_mpi_override.c

@@ -238,7 +238,9 @@ int MPI_Sendrecv(MPI_CONST_TYPE void* sendbuf, int sendcount,
 static inline
 void __fenix_override_request(int ret, MPI_Request *request)
 {
-    if(ret != MPI_SUCCESS) return;
+    if(ret != MPI_SUCCESS) {
+       return;
+    }
 
     assert(*request != MPI_REQUEST_NULL);
 
@@ -265,28 +267,55 @@ int MPI_Irecv(void *buf, int count, MPI_Datatype datatype,
     int ret;
     ret = PMPI_Irecv(buf, count, datatype, source, tag,
                      __fenix_replace_comm(comm), request);
+
     __fenix_override_request(ret, request);
     __fenix_test_MPI_inline(ret, "MPI_Irecv");
     return ret;
 }
 
 int MPI_Wait(MPI_Request *fenix_request, MPI_Status *status)
 {
-    int ret;
+    int ret, is_cancelled = 1;
     MPI_Request request = MPI_REQUEST_NULL;
-    if(*fenix_request != MPI_REQUEST_NULL)
-        __fenix_request_store_get(&fenix.request_store,
-				  *((int *) fenix_request),
-				  &request);
+    if(*fenix_request != MPI_REQUEST_NULL){
+        if(*fenix_request == FENIX_REQUEST_CANCELLED){
+            is_cancelled = 1;
+        } else {
+            int retval = 
+               __fenix_request_store_get(&fenix.request_store, *((int*)fenix_request), &request);
+
+            if(retval == FENIX_ERROR_CANCELLED) {
+               is_cancelled = 1;
+            }    
+
+            if(retval == FENIX_REQUEST_COMPLETED){
+               if(status != MPI_STATUS_IGNORE)
+                  __fenix_request_store_get_status(&fenix.request_store, *((int*)fenix_request), status);
+               *fenix_request = MPI_REQUEST_NULL;
+               return;
+            }
+        }
+    }
 
     ret = PMPI_Wait(&request, status);
-    if(ret == MPI_SUCCESS) {
+
+    if(ret == MPI_SUCCESS && (*fenix_request != MPI_REQUEST_NULL) && (*fenix_request != FENIX_REQUEST_CANCELLED)) {
         __fenix_request_store_remove(&fenix.request_store,
 				     *((int *) fenix_request));
         assert(request == MPI_REQUEST_NULL);
-	*fenix_request = MPI_REQUEST_NULL;
+	     *fenix_request = MPI_REQUEST_NULL;
+    }
+    if(ret == MPI_ERR_PROC_FAILED || ret == MPI_ERR_REVOKED){
+      __fenix_request_store_cancel(&fenix.request_store, *((int*)fenix_request), status);
+      *fenix_request = FENIX_REQUEST_CANCELLED;
     }
     __fenix_test_MPI_inline(ret, "MPI_Wait");
+
+
+    if(is_cancelled){
+       *fenix_request = FENIX_REQUEST_CANCELLED;  
+       return FENIX_ERROR_CANCELLED;
+    }
     return ret;
 }
 
@@ -297,11 +326,13 @@ int MPI_Waitall(int count, MPI_Request array_of_fenix_requests[],
 {
     // The list (array_of_requests) may contain null or inactive handles.
     int ret, i;
-    for(i=0 ; i<count ; i++)
-        if(array_of_fenix_requests[i] != MPI_REQUEST_NULL)
-	    __fenix_request_store_getremove(&fenix.request_store,
+    for(i=0 ; i<count ; i++){
+        if(array_of_fenix_requests[i] != MPI_REQUEST_NULL){
+             __fenix_request_store_getremove(&fenix.request_store,
 					    *((int *)&(array_of_fenix_requests[i])),
 					    &(array_of_fenix_requests[i]));
+        }
+    }
 
     ret = PMPI_Waitall(count, array_of_fenix_requests, array_of_statuses);
     __fenix_test_MPI_inline(ret, "MPI_Waitall");
@@ -333,8 +364,52 @@ int MPI_Waitall(int count, MPI_Request array_of_fenix_requests[],
 
 int MPI_Test(MPI_Request *request, int *flag, MPI_Status *status)
 {
-#warning "TODO"
-  printf("Fenix: need to implement MPI_Test\n");
+    int ret;
+    int is_cancelled = 0;
+    MPI_Request real_req = MPI_REQUEST_NULL;
+
+    if(*request != MPI_REQUEST_NULL){
+        if(*request == FENIX_REQUEST_CANCELLED){
+            is_cancelled = 1;
+        } else {
+            int retval = 
+               __fenix_request_store_get(&fenix.request_store, *((int*)request), &real_req);
+
+            if(retval == FENIX_ERROR_CANCELLED) {
+               is_cancelled = 1;
+            }
+
+            if(retval == FENIX_REQUEST_COMPLETED){
+               *flag = 1;
+               if(status != MPI_STATUS_IGNORE)
+                  __fenix_request_store_get_status(&fenix.request_store, *((int*)request), status);
+               *request = MPI_REQUEST_NULL;
+               return;
+            }
+        }
+    }
+
+
+    ret = PMPI_Test(&real_req, flag, status);
+    if(ret == MPI_ERR_PROC_FAILED || ret == MPI_ERR_REVOKED){
+      __fenix_request_store_cancel(&fenix.request_store, *((int*)request), status);
+       *request = FENIX_REQUEST_CANCELLED;
+    }
+
+    __fenix_test_MPI_inline(ret, "MPI_Test");
+
+    if(*flag && *request != MPI_REQUEST_NULL && *request != FENIX_REQUEST_CANCELLED && ret == MPI_SUCCESS){
+       //This request is done, it can be removed from the store.
+       __fenix_request_store_remove(&fenix.request_store, *((int*)request));
+       *request = MPI_REQUEST_NULL;
+    }
+
+    if(is_cancelled){
+       *request = FENIX_REQUEST_CANCELLED;
+       return FENIX_ERROR_CANCELLED;
+    }
+
+    else return ret;
 }
 
 int MPI_Cancel(MPI_Request *request)