merge gpu main

src/camp_solver.c

@@ -96,8 +96,8 @@ void *solver_new(int n_state_var, int n_cells, int *var_type, int n_rxn,
                  int n_sub_model, int n_sub_model_int_param,
                  int n_sub_model_float_param, int n_sub_model_env_param,
                  int use_cpu) {
-  // Create the SolverData object
 
+  // Create the SolverData object
   SolverData *sd = (SolverData *)malloc(sizeof(SolverData));
   if (sd == NULL) {
     printf("\n\nERROR allocating space for SolverData\n\n");

src/cuda/cvode_init.cu

@@ -43,8 +43,9 @@ void constructor_cvode_gpu(SolverData *sd){
                       MPI_INFO_NULL, &commNode);
   MPI_Comm_rank(commNode, &rankNode);
   MPI_Comm_size(commNode, &sizeNode);
-  int MPIsPerGPU = sizeNode / nGPUsMax;
-  cudaSetDevice(rankNode / MPIsPerGPU);
+  int MPIsPerGPU = (sizeNode + nGPUsMax - 1) / nGPUsMax;
+  int iDevice = rankNode / MPIsPerGPU;
+  cudaSetDevice(iDevice);
   mGPU->n_rxn=md->n_rxn;
   mGPU->n_rxn_env_data=md->n_rxn_env_data;
   HANDLE_ERROR(cudaMalloc((void **) &mGPU->state, state_size));

test/monarch/TestMonarch.py

@@ -17,13 +17,13 @@ def all_timesteps():
   conf.profileCuda = ""
   # conf.profileCuda = "nvprof"
   # conf.profileCuda = "nsight"
-  conf.is_import = True
-  conf.mpiProcessesCaseBase = 40
-  conf.mpiProcessesCaseOptimList = [40]
-  conf.cells = [100000]
-  conf.timeSteps = 720
+  #conf.is_import = True
+  conf.mpiProcessesCaseBase = 10
+  conf.mpiProcessesCaseOptimList = [1,2,3,4]
+  conf.cells = [10000]
+  conf.timeSteps = 1
   conf.caseBase = "CPU One-cell"
-  # conf.caseBase = "GPU BDF"
+  #conf.caseBase = "GPU BDF"
   conf.casesOptim = []
   # conf.casesOptim.append("CPU One-cell")
   conf.casesOptim.append("GPU BDF")

test/monarch/TestMonarch2.py

@@ -20,7 +20,7 @@ def all_timesteps():
   conf.is_import = True
   conf.mpiProcessesCaseBase = 20
   conf.mpiProcessesCaseOptimList = [20]
-  conf.cells = [10000]
+  conf.cells = [100000]
   conf.timeSteps = 720
   conf.caseBase = "CPU One-cell"
   # conf.caseBase = "GPU BDF"

test/monarch/TestMonarch4.py

@@ -19,8 +19,8 @@ def all_timesteps():
   # conf.profileCuda = "nsight"
   conf.is_import = True
   conf.mpiProcessesCaseBase = 1
-  conf.mpiProcessesCaseOptimList = [10,20,30,40]
-  conf.cells = [10000]
+  conf.mpiProcessesCaseOptimList = [1,2,3,4]
+  conf.cells = [100000]
   conf.timeSteps = 720
   conf.caseBase = "CPU One-cell"
   # conf.caseBase = "GPU BDF"

test/monarch/camp_monarch_interface.F90

@@ -54,10 +54,10 @@ module camp_monarch_interface
   contains
     procedure :: integrate
     procedure :: get_init_conc
+    procedure :: finalize
     procedure, private :: load
     procedure, private :: create_map
     procedure, private :: load_init_conc
-    final :: finalize
   end type camp_monarch_interface_t
 
   interface camp_monarch_interface_t
@@ -808,8 +808,8 @@ subroutine get_init_conc(this, MONARCH_conc, water_conc, &
     end if
   end subroutine get_init_conc
 
-  elemental subroutine finalize(this)
-    type(camp_monarch_interface_t), intent(inout) :: this
+  subroutine finalize(this)
+    class(camp_monarch_interface_t) :: this
     if (associated(this%camp_core)) &
             deallocate(this%camp_core)
     if (associated(this%camp_state)) &
@@ -824,13 +824,14 @@ elemental subroutine finalize(this)
             deallocate(this%init_conc_camp_id)
     if (allocated(this%init_conc)) &
             deallocate(this%init_conc)
-    if (associated(this%species_map_data)) &
-            deallocate(this%species_map_data)
-    if (associated(this%init_conc_data)) &
-            deallocate(this%init_conc_data)
-    if (associated(this%property_set)) &
-            deallocate(this%property_set)
-
-  end subroutine finalize
+    if(camp_mpi_rank()==0) then
+      if (associated(this%init_conc_data)) &
+              deallocate(this%init_conc_data)
+      if (associated(this%species_map_data)) &
+              deallocate(this%species_map_data)
+      if (associated(this%property_set)) &
+              deallocate(this%property_set)
+    end if
+  end subroutine
 
 end module

test/monarch/mainMonarch.py

@@ -52,13 +52,13 @@ def run(conf):
       exec_str += 'ddt --connect '
   except Exception:
     pass
-  maxCoresPerNode = 40
+  maxCoresPerNode = 40 #CTE-POWER Architecture
   if conf.mpiProcesses <= maxCoresPerNode:
     exec_str += "mpirun -v -np " + str(
       conf.mpiProcesses) + " --bind-to core " #for plogin (fails squeue)
   else:
     exec_str += "srun --cpu-bind=core -n " + str(
-      conf.mpiProcesses) + " " #foqueue (slow plogin)
+      conf.mpiProcesses) + " " #for queue (slow plogin)
   if (conf.profileCuda == "nvprof" and conf.caseGpuCpu ==
       "GPU"):
     pathNvprof = ("../../compile/power9/" +

test/monarch/mock_monarch.F90

@@ -192,6 +192,7 @@ program mock_monarch_t
   deallocate(camp_input_file)
   deallocate(output_path)
   deallocate(output_file_title)
+  call camp_interface%finalize()
   deallocate(camp_interface)
   call camp_mpi_finalize()
 

test/monarch/run.sh

@@ -19,8 +19,8 @@ make_run() {
   cd ../../build
   make -j 4
   cd $curr_path
-  #python TestMonarch.py
-  python checkGPU.py
+  python TestMonarch4.py
+  #python checkGPU.py
 }
 
 make_run

test/monarch/sbatch_run.sh

@@ -16,9 +16,7 @@ make_run(){
   cd ../../build
   make -j 4
   cd $curr_path
-  python TestMonarch1.py
   python TestMonarch2.py
-  python TestMonarch3.py
   python TestMonarch4.py
 }
 time make_run