merge gpu branch onto main

CMakeLists.txt

@@ -225,12 +225,10 @@ endif()
 
 ######################################################################
 # copy dirs
-if(DISABLE_TESTS)
-else ()
-SET(ENABLE_TESTS ON)
-endif ()
 
-if(ENABLE_TESTS)
+SET(USE_TESTS ON)
+
+if(USE_TESTS)
 add_custom_target(copy_data ALL ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/data ${CMAKE_BINARY_DIR}/data_run)
 add_custom_target(copy_test ALL ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/test ${CMAKE_BINARY_DIR}/test_run)
 add_custom_target(copy_mechanism ALL ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/mechanisms ${CMAKE_BINARY_DIR}/mechanisms_run)
@@ -241,23 +239,24 @@ endif()
 ######################################################################
 # Unit test macro
 
-if (ENABLE_TESTS)
-  macro(do_unit_test test_name result)
-    if(ENABLE_MPI)
-      add_test(unit_test_${test_name} mpirun -v -np 1 ${CMAKE_BINARY_DIR}/unit_test_${test_name})
-    else()
-      add_test(unit_test_${test_name} ${CMAKE_BINARY_DIR}/unit_test_${test_name})
-    endif()
-    set_tests_properties(unit_test_${test_name}
-            PROPERTIES PASS_REGULAR_EXPRESSION ${result})
-  endmacro(do_unit_test)
+if (USE_TESTS)
+macro(do_unit_test test_name result)
+if(ENABLE_MPI)
+  add_test(unit_test_${test_name} mpirun -v -np 2 ${CMAKE_BINARY_DIR}/unit_test_${test_name})
+else()
+  add_test(unit_test_${test_name} ${CMAKE_BINARY_DIR}/unit_test_${test_name})
+endif()
+set_tests_properties(unit_test_${test_name}
+        PROPERTIES PASS_REGULAR_EXPRESSION ${result})
+endmacro(do_unit_test)
 endif()
 
 ######################################################################
 # tests
 
 enable_testing()
-if (ENABLE_TESTS)
+
+if (USE_TESTS)
 do_unit_test(property "PASS")
 do_unit_test(chem_spec_data "PASS")
 do_unit_test(aero_phase_data "PASS")
@@ -277,7 +276,7 @@ if (ENABLE_GPU)
   add_test(NAME test_gpu COMMAND checkGPU.sh WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test/monarch)
 endif()
 
-if (ENABLE_TESTS)
+if (USE_TESTS)
     # New unit tests (UNDER DEVELOPMENT)
   add_test(test_rxn_arrhenius_mech ${CMAKE_BINARY_DIR}/test_run/unit_tests/input_files/run_rxn_arrhenius.sh ${MPI_TEST_FLAG})
 
@@ -406,7 +405,7 @@ set(SUB_MODELS_SRC ${SUB_MODELS_F_SRC} ${SUB_MODELS_C_SRC})
 set(CAMP_C_SRC
         src/camp_solver.c src/rxn_solver.c src/aero_phase_solver.c
         src/aero_rep_solver.c src/sub_model_solver.c
-        src/debug_and_stats/camp_debug_2.c
+        src/camp_debug.c
         src/time_derivative.c
         src/Jacobian.c src/debug_diff_check.c)
 
@@ -479,14 +478,6 @@ install(
   )
 endif()
 
-######################################################################
-# camp-chem box model
-
-add_executable(camp_box_model test/camp_box_model_data.F90
-                              test/camp_box_model.F90)
-
-target_link_libraries(camp_box_model camplib)
-
 ######################################################################
 # test_chemistry_cb05cl_ae5
 
@@ -538,7 +529,16 @@ if (ENABLE_GPU)
   target_link_libraries(mock_monarch camplib)
 endif()
 
-if (ENABLE_TESTS)
+if (USE_TESTS)
+
+######################################################################
+# camp-chem box model
+
+add_executable(camp_box_model test/camp_box_model_data.F90
+    test/camp_box_model.F90)
+
+target_link_libraries(camp_box_model camplib)
+
 
 ######################################################################
 # test_chemistry_cb05cl_ae5

compile/power9/check.sh

@@ -2,7 +2,6 @@ set -e
 cd ../../build
 make -j 4
 ctest --output-on-failure
-#make test
 #./unit_test_aero_rep_single_particle
 cd ../test/monarch
 #./checkGPU.sh

compile/power9/compile.camp.sh

@@ -37,7 +37,6 @@ elif [ LOCAL_MACHINE==CGUZMAN ]; then
       echo "MPI is not installed. Installing..."
       sudo apt update
       sudo apt install -y mpi-default-dev
-      #if run | Invalid MIT-MAGIC-COOKIE-1 key THEN sudo apt-remove openmpi-bin AND sudo apt-get install libcr-dev mpich2 mpich2-doc
   fi
 else
   echo "Unknown architecture"
@@ -65,8 +64,9 @@ cmake -D CMAKE_C_COMPILER=$(which mpicc) \
 -D ENABLE_GPU=ON \
 -D ENABLE_GSL:BOOL=FALSE \
 -D ENABLE_NETCDF=ON \
--D DISABLE_INSTALL_OPTIONS=TRUE \
 ..
 
+ln -sf ../test/monarch/settings
+ln -sf ../test/monarch/out
 make -j 4 VERBOSE=1
 cd $curr_path

data/CAMP_v1_paper/README.md

@@ -2,15 +2,12 @@
 
 Configurations for recreating experiments for:
 
- * M. Dawson, C. Guzman, J. H. Curtis, M. Acosta, S. Zhu, D. Dabdub,
-     A. Conley, M. West, N. Riemer, and O. Jorba (2021),
-     Chemistry Across Multiple Phases (CAMP) version 1.0: An
-     Integrated multi-phase chemistry model, in preparation
+ * Dawson, M. L., Guzman, C., Curtis, J. H., Acosta, M., Zhu, S., Dabdub, D., Conley, A., West, M., Riemer, N., and Jorba, O.: Chemistry Across Multiple Phases (CAMP) version 1.0: an integrated multiphase chemistry model, Geosci. Model Dev., 15, 3663–3689, https://doi.org/10.5194/gmd-15-3663-2022, 2022.
 
 
 The binned and modal box model experiments are run as part of the CAMP testing suite. The results will be in the build folder under:
 
 ```
 data_run/CAMP_v1_paper/binned/out/
 data_run/CAMP_v1_paper/modal/out/
-```
+```

data/CAMP_v1_paper/binned/mock_monarch.F90

@@ -362,6 +362,9 @@ program mock_monarch
   ! finalize mpi
   call camp_mpi_finalize()
 
+  ! Free the interface and the solver
+  deallocate(camp_interface)
+
 contains
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

data/CAMP_v1_paper/modal/mock_monarch.F90

@@ -344,6 +344,9 @@ program mock_monarch
   ! finalize mpi
   call camp_mpi_finalize()
 
+  ! Free the interface and the solver
+  deallocate(camp_interface)
+
 contains
 
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

doc/camp_tutorial/boot_camp/part_1_code/box_model.F90

@@ -90,11 +90,12 @@ program box_model
                      camp_state%state_var( idx_O2  )
   end do
 
-  deallocate( camp_state )
-
 #ifdef CAMP_USE_MPI
   call camp_mpi_finalize( )
 #endif
 
+  deallocate( camp_core )
+  deallocate( camp_state )
+
 end program box_model
 !! [Solve and output]

doc/camp_tutorial/boot_camp/part_3_code/box_model.F90

@@ -108,6 +108,7 @@ program box_model
                      camp_state%state_var( idx_O2  )
   end do
 
+  deallocate( camp_core )
   deallocate( camp_state )
 
 #ifdef CAMP_USE_MPI

doc/camp_tutorial/boot_camp/part_4_code/box_model.F90

@@ -170,6 +170,7 @@ program box_model
 #endif
   !! [output]
 
+  deallocate( camp_core )
   deallocate( camp_state )
 
 end program box_model

doc/references.bib

@@ -1,3 +1,39 @@
+@article{Tie2003,
+author = {Tie, Xuexi and Emmons, Louisa and Horowitz, Larry and Brasseur, Guy and Ridley, Brian and Atlas, Elliot and Stround, Craig and Hess, Peter and Klonecki, Andrzej and Madronich, Sasha and Talbot, Robert and Dibb, Jack},
+title = {Effect of sulfate aerosol on tropospheric NOx and ozone budgets: Model simulations and TOPSE evidence},
+journal = {Journal of Geophysical Research: Atmospheres},
+volume = {108},
+number = {D4},
+pages = {},
+keywords = {tropospheric aerosol, NOx, ozone},
+doi = {https://doi.org/10.1029/2001JD001508},
+url = {https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1029/2001JD001508},
+eprint = {https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1029/2001JD001508},
+abstract = {The distributions of NOx and O3 are analyzed during TOPSE (Tropospheric Ozone Production about the Spring Equinox). In this study these data are compared with the calculations of a global chemical/transport model (Model for OZone And Related chemical Tracers (MOZART)). Specifically, the effect that hydrolysis of N2O5 on sulfate aerosols has on tropospheric NOx and O3 budgets is studied. The results show that without this heterogeneous reaction, the model significantly overestimates NOx concentrations at high latitudes of the Northern Hemisphere (NH) in winter and spring in comparison to the observations during TOPSE; with this reaction, modeled NOx concentrations are close to the measured values. This comparison provides evidence that the hydrolysis of N2O5 on sulfate aerosol plays an important role in controlling the tropospheric NOx and O3 budgets. The calculated reduction of NOx attributed to this reaction is 80 to 90\% in winter at high latitudes over North America. Because of the reduction of NOx, O3 concentrations are also decreased. The maximum O3 reduction occurs in spring although the maximum NOx reduction occurs in winter when photochemical O3 production is relatively low. The uncertainties related to uptake coefficient and aerosol loading in the model is analyzed. The analysis indicates that the changes in NOx due to these uncertainties are much smaller than the impact of hydrolysis of N2O5 on sulfate aerosol. The effect that hydrolysis of N2O5 on global NOx and O3 budgets are also assessed by the model. The results suggest that in the Northern Hemisphere, the average NOx budget decreases 50\% due to this reaction in winter and 5\% in summer. The average O3 budget is reduced by 8\% in winter and 6\% in summer. In the Southern Hemisphere (SH), the sulfate aerosol loading is significantly smaller than in the Northern Hemisphere. As a result, sulfate aerosol has little impact on NOx and O3 budgets of the Southern Hemisphere.},
+year = {2003}
+}
+@article{Wennberg2018,
+author = {Wennberg, Paul O. and Bates, Kelvin H. and Crounse, John D. and Dodson, Leah G. and McVay, Renee C. and Mertens, Laura A. and Nguyen, Tran B. and Praske, Eric and Schwantes, Rebecca H. and Smarte, Matthew D. and St Clair, Jason M. and Teng, Alexander P. and Zhang, Xuan and Seinfeld, John H.},
+title = {Gas-Phase Reactions of Isoprene and Its Major Oxidation Products},
+journal = {Chemical Reviews},
+volume = {118},
+number = {7},
+pages = {3337-3390},
+year = {2018},
+doi = {10.1021/acs.chemrev.7b00439},
+note ={PMID: 29522327},
+URL = {https://doi.org/10.1021/acs.chemrev.7b00439},
+eprint = {https://doi.org/10.1021/acs.chemrev.7b00439}
+}
+@techreport{JPL15,
+author = {J. B. Burkholder, S. P. Sander, J. Abbatt, J. R. Barker, R. E. Huie, C. E. Kolb, M. J. Kurylo, V. L. Orkin, D. M.
+Wilmouth, and P. H. Wine},
+title = {Chemical Kinetics and Photochemical Data for Use in Atmospheric Studies, Evaluation No. 18 JPL Publication 15-10},
+institution = {Jet Propulsion Laboratory},
+location = {Pasadena},
+year = {2015},
+url = {http://jpldataeval.jpl.nasa.gov}
+}
 @article{Ervens2003,
 abstract = {A detailed and extended chemical mechanism describing tropospheric aqueous phase chemistry (147 species and 438 reactions) is presented here as Chemical Aqueous Phase Radical Mechanism (CAPRAM) 2.4 (MODAC mechanism). The mechanism based on the former version 2.3 [ Herrmann et al., 2000] contains extended organic and transition metal chemistry and is formulated more explicitly based on a critical review of the literature. The aqueous chemistry has been coupled to the gas phase mechanism Regional Atmospheric Chemistry Modeling (RACM) [ Stockwell et al., 1997], and phase exchange accounted for using the resistance model of Schwartz [1986]. A method for estimating mass accommodation coefficients ({\&}{\#}945;) is described, which accounts for functional groups contained in a particular compound. A condensed version has also been developed to allow the use of CAPRAM 2.4 (MODAC mechanism) in higher-scale models. Here the reproducibility of the concentration levels of selected target species (i.e., NO x , S(IV), H2O2, NO3, OH, O3, and H+) within the limits of ± 5{\%} was used as a goal for eliminating insignificant reactions from the complete CAPRAM 2.4 (MODAC mechanism). This has been done using a range of initial conditions chosen to represent different atmospheric scenarios, and this produces a robust and concise set of reactions. The most interesting results are obtained using atmospheric conditions typical for an urban scenario, and the effects introduced by updating the aqueous phase chemistry are highlighted, in particular, with regard to radicals, redox cycling of transition metal ions and organic compounds. Finally, the reduced scheme has been incorporated into a one-dimensional (1-D) marine cloud model to demonstrate the applicability of this mechanism.},
 author = {Ervens, B.},

src/Jacobian.c

@@ -60,13 +60,13 @@ int jacobian_initialize(Jacobian *jac, unsigned int num_spec,
     return 0;
   }
   jac->production_partials =
-      (long double *)malloc(num_elem * sizeof(long double));
+      (double *)malloc(num_elem * sizeof(double));
   if (!jac->production_partials) {
     free(jac->col_ptrs);
     free(jac->row_ids);
     return 0;
   }
-  jac->loss_partials = (long double *)malloc(num_elem * sizeof(long double));
+  jac->loss_partials = (double *)malloc(num_elem * sizeof(double));
   if (!jac->loss_partials) {
     free(jac->col_ptrs);
     free(jac->row_ids);
@@ -171,13 +171,13 @@ unsigned int jacobian_build_matrix(Jacobian *jac) {
     exit(EXIT_FAILURE);
   }
   jac->production_partials =
-      (long double *)malloc(jac->num_elem * sizeof(long double));
+      (double *)malloc(jac->num_elem * sizeof(double));
   if (!jac->production_partials) {
     jacobian_free(jac);
     return 0;
   }
   jac->loss_partials =
-      (long double *)malloc(jac->num_elem * sizeof(long double));
+      (double *)malloc(jac->num_elem * sizeof(double));
   if (!jac->loss_partials) {
     jacobian_free(jac);
     return 0;
@@ -228,16 +228,16 @@ void jacobian_output(Jacobian jac, double *dest_array) {
   for (unsigned int i_col = 0; i_col < jac.num_spec; ++i_col) {
     for (unsigned int i_elem = jac.col_ptrs[i_col];
          i_elem < jac.col_ptrs[i_col + 1]; ++i_elem) {
-      long double drf_dy = jac.production_partials[i_elem];
-      long double drr_dy = jac.loss_partials[i_elem];
+      double drf_dy = jac.production_partials[i_elem];
+      double drr_dy = jac.loss_partials[i_elem];
       dest_array[i_elem] = drf_dy - drr_dy;
     }
   }
 }
 
 void jacobian_add_value(Jacobian jac, unsigned int elem_id,
                         unsigned int prod_or_loss,
-                        long double jac_contribution) {
+                        double jac_contribution) {
   if (prod_or_loss == JACOBIAN_PRODUCTION)
     jac.production_partials[elem_id] += jac_contribution;
   if (prod_or_loss == JACOBIAN_LOSS)

src/Jacobian.h

@@ -33,9 +33,8 @@ typedef struct {
   unsigned int num_elem;   // Number of potentially non-zero Jacobian elements
   unsigned int *col_ptrs;  // Index of start/end of each column in data array
   unsigned int *row_ids;   // Row id of each Jacobian element in data array
-  long double
-      *production_partials;    // Data array for productions rate partial derivs
-  long double *loss_partials;  // Data array for loss rate partial derivs
+  double *production_partials;    // Data array for productions rate partial derivs
+  double *loss_partials;  // Data array for loss rate partial derivs
   JacobianColumnElements *elements;  // Jacobian elements flagged for inclusion
 } Jacobian;
 
@@ -138,7 +137,7 @@ void jacobian_output(Jacobian jac, double *dest_array);
  */
 void jacobian_add_value(Jacobian jac, unsigned int elem_id,
                         unsigned int prod_or_loss,
-                        long double jac_contribution);
+                        double jac_contribution);
 
 /** \brief Prints the Jacobian structure
  *

src/aero_phase_solver.c

@@ -101,15 +101,15 @@ void aero_phase_get_mass__kg_m3(ModelData *model_data, int aero_phase_idx,
             [model_data->aero_phase_float_indices[aero_phase_idx]]);
 
   // Sum the mass and MW
-  long double l_mass = MINIMUM_MASS_;
-  long double moles = MINIMUM_MASS_ / MINIMUM_MW_;
+  double l_mass = MINIMUM_MASS_;
+  double moles = MINIMUM_MASS_ / MINIMUM_MW_;
   int i_jac = 0;
   for (int i_spec = 0; i_spec < NUM_STATE_VAR_; i_spec++) {
     if (SPEC_TYPE_(i_spec) == CHEM_SPEC_VARIABLE ||
         SPEC_TYPE_(i_spec) == CHEM_SPEC_CONSTANT ||
         SPEC_TYPE_(i_spec) == CHEM_SPEC_PSSA) {
       l_mass += state_var[i_spec];
-      moles += state_var[i_spec] / (long double)MW_(i_spec);
+      moles += state_var[i_spec] / (double)MW_(i_spec);
       if (jac_elem_mass) jac_elem_mass[i_jac] = 1.0L;
       if (jac_elem_MW) jac_elem_MW[i_jac] = 1.0L / MW_(i_spec);
       i_jac++;

src/camp_common.h

@@ -270,7 +270,6 @@ typedef struct {
   float rate_cells_gpu;
 #endif
   int use_cpu;
-  int nGPUs;
 
   void *cvode_mem;       // CVodeMem object
   ModelData model_data;  // Model data (used during initialization and solving)