Constituent updates

.github/workflows/capgen_unit_tests.yaml

@@ -0,0 +1,22 @@
+name: Capgen Unit Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  push:
+    branches:
+      #Trigger workflow on push to any branch or branch heirarchy:
+      - '**'
+
+jobs:
+  unit_tests:
+    if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || github.repository == 'NCAR/ccpp-framework'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: update repos and install dependencies
+      run: sudo apt-get update && sudo apt-get install -y build-essential gfortran cmake python3 git
+    - name: Run unit tests
+      run: cd test && ./run_tests.sh
+

doc/HelloWorld/CMakeLists.txt

@@ -1,4 +1,4 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.10)
 PROJECT(HelloWorld)
 ENABLE_LANGUAGE(Fortran)
 
@@ -166,7 +166,8 @@ list(APPEND DTABLE_CMD "${CCPP_CAP_FILES}/datatable.xml")
 list(APPEND DTABLE_CMD "--ccpp-files")
 list(APPEND DTABLE_CMD "--separator=\\;")
 string(REPLACE ";" " " DTABLE_STRING "${DTABLE_CMD}")
-MESSAGE(STATUS "Running: ${DTABLE_STRING}")
+string(STRIP ${DTABLE_STRING} DTABLE_STRING)
+MESSAGE(STATUS "Running: ${DTABLE_STRING};")
 EXECUTE_PROCESS(COMMAND ${DTABLE_CMD} OUTPUT_VARIABLE CCPP_CAPS
                 RESULT_VARIABLE RES
                 OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)

scripts/ccpp_capgen.py

@@ -15,6 +15,7 @@
 import logging
 import re
 # CCPP framework imports
+from ccpp_database_obj import CCPPDatabaseObj
 from ccpp_datafile import generate_ccpp_datatable
 from ccpp_suite import API
 from file_utils import check_for_writeable_file, remove_dir, replace_paths
@@ -26,11 +27,13 @@
 from host_model import HostModel
 from metadata_table import parse_metadata_file, SCHEME_HEADER_TYPE
 from parse_tools import init_log, set_log_level, context_string
+from parse_tools import register_fortran_ddt_name
 from parse_tools import CCPPError, ParseInternalError
 
 ## Capture the Framework root
-__SCRIPT_PATH = os.path.dirname(__file__)
-__FRAMEWORK_ROOT = os.path.abspath(os.path.join(__SCRIPT_PATH, os.pardir))
+_SCRIPT_PATH = os.path.dirname(__file__)
+_FRAMEWORK_ROOT = os.path.abspath(os.path.join(_SCRIPT_PATH, os.pardir))
+_SRC_ROOT = os.path.join(_FRAMEWORK_ROOT, "src")
 ## Init this now so that all Exceptions can be trapped
 _LOGGER = init_log(os.path.basename(__file__))
 
@@ -43,6 +46,11 @@
 ## Metadata table types where order is significant
 _ORDERED_TABLE_TYPES = [SCHEME_HEADER_TYPE]
 
+## CCPP Framework supported DDT types
+_CCPP_FRAMEWORK_DDT_TYPES = ["ccpp_hash_table_t",
+                             "ccpp_hashable_t",
+                             "ccpp_hashable_char_t"]
+
 ###############################################################################
 def delete_pathnames_from_file(capfile, logger):
 ###############################################################################
@@ -559,7 +567,7 @@ def clean_capgen(cap_output_file, logger):
     set_log_level(logger, log_level)
 
 ###############################################################################
-def capgen(run_env):
+def capgen(run_env, return_db=False):
 ###############################################################################
     """Parse indicated host, scheme, and suite files.
     Generate code to allow host model to run indicated CCPP suites."""
@@ -578,12 +586,22 @@ def capgen(run_env):
         # Try to create output_dir (let it crash if it fails)
         os.makedirs(run_env.output_dir)
     # end if
+    # Pre-register base CCPP DDT types
+    for ddt_name in _CCPP_FRAMEWORK_DDT_TYPES:
+        register_fortran_ddt_name(ddt_name)
+    # end for
+    src_dir = os.path.join(_FRAMEWORK_ROOT, "src")
     host_files = run_env.host_files
     host_name = run_env.host_name
     scheme_files = run_env.scheme_files
     # We need to create three lists of files, hosts, schemes, and SDFs
     host_files = create_file_list(run_env.host_files, ['meta'], 'Host',
                                   run_env.logger)
+    # The host model needs to know about the constituents module
+    const_mod = os.path.join(_SRC_ROOT, "ccpp_constituent_prop_mod.meta")
+    if const_mod not in host_files:
+        host_files.append(const_mod)
+    # end if
     scheme_files = create_file_list(run_env.scheme_files, ['meta'],
                                     'Scheme', run_env.logger)
     sdfs = create_file_list(run_env.suites, ['xml'], 'Suite', run_env.logger)
@@ -595,10 +613,18 @@ def capgen(run_env):
     # First up, handle the host files
     host_model = parse_host_model_files(host_files, host_name, run_env)
     # Next, parse the scheme files
+    # We always need to parse the ccpp_constituent_prop_ptr_t DDT
+    ##XXgoldyXX: Should this be in framework_env.py?
+    const_prop_mod = os.path.join(src_dir, "ccpp_constituent_prop_mod.meta")
+    if const_prop_mod not in scheme_files:
+        scheme_files= [const_prop_mod] + scheme_files
+    # end if
     scheme_headers, scheme_tdict = parse_scheme_files(scheme_files, run_env)
-    ddts = host_model.ddt_lib.keys()
-    if ddts and run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG):
-        run_env.logger.debug("DDT definitions = {}".format(ddts))
+    if run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG):
+        ddts = host_model.ddt_lib.keys()
+        if ddts:
+            run_env.logger.debug("DDT definitions = {}".format(ddts))
+        # end if
     # end if
     plist = host_model.prop_list('local_name')
     if run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG):
@@ -628,7 +654,8 @@ def capgen(run_env):
     cap_filenames = ccpp_api.write(outtemp_dir, run_env)
     if run_env.generate_host_cap:
         # Create a cap file
-        host_files = [write_host_cap(host_model, ccpp_api,
+        cap_module = host_model.ccpp_cap_name()
+        host_files = [write_host_cap(host_model, ccpp_api, cap_module,
                                      outtemp_dir, run_env)]
     else:
         host_files = list()
@@ -646,10 +673,13 @@ def capgen(run_env):
     # end if
     # Finally, create the database of generated files and caps
     # This can be directly in output_dir because it will not affect dependencies
-    src_dir = os.path.join(__FRAMEWORK_ROOT, "src")
     generate_ccpp_datatable(run_env, host_model, ccpp_api,
                             scheme_headers, scheme_tdict, host_files,
                             cap_filenames, kinds_file, src_dir)
+    if return_db:
+        return CCPPDatabaseObj(run_env, host_model=host_model, api=ccpp_api)
+    # end if
+    return None
 
 ###############################################################################
 def _main_func():
@@ -665,7 +695,7 @@ def _main_func():
     if framework_env.clean:
         clean_capgen(framework_env.datatable_file, framework_env.logger)
     else:
-        capgen(framework_env)
+        _ = capgen(framework_env)
     # end if (clean)
 
 ###############################################################################

scripts/ccpp_database_obj.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+"""
+Define the CCPPDatabaseObj object
+Object definition and methods to provide information from a run of capgen.
+"""
+
+from host_model import HostModel
+from ccpp_suite import API
+
+class CCPPDatabaseObjError(ValueError):
+    """Error class specific to CCPPDatabaseObj.
+    All uses of this error should be internal (i.e., programmer error,
+       not user error)."""
+
+    def __init__(self, message):
+        """Initialize this exception"""
+        super().__init__(message)
+
+class CCPPDatabaseObj:
+    """Ojbect with data and methods to provide information from a run of capgen.
+    """
+
+    def __init__(self, run_env, host_model=None, api=None, database_file=None):
+        """Initialize this CCPPDatabaseObj.
+        If <database_file> is not None, all other inputs MUST be None and
+           the object is created from the database table created by capgen.
+        To initialize the object from an in-memory capgen run, ALL other
+           inputs MUST be passed (i.e., not None) and it is an error to pass
+           a value for <database_file>.
+        """
+
+        runtime_obj = all([host_model is not None, api is not None])
+        self.__host_model = None
+        self.__api = None
+        self.__database_file = None
+        if runtime_obj and database_file:
+            emsg = "Cannot provide both runtime arguments and database_file."
+        elif (not runtime_obj) and (not database_file):
+            emsg = "Must provide either database_file or all runtime arguments."
+        else:
+            emsg = ""
+        # end if
+        if emsg:
+            raise CCPPDatabaseObjError(f"ERROR: {emsg}")
+        # end if
+        if runtime_obj:
+            self.__host_model = host_model
+            self.__api = api
+        else:
+            self.db_from_file(run_env, database_file)
+        # end if
+
+    def db_from_file(self, run_env, database_file):
+        """Create the necessary internal data structures from a CCPP
+              datatable.xml file created by capgen.
+        """
+        metadata_tables = {}
+        host_name = "cam"
+        self.__host_model = HostModel(metadata_tables, host_name, run_env)
+        self.__api = API(sdfs, host_model, scheme_headers, run_env)
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")
+
+    def host_model_dict(self):
+        """Return the host model dictionary for this CCPP DB object"""
+        if self.__host_model is not None:
+            return self.__host_model
+        # end if
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")
+
+    def suite_list(self):
+        """Return a list of suites built into the API"""
+        if self.__api is not None:
+            return list(self.__api.suites)
+        # end if
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")
+
+    def constituent_dictionary(self, suite):
+        """Return the constituent dictionary for <suite>"""
+        return suite.constituent_dictionary()
+
+    def call_list(self, phase):
+        """Return the API call list for <phase>"""
+        if self.__api is not None:
+            return self.__api.call_list(phase)
+        # end if
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")

scripts/ccpp_datafile.py

@@ -652,12 +652,13 @@ def _new_var_entry(parent, var, full_entry=True):
     """Create a variable sub-element of <parent> with information from <var>.
     If <full_entry> is False, only include standard name and intent.
     """
-    prop_list = ["intent"]
+    prop_list = ["intent", "local_name"]
     if full_entry:
         prop_list.extend(["allocatable", "active", "default_value",
                           "diagnostic_name", "diagnostic_name_fixed",
                           "kind", "persistence", "polymorphic", "protected",
-                          "state_variable", "type", "units"])
+                          "state_variable", "type", "units", "molar_mass",
+                          "advected"])
         prop_list.extend(Var.constituent_property_names())
     # end if
     ventry = ET.SubElement(parent, "var")
@@ -671,9 +672,13 @@ def _new_var_entry(parent, var, full_entry=True):
     if full_entry:
         dims = var.get_dimensions()
         if dims:
-            dim_entry = ET.SubElement(ventry, "dimensions")
-            dim_entry.text = " ".join(dims)
+            v_entry = ET.SubElement(ventry, "dimensions")
+            v_entry.text = " ".join(dims)
         # end if
+        v_entry = ET.SubElement(ventry, "source_type")
+        v_entry.text = var.source.ptype.lower()
+        v_entry = ET.SubElement(ventry, "source_name")
+        v_entry.text = var.source.name.lower()
     # end if
 
 ###############################################################################

scripts/ccpp_state_machine.py

@@ -3,11 +3,11 @@
 # CCPP framework imports
 from state_machine import StateMachine
 
-_INIT_ST = r"(?:(?i)init(?:ial(?:ize)?)?)"
-_FINAL_ST = r"(?:(?i)final(?:ize)?)"
-_RUN_ST = r"(?:(?i)run)"
-_TS_INIT_ST = r"(?:(?i)timestep_init(?:ial(?:ize)?)?)"
-_TS_FINAL_ST = r"(?:(?i)timestep_final(?:ize)?)"
+_INIT_ST = r"(?:init(?:ial(?:ize)?)?)"
+_FINAL_ST = r"(?:final(?:ize)?)"
+_RUN_ST = r"(?:run)"
+_TS_INIT_ST = r"(?:timestep_init(?:ial(?:ize)?)?)"
+_TS_FINAL_ST = r"(?:timestep_final(?:ize)?)"
 
 # Allowed CCPP transitions
 # pylint: disable=bad-whitespace