Capgen cleanup

CODEOWNERS

@@ -3,7 +3,7 @@
 
 # These owners will be the default owners for everything in the repo.
 #*       @defunkt
-*       @climbfuji @grantfirl @gold2718 @mkavulich
+*       @climbfuji @gold2718 @dustinswales @mwaxmonsky @peverwhee @grantfirl @mkavulich
 
 # Order is important. The last matching pattern has the most precedence.
 # So if a pull request only touches javascript files, only these owners

doc/HelloWorld/CMakeLists.txt

@@ -1,4 +1,4 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.10)
 PROJECT(HelloWorld)
 ENABLE_LANGUAGE(Fortran)
 
@@ -166,7 +166,8 @@ list(APPEND DTABLE_CMD "${CCPP_CAP_FILES}/datatable.xml")
 list(APPEND DTABLE_CMD "--ccpp-files")
 list(APPEND DTABLE_CMD "--separator=\\;")
 string(REPLACE ";" " " DTABLE_STRING "${DTABLE_CMD}")
-MESSAGE(STATUS "Running: ${DTABLE_STRING}")
+string(STRIP ${DTABLE_STRING} DTABLE_STRING)
+MESSAGE(STATUS "Running: ${DTABLE_STRING};")
 EXECUTE_PROCESS(COMMAND ${DTABLE_CMD} OUTPUT_VARIABLE CCPP_CAPS
                 RESULT_VARIABLE RES
                 OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)

scripts/ccpp_capgen.py

@@ -15,6 +15,7 @@
 import logging
 import re
 # CCPP framework imports
+from ccpp_database_obj import CCPPDatabaseObj
 from ccpp_datafile import generate_ccpp_datatable
 from ccpp_suite import API
 from file_utils import check_for_writeable_file, remove_dir, replace_paths
@@ -559,7 +560,7 @@ def clean_capgen(cap_output_file, logger):
     set_log_level(logger, log_level)
 
 ###############################################################################
-def capgen(run_env):
+def capgen(run_env, return_db=False):
 ###############################################################################
     """Parse indicated host, scheme, and suite files.
     Generate code to allow host model to run indicated CCPP suites."""
@@ -650,6 +651,10 @@ def capgen(run_env):
     generate_ccpp_datatable(run_env, host_model, ccpp_api,
                             scheme_headers, scheme_tdict, host_files,
                             cap_filenames, kinds_file, src_dir)
+    if return_db:
+        return CCPPDatabaseObj(run_env, host_model=host_model, api=ccpp_api)
+    # end if
+    return None
 
 ###############################################################################
 def _main_func():
@@ -665,7 +670,7 @@ def _main_func():
     if framework_env.clean:
         clean_capgen(framework_env.datatable_file, framework_env.logger)
     else:
-        capgen(framework_env)
+        _ = capgen(framework_env)
     # end if (clean)
 
 ###############################################################################

scripts/ccpp_database_obj.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+"""
+Define the CCPPDatabaseObj object
+Object definition and methods to provide information from a run of capgen.
+"""
+
+from host_model import HostModel
+from ccpp_suite import API
+
+class CCPPDatabaseObjError(ValueError):
+    """Error class specific to CCPPDatabaseObj.
+    All uses of this error should be internal (i.e., programmer error,
+       not user error)."""
+
+    def __init__(self, message):
+        """Initialize this exception"""
+        super().__init__(message)
+
+class CCPPDatabaseObj:
+    """Object with data and methods to provide information from a run of capgen.
+    """
+
+    def __init__(self, run_env, host_model=None, api=None, database_file=None):
+        """Initialize this CCPPDatabaseObj.
+        If <database_file> is not None, all other inputs MUST be None and
+           the object is created from the database table created by capgen.
+        To initialize the object from an in-memory capgen run, ALL other
+           inputs MUST be passed (i.e., not None) and it is an error to pass
+           a value for <database_file>.
+        """
+
+        runtime_obj = all([host_model is not None, api is not None])
+        self.__host_model = None
+        self.__api = None
+        self.__database_file = None
+        if runtime_obj and database_file:
+            emsg = "Cannot provide both runtime arguments and database_file."
+        elif (not runtime_obj) and (not database_file):
+            emsg = "Must provide either database_file or all runtime arguments."
+        else:
+            emsg = ""
+        # end if
+        if emsg:
+            raise CCPPDatabaseObjError(f"ERROR: {emsg}")
+        # end if
+        if runtime_obj:
+            self.__host_model = host_model
+            self.__api = api
+        else:
+            self.db_from_file(run_env, database_file)
+        # end if
+
+    def db_from_file(self, run_env, database_file):
+        """Create the necessary internal data structures from a CCPP
+              datatable.xml file created by capgen.
+        """
+        metadata_tables = {}
+        host_name = "host"
+        self.__host_model = HostModel(metadata_tables, host_name, run_env)
+        self.__api = API(sdfs, host_model, scheme_headers, run_env)
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")
+
+    def host_model_dict(self):
+        """Return the host model dictionary for this CCPP DB object"""
+        if self.__host_model is not None:
+            return self.__host_model
+        # end if
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")
+
+    def suite_list(self):
+        """Return a list of suites built into the API"""
+        if self.__api is not None:
+            return list(self.__api.suites)
+        # end if
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")
+
+    def constituent_dictionary(self, suite):
+        """Return the constituent dictionary for <suite>"""
+        return suite.constituent_dictionary()
+
+    def call_list(self, phase):
+        """Return the API call list for <phase>"""
+        if self.__api is not None:
+            return self.__api.call_list(phase)
+        # end if
+        raise CCPPDatabaseObjError("ERROR: <database_file> not supported")

scripts/ccpp_datafile.py

@@ -27,10 +27,6 @@
 from parse_tools import read_xml_file, PrettyElementTree
 from suite_objects import VerticalLoop, Subcycle
 
-# Find python version
-PY3 = sys.version_info[0] > 2
-PYSUBVER = sys.version_info[1]
-
 # Global data
 _INDENT_STR = "  "
 
@@ -652,7 +648,7 @@ def _new_var_entry(parent, var, full_entry=True):
     """Create a variable sub-element of <parent> with information from <var>.
     If <full_entry> is False, only include standard name and intent.
     """
-    prop_list = ["intent"]
+    prop_list = ["intent", "local_name"]
     if full_entry:
         prop_list.extend(["allocatable", "active", "default_value",
                           "diagnostic_name", "diagnostic_name_fixed",
@@ -671,9 +667,13 @@ def _new_var_entry(parent, var, full_entry=True):
     if full_entry:
         dims = var.get_dimensions()
         if dims:
-            dim_entry = ET.SubElement(ventry, "dimensions")
-            dim_entry.text = " ".join(dims)
+            v_entry = ET.SubElement(ventry, "dimensions")
+            v_entry.text = " ".join(dims)
         # end if
+        v_entry = ET.SubElement(ventry, "source_type")
+        v_entry.text = var.source.ptype.lower()
+        v_entry = ET.SubElement(ventry, "source_name")
+        v_entry.text = var.source.name.lower()
     # end if
 
 ###############################################################################

scripts/ccpp_suite.py

@@ -17,8 +17,7 @@
 from fortran_tools import FortranWriter
 from framework_env import CCPPFrameworkEnv
 from metavar import Var, VarDictionary, ccpp_standard_var
-from metavar import CCPP_CONSTANT_VARS, CCPP_LOOP_VAR_STDNAMES
-from parse_tools import ParseContext, ParseSource, context_string
+from parse_tools import ParseContext, ParseSource
 from parse_tools import ParseInternalError, CCPPError
 from parse_tools import read_xml_file, validate_xml_file, find_schema_version
 from parse_tools import init_log, set_log_to_null
@@ -31,12 +30,12 @@
 ###############################################################################
 
 # Source for internally generated variables.
-_API_SOURCE_NAME = "CCPP_API"
+API_SOURCE_NAME = "CCPP_API"
 # Use the constituent source type for consistency
 _API_SUITE_VAR_NAME = ConstituentVarDict.constitutent_source_type()
 _API_SCHEME_VAR_NAME = "scheme"
 _API_CONTEXT = ParseContext(filename="ccpp_suite.py")
-_API_SOURCE = ParseSource(_API_SOURCE_NAME, _API_SCHEME_VAR_NAME, _API_CONTEXT)
+_API_SOURCE = ParseSource(API_SOURCE_NAME, _API_SCHEME_VAR_NAME, _API_CONTEXT)
 _API_LOGGING = init_log('ccpp_suite')
 set_log_to_null(_API_LOGGING)
 _API_DUMMY_RUN_ENV = CCPPFrameworkEnv(_API_LOGGING,
@@ -299,7 +298,7 @@ def find_variable(self, standard_name=None, source_var=None,
                     # Remove this entry to avoid looping back here
                     del self.__gvar_stdnames[standard_name]
                     # Let everyone know this is now a Suite variable
-                    var.source = ParseSource(_API_SOURCE_NAME,
+                    var.source = ParseSource(API_SOURCE_NAME,
                                              _API_SUITE_VAR_NAME,
                                              var.context)
                     self.add_variable(var, self.__run_env)
@@ -730,9 +729,8 @@ def write_var_set_loop(ofile, varlist_name, var_list, indent,
 
     def write_suite_part_list_sub(self, ofile, errmsg_name, errcode_name):
         """Write the suite-part list subroutine"""
-        oline = "suite_name, part_list, {errmsg}, {errcode}"
-        inargs = oline.format(errmsg=errmsg_name, errcode=errcode_name)
-        ofile.write("\nsubroutine {}({})".format(API.__part_fname, inargs), 1)
+        inargs = f"suite_name, part_list, {errmsg_name}, {errcode_name}"
+        ofile.write(f"subroutine {API.__part_fname}({inargs})", 1)
         oline = "character(len=*),              intent(in)  :: suite_name"
         ofile.write(oline, 2)
         oline = "character(len=*), allocatable, intent(out) :: part_list(:)"
@@ -741,22 +739,22 @@ def write_suite_part_list_sub(self, ofile, errmsg_name, errcode_name):
         self._errcode_var.write_def(ofile, 2, self)
         else_str = ''
         ename = self._errcode_var.get_prop_value('local_name')
-        ofile.write("{} = 0".format(ename), 2)
+        ofile.write(f"{ename} = 0", 2)
         ename = self._errmsg_var.get_prop_value('local_name')
-        ofile.write("{} = ''".format(ename), 2)
+        ofile.write(f"{ename} = ''", 2)
         for suite in self.suites:
             oline = "{}if(trim(suite_name) == '{}') then"
             ofile.write(oline.format(else_str, suite.name), 2)
             API.write_var_set_loop(ofile, 'part_list', suite.part_list(), 3)
             else_str = 'else '
         # end for
         ofile.write("else", 2)
-        emsg = "write({errmsg}, '(3a)')".format(errmsg=errmsg_name)
+        emsg = f"write({errmsg_name}, '(3a)')"
         emsg += "'No suite named ', trim(suite_name), ' found'"
         ofile.write(emsg, 3)
-        ofile.write("{errcode} = 1".format(errcode=errcode_name), 3)
+        ofile.write(f"{errcode_name} = 1", 3)
         ofile.write("end if", 2)
-        ofile.write("end subroutine {}".format(API.__part_fname), 1)
+        ofile.write(f"end subroutine {API.__part_fname}", 1)
 
     def write_req_vars_sub(self, ofile, errmsg_name, errcode_name):
         """Write the required variables subroutine"""
@@ -807,9 +805,9 @@ def write_req_vars_sub(self, ofile, errmsg_name, errcode_name):
             parent = suite.parent
             # Collect all the suite variables
             oline = "{}if(trim(suite_name) == '{}') then"
-            input_vars = [set(), set(), set()] # leaves, arrrays, leaf elements
-            inout_vars = [set(), set(), set()] # leaves, arrrays, leaf elements
-            output_vars = [set(), set(), set()] # leaves, arrrays, leaf elements
+            input_vars = [set(), set(), set()] # leaves, arrays, leaf elements
+            inout_vars = [set(), set(), set()] # leaves, arrays, leaf elements
+            output_vars = [set(), set(), set()] # leaves, arrays, leaf elements
             for part in suite.groups:
                 for var in part.call_list.variable_list():
                     stdname = var.get_prop_value("standard_name")

scripts/common.py

@@ -169,14 +169,7 @@ def escape_tex(text):
 
 def isstring(s):
     """Return true if a variable is a string"""
-    # We use Python 3
-    if (sys.version_info.major == 3):
-        return isinstance(s, str)
-    # We use Python 2
-    elif (sys.version_info.major == 2):
-        return isinstance(s, basestring)
-    else:
-        raise Exception('Unknown Python version')
+    return isinstance(s, str)
 
 def string_to_python_identifier(string):
     """Replaces forbidden characters in strings with standard substitutions

scripts/constituents.py

@@ -398,7 +398,7 @@ def constituent_module_name(self):
         if not ((self.parent is not None) and
                 hasattr(self.parent.parent, "constituent_module")):
             emsg = "ConstituentVarDict parent not HostModel?"
-            emsg += "\nparent is '{}'".format(type(self.parent.parent))
+            emsg += f"\nparent is '{type_name(self.parent.parent)}'"
             raise ParseInternalError(emsg)
         # end if
         return self.parent.parent.constituent_module

scripts/file_utils.py

@@ -13,11 +13,6 @@
 import os
 # CCPP framework imports
 from parse_tools import CCPPError, ParseInternalError
-#XXgoldyXX: v Crap required to support python 2
-import sys
-# Find python version
-PY3 = sys.version_info[0] > 2
-#XXgoldyXX: ^ Crap required to support python 2
 
 # Standardize name of generated kinds file and module
 KINDS_MODULE = 'ccpp_kinds'
@@ -300,13 +295,7 @@ def move_modified_files(src_dir, dest_dir, overwrite=False, remove_src=False):
                 fmove = True
             # end if
             if fmove:
-#XXgoldyXX: v Crap required to support python 2
-                if PY3:
-                    os.replace(src_path, dest_path)
-                else:
-                    os.rename(src_path, dest_path)
-                # end if
-#XXgoldyXX: ^ Crap required to support python 2
+                os.replace(src_path, dest_path)
             else:
                 os.remove(src_path)
             # end if

scripts/fortran_tools/parse_fortran.py

@@ -532,7 +532,8 @@ def class_match(cls, line):
     @classmethod
     def type_def_line(cls, line):
         """Return a type information if <line> represents the start
-        of a type definition"""
+              of a type definition.
+           Otherwise, return None"""
         type_def = None
         if not cls.type_match(line):
             if '!' in line:
@@ -629,7 +630,8 @@ def ftype_factory(line, context):
 def fortran_type_definition(line):
 ########################################################################
     """Return a type information if <line> represents the start
-        of a type definition"""
+        of a type definition.
+       Otherwise, return None."""
     return FtypeTypeDecl.type_def_line(line)
 
 ########################################################################
@@ -720,8 +722,8 @@ def parse_fortran_var_decl(line, source, run_env):
             varprops = Ftype.parse_attr_specs(elements[0].strip(), context)
             for prop in varprops:
                 if prop[0:6] == 'intent':
-                    if source.type != 'scheme':
-                        typ = source.type
+                    if source.ptype != 'scheme':
+                        typ = source.ptype
                         errmsg = 'Invalid variable declaration, {}, intent'
                         errmsg = errmsg + ' not allowed in {} variable'
                         if run_env.logger is not None:

scripts/framework_env.py

@@ -103,8 +103,8 @@ def __init__(self, logger, ndict=None, verbose=0, clean=False,
             # String of definitions, separated by spaces
             preproc_list = [x.strip() for x in preproc_defs.split(' ') if x]
         else:
-            wmsg = "Error: Bad preproc list type, '{}'"
-            emsg += esep + wmsg.format(type(preproc_defs))
+            wmsg = f"Error: Bad preproc list type, '{type_name(preproc_defs)}'"
+            emsg += esep + wmsg
             esep = '\n'
         # end if
         # Turn the list into a dictionary