From 6a6cc0d6d0d45b726488dfb461bcce78cd672c0a Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Mon, 21 Aug 2023 13:34:17 -0600 Subject: [PATCH 1/8] initial stab at cleanup-only PR --- doc/HelloWorld/CMakeLists.txt | 5 +- scripts/ccpp_capgen.py | 14 +- scripts/ccpp_database_obj.py | 87 +++++++++ scripts/ccpp_datafile.py | 14 +- scripts/ccpp_state_machine.py | 10 +- scripts/ccpp_suite.py | 34 ++-- scripts/common.py | 9 +- scripts/ddt_library.py | 104 +++++----- scripts/file_utils.py | 13 +- scripts/fortran_tools/fortran_write.py | 200 ++++++++++++++++---- scripts/fortran_tools/parse_fortran.py | 10 +- scripts/fortran_tools/parse_fortran_file.py | 52 +++-- scripts/framework_env.py | 4 +- scripts/metadata_table.py | 38 ++-- scripts/metavar.py | 22 +-- scripts/parse_tools/__init__.py | 3 +- scripts/parse_tools/parse_checkers.py | 2 +- scripts/parse_tools/parse_object.py | 8 +- scripts/parse_tools/parse_source.py | 83 ++++---- scripts/parse_tools/xml_tools.py | 158 +++++++--------- scripts/state_machine.py | 7 +- scripts/suite_objects.py | 8 +- 22 files changed, 541 insertions(+), 344 deletions(-) create mode 100644 scripts/ccpp_database_obj.py diff --git a/doc/HelloWorld/CMakeLists.txt b/doc/HelloWorld/CMakeLists.txt index 4b0cb208..7b480203 100644 --- a/doc/HelloWorld/CMakeLists.txt +++ b/doc/HelloWorld/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +CMAKE_MINIMUM_REQUIRED(VERSION 3.10) PROJECT(HelloWorld) ENABLE_LANGUAGE(Fortran) @@ -166,7 +166,8 @@ list(APPEND DTABLE_CMD "${CCPP_CAP_FILES}/datatable.xml") list(APPEND DTABLE_CMD "--ccpp-files") list(APPEND DTABLE_CMD "--separator=\\;") string(REPLACE ";" " " DTABLE_STRING "${DTABLE_CMD}") -MESSAGE(STATUS "Running: ${DTABLE_STRING}") +string(STRIP ${DTABLE_STRING} DTABLE_STRING) +MESSAGE(STATUS "Running: ${DTABLE_STRING};") EXECUTE_PROCESS(COMMAND ${DTABLE_CMD} OUTPUT_VARIABLE CCPP_CAPS RESULT_VARIABLE RES OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE) diff --git a/scripts/ccpp_capgen.py b/scripts/ccpp_capgen.py index 7a12729a..d28bfd6a 100755 --- a/scripts/ccpp_capgen.py +++ b/scripts/ccpp_capgen.py @@ -15,6 +15,7 @@ import logging import re # CCPP framework imports +from ccpp_database_obj import CCPPDatabaseObj from ccpp_datafile import generate_ccpp_datatable from ccpp_suite import API from file_utils import check_for_writeable_file, remove_dir, replace_paths @@ -30,7 +31,7 @@ ## Capture the Framework root __SCRIPT_PATH = os.path.dirname(__file__) -__FRAMEWORK_ROOT = os.path.abspath(os.path.join(__SCRIPT_PATH, os.pardir)) +__FRAMEWORK_ROOT = os.path.abspath(os.path.join(_SCRIPT_PATH, os.pardir)) ## Init this now so that all Exceptions can be trapped _LOGGER = init_log(os.path.basename(__file__)) @@ -559,7 +560,7 @@ def clean_capgen(cap_output_file, logger): set_log_level(logger, log_level) ############################################################################### -def capgen(run_env): +def capgen(run_env, return_db=False): ############################################################################### """Parse indicated host, scheme, and suite files. Generate code to allow host model to run indicated CCPP suites.""" @@ -628,7 +629,8 @@ def capgen(run_env): cap_filenames = ccpp_api.write(outtemp_dir, run_env) if run_env.generate_host_cap: # Create a cap file - host_files = [write_host_cap(host_model, ccpp_api, + cap_module = host_model.ccpp_cap_name() + host_files = [write_host_cap(host_model, ccpp_api, cap_module, outtemp_dir, run_env)] else: host_files = list() @@ -650,6 +652,10 @@ def capgen(run_env): generate_ccpp_datatable(run_env, host_model, ccpp_api, scheme_headers, scheme_tdict, host_files, cap_filenames, kinds_file, src_dir) + if return_db: + return CCPPDatabaseObj(run_env, host_model=host_model, api=ccpp_api) + # end if + return None ############################################################################### def _main_func(): @@ -665,7 +671,7 @@ def _main_func(): if framework_env.clean: clean_capgen(framework_env.datatable_file, framework_env.logger) else: - capgen(framework_env) + _ = capgen(framework_env) # end if (clean) ############################################################################### diff --git a/scripts/ccpp_database_obj.py b/scripts/ccpp_database_obj.py new file mode 100644 index 00000000..31aa450c --- /dev/null +++ b/scripts/ccpp_database_obj.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +""" +Define the CCPPDatabaseObj object +Object definition and methods to provide information from a run of capgen. +""" + +from host_model import HostModel +from ccpp_suite import API + +class CCPPDatabaseObjError(ValueError): + """Error class specific to CCPPDatabaseObj. + All uses of this error should be internal (i.e., programmer error, + not user error).""" + + def __init__(self, message): + """Initialize this exception""" + super().__init__(message) + +class CCPPDatabaseObj: + """Ojbect with data and methods to provide information from a run of capgen. + """ + + def __init__(self, run_env, host_model=None, api=None, database_file=None): + """Initialize this CCPPDatabaseObj. + If is not None, all other inputs MUST be None and + the object is created from the database table created by capgen. + To initialize the object from an in-memory capgen run, ALL other + inputs MUST be passed (i.e., not None) and it is an error to pass + a value for . + """ + + runtime_obj = all([host_model is not None, api is not None]) + self.__host_model = None + self.__api = None + self.__database_file = None + if runtime_obj and database_file: + emsg = "Cannot provide both runtime arguments and database_file." + elif (not runtime_obj) and (not database_file): + emsg = "Must provide either database_file or all runtime arguments." + else: + emsg = "" + # end if + if emsg: + raise CCPPDatabaseObjError(f"ERROR: {emsg}") + # end if + if runtime_obj: + self.__host_model = host_model + self.__api = api + else: + self.db_from_file(run_env, database_file) + # end if + + def db_from_file(self, run_env, database_file): + """Create the necessary internal data structures from a CCPP + datatable.xml file created by capgen. + """ + metadata_tables = {} + host_name = "cam" + self.__host_model = HostModel(metadata_tables, host_name, run_env) + self.__api = API(sdfs, host_model, scheme_headers, run_env) + raise CCPPDatabaseObjError("ERROR: not supported") + + def host_model_dict(self): + """Return the host model dictionary for this CCPP DB object""" + if self.__host_model is not None: + return self.__host_model + # end if + raise CCPPDatabaseObjError("ERROR: not supported") + + def suite_list(self): + """Return a list of suites built into the API""" + if self.__api is not None: + return list(self.__api.suites) + # end if + raise CCPPDatabaseObjError("ERROR: not supported") + + def constituent_dictionary(self, suite): + """Return the constituent dictionary for """ + return suite.constituent_dictionary() + + def call_list(self, phase): + """Return the API call list for """ + if self.__api is not None: + return self.__api.call_list(phase) + # end if + raise CCPPDatabaseObjError("ERROR: not supported") diff --git a/scripts/ccpp_datafile.py b/scripts/ccpp_datafile.py index 158d9dec..562a8f4b 100755 --- a/scripts/ccpp_datafile.py +++ b/scripts/ccpp_datafile.py @@ -27,10 +27,6 @@ from parse_tools import read_xml_file, PrettyElementTree from suite_objects import VerticalLoop, Subcycle -# Find python version -PY3 = sys.version_info[0] > 2 -PYSUBVER = sys.version_info[1] - # Global data _INDENT_STR = " " @@ -652,7 +648,7 @@ def _new_var_entry(parent, var, full_entry=True): """Create a variable sub-element of with information from . If is False, only include standard name and intent. """ - prop_list = ["intent"] + prop_list = ["intent", "local_name"] if full_entry: prop_list.extend(["allocatable", "active", "default_value", "diagnostic_name", "diagnostic_name_fixed", @@ -671,9 +667,13 @@ def _new_var_entry(parent, var, full_entry=True): if full_entry: dims = var.get_dimensions() if dims: - dim_entry = ET.SubElement(ventry, "dimensions") - dim_entry.text = " ".join(dims) + v_entry = ET.SubElement(ventry, "dimensions") + v_entry.text = " ".join(dims) # end if + v_entry = ET.SubElement(ventry, "source_type") + v_entry.text = var.source.ptype.lower() + v_entry = ET.SubElement(ventry, "source_name") + v_entry.text = var.source.name.lower() # end if ############################################################################### diff --git a/scripts/ccpp_state_machine.py b/scripts/ccpp_state_machine.py index 30540fc9..0de2c7bd 100644 --- a/scripts/ccpp_state_machine.py +++ b/scripts/ccpp_state_machine.py @@ -3,11 +3,11 @@ # CCPP framework imports from state_machine import StateMachine -_INIT_ST = r"(?:(?i)init(?:ial(?:ize)?)?)" -_FINAL_ST = r"(?:(?i)final(?:ize)?)" -_RUN_ST = r"(?:(?i)run)" -_TS_INIT_ST = r"(?:(?i)timestep_init(?:ial(?:ize)?)?)" -_TS_FINAL_ST = r"(?:(?i)timestep_final(?:ize)?)" +_INIT_ST = r"(?:init(?:ial(?:ize)?)?)" +_FINAL_ST = r"(?:final(?:ize)?)" +_RUN_ST = r"(?:run)" +_TS_INIT_ST = r"(?:timestep_init(?:ial(?:ize)?)?)" +_TS_FINAL_ST = r"(?:timestep_final(?:ize)?)" # Allowed CCPP transitions # pylint: disable=bad-whitespace diff --git a/scripts/ccpp_suite.py b/scripts/ccpp_suite.py index 8b9a1aeb..ba8f8609 100644 --- a/scripts/ccpp_suite.py +++ b/scripts/ccpp_suite.py @@ -17,8 +17,7 @@ from fortran_tools import FortranWriter from framework_env import CCPPFrameworkEnv from metavar import Var, VarDictionary, ccpp_standard_var -from metavar import CCPP_CONSTANT_VARS, CCPP_LOOP_VAR_STDNAMES -from parse_tools import ParseContext, ParseSource, context_string +from parse_tools import ParseContext, ParseSource from parse_tools import ParseInternalError, CCPPError from parse_tools import read_xml_file, validate_xml_file, find_schema_version from parse_tools import init_log, set_log_to_null @@ -31,12 +30,12 @@ ############################################################################### # Source for internally generated variables. -_API_SOURCE_NAME = "CCPP_API" +API_SOURCE_NAME = "CCPP_API" # Use the constituent source type for consistency _API_SUITE_VAR_NAME = ConstituentVarDict.constitutent_source_type() _API_SCHEME_VAR_NAME = "scheme" _API_CONTEXT = ParseContext(filename="ccpp_suite.py") -_API_SOURCE = ParseSource(_API_SOURCE_NAME, _API_SCHEME_VAR_NAME, _API_CONTEXT) +_API_SOURCE = ParseSource(API_SOURCE_NAME, _API_SCHEME_VAR_NAME, _API_CONTEXT) _API_LOGGING = init_log('ccpp_suite') set_log_to_null(_API_LOGGING) _API_DUMMY_RUN_ENV = CCPPFrameworkEnv(_API_LOGGING, @@ -299,7 +298,7 @@ def find_variable(self, standard_name=None, source_var=None, # Remove this entry to avoid looping back here del self.__gvar_stdnames[standard_name] # Let everyone know this is now a Suite variable - var.source = ParseSource(_API_SOURCE_NAME, + var.source = ParseSource(API_SOURCE_NAME, _API_SUITE_VAR_NAME, var.context) self.add_variable(var, self.__run_env) @@ -730,9 +729,9 @@ def write_var_set_loop(ofile, varlist_name, var_list, indent, def write_suite_part_list_sub(self, ofile, errmsg_name, errcode_name): """Write the suite-part list subroutine""" - oline = "suite_name, part_list, {errmsg}, {errcode}" - inargs = oline.format(errmsg=errmsg_name, errcode=errcode_name) - ofile.write("\nsubroutine {}({})".format(API.__part_fname, inargs), 1) + ofile.blank_line() + inargs = f"suite_name, part_list, {errmsg_name}, {errcode_name}" + ofile.write(f"subroutine {API.__part_fname}({inargs})", 1) oline = "character(len=*), intent(in) :: suite_name" ofile.write(oline, 2) oline = "character(len=*), allocatable, intent(out) :: part_list(:)" @@ -741,9 +740,9 @@ def write_suite_part_list_sub(self, ofile, errmsg_name, errcode_name): self._errcode_var.write_def(ofile, 2, self) else_str = '' ename = self._errcode_var.get_prop_value('local_name') - ofile.write("{} = 0".format(ename), 2) + ofile.write(f"{ename} = ''", 2) ename = self._errmsg_var.get_prop_value('local_name') - ofile.write("{} = ''".format(ename), 2) + ofile.write(f"{ename} = ''", 2) for suite in self.suites: oline = "{}if(trim(suite_name) == '{}') then" ofile.write(oline.format(else_str, suite.name), 2) @@ -751,12 +750,12 @@ def write_suite_part_list_sub(self, ofile, errmsg_name, errcode_name): else_str = 'else ' # end for ofile.write("else", 2) - emsg = "write({errmsg}, '(3a)')".format(errmsg=errmsg_name) + emsg = f"write({errmsg_name}, '(3a)')" emsg += "'No suite named ', trim(suite_name), ' found'" ofile.write(emsg, 3) - ofile.write("{errcode} = 1".format(errcode=errcode_name), 3) + ofile.write(f"{errcode_name} = 1", 3) ofile.write("end if", 2) - ofile.write("end subroutine {}".format(API.__part_fname), 1) + ofile.write(f"end subroutine {API.__part_fname}", 1) def write_req_vars_sub(self, ofile, errmsg_name, errcode_name): """Write the required variables subroutine""" @@ -807,9 +806,9 @@ def write_req_vars_sub(self, ofile, errmsg_name, errcode_name): parent = suite.parent # Collect all the suite variables oline = "{}if(trim(suite_name) == '{}') then" - input_vars = [set(), set(), set()] # leaves, arrrays, leaf elements - inout_vars = [set(), set(), set()] # leaves, arrrays, leaf elements - output_vars = [set(), set(), set()] # leaves, arrrays, leaf elements + input_vars = [set(), set(), set()] # leaves, arrays, leaf elements + inout_vars = [set(), set(), set()] # leaves, arrays, leaf elements + output_vars = [set(), set(), set()] # leaves, arrays, leaf elements for part in suite.groups: for var in part.call_list.variable_list(): stdname = var.get_prop_value("standard_name") @@ -821,7 +820,8 @@ def write_req_vars_sub(self, ofile, errmsg_name, errcode_name): protected = pvar.get_prop_value("protected") # end if # end if - elements = var.intrinsic_elements(check_dict=self.parent) + elements = var.intrinsic_elements(check_dict=self.parent, + ddt_lib=self.__ddt_lib) if (intent == 'in') and (not protected): if isinstance(elements, list): input_vars[1].add(stdname) diff --git a/scripts/common.py b/scripts/common.py index 0ac2c74d..7f4e2f43 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -165,14 +165,7 @@ def escape_tex(text): def isstring(s): """Return true if a variable is a string""" - # We use Python 3 - if (sys.version_info.major == 3): - return isinstance(s, str) - # We use Python 2 - elif (sys.version_info.major == 2): - return isinstance(s, basestring) - else: - raise Exception('Unknown Python version') + return isinstance(s, str) def string_to_python_identifier(string): """Replaces forbidden characters in strings with standard substitutions diff --git a/scripts/ddt_library.py b/scripts/ddt_library.py index 30614226..4be53963 100644 --- a/scripts/ddt_library.py +++ b/scripts/ddt_library.py @@ -44,11 +44,11 @@ def __init__(self, new_field, var_ref, run_env, recur=False): else: # Recurse to find correct (tail) location for self.__field = VarDDT(new_field, var_ref.field, run_env, recur=True) - # End if + # end if if ((not recur) and run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG)): run_env.logger.debug('Adding DDT field, {}'.format(self)) - # End if + # end if def is_ddt(self): """Return True iff is a DDT type.""" @@ -66,18 +66,18 @@ def get_prop_value(self, name): pvalue = super().get_prop_value(name) else: pvalue = self.field.get_prop_value(name) - # End if + # end if return pvalue def intrinsic_elements(self, check_dict=None): """Return the Var intrinsic elements for the leaf Var object. - See Var.intrinsic_elem for details + See Var.intrinsic_elements for details """ if self.field is None: pvalue = super().intrinsic_elements(check_dict=check_dict) else: pvalue = self.field.intrinsic_elements(check_dict=check_dict) - # End if + # end if return pvalue def clone(self, subst_dict, source_name=None, source_type=None, @@ -98,7 +98,7 @@ def clone(self, subst_dict, source_name=None, source_type=None, source_name=source_name, source_type=source_type, context=context) - # End if + # end if return clone_var def call_string(self, var_dict, loop_vars=None): @@ -109,7 +109,7 @@ def call_string(self, var_dict, loop_vars=None): if self.field is not None: call_str += '%' + self.field.call_string(var_dict, loop_vars=loop_vars) - # End if + # end if return call_str def write_def(self, outfile, indent, ddict, allocatable=False, dummy=False): @@ -122,7 +122,7 @@ def write_def(self, outfile, indent, ddict, allocatable=False, dummy=False): else: self.field.write_def(outfile, indent, ddict, allocatable=allocatable, dummy=dummy) - # End if + # end if @staticmethod def __var_rep(var, prefix=""): @@ -137,14 +137,14 @@ def __var_rep(var, prefix=""): lstr = '{}%{}({})'.format(prefix, lname, ', '.join(ldims)) else: lstr = '{}({})'.format(lname, ', '.join(ldims)) - # End if + # end if else: if prefix: lstr = '{}%{}'.format(prefix, lname) else: lstr = '{}'.format(lname) - # End if - # End if + # end if + # end if return lstr def __repr__(self): @@ -160,9 +160,9 @@ def __repr__(self): elif isinstance(field, Var): lstr = self.__var_rep(field, prefix=lstr) field = None - # End if + # end if sep = '%' - # End while + # end while return "".format(lstr) def __str__(self): @@ -201,33 +201,33 @@ def __init__(self, name, run_env, ddts=None, logger=None): ddts = list() elif not isinstance(ddts, list): ddts = [ddts] - # End if + # end if # Add all the DDT headers, then process for ddt in ddts: if not isinstance(ddt, MetadataSection): errmsg = 'Invalid DDT metadata type, {}' - raise ParseInternalError(errmsg.format(type(ddt))) - # End if + raise ParseInternalError(errmsg.format(type(ddt).__name__)) + # end if if not ddt.header_type == 'ddt': errmsg = 'Metadata table header is for a {}, should be DDT' raise ParseInternalError(errmsg.format(ddt.header_type)) - # End if + # end if if ddt.title in self: errmsg = "Duplicate DDT, {}, found{}, original{}" ctx = context_string(ddt.source.context) octx = context_string(self[ddt.title].source.context) raise CCPPError(errmsg.format(ddt.title, ctx, octx)) - # End if - if logger is not None: - lmsg = 'Adding DDT {} to {}' - logger.debug(lmsg.format(ddt.title, self.name)) - # End if + # end if + if logger and logger.isEnabledFor(logging.DEBUG): + lmsg = f"Adding DDT {ddt.title} to {self.name}" + logger.debug(lmsg) + # end if self[ddt.title] = ddt dlen = len(ddt.module) if dlen > self.__max_mod_name_len: self.__max_mod_name_len = dlen - # End if - # End for + # end if + # end for def check_ddt_type(self, var, header, lname=None): """If is a DDT, check to make sure it is in this DDT library. @@ -239,16 +239,21 @@ def check_ddt_type(self, var, header, lname=None): if vtype not in self: if lname is None: lname = var.get_prop_value('local_name') - # End if + # end if errmsg = 'Variable {} is of unknown type ({}) in {}' ctx = context_string(var.context) raise CCPPError(errmsg.format(lname, vtype, header.title, ctx)) - # End if - # End if (no else needed) + # end if + # end if (no else needed) - def collect_ddt_fields(self, var_dict, var, run_env, ddt=None): + def collect_ddt_fields(self, var_dict, var, run_env, + ddt=None, skip_duplicates=False): """Add all the reachable fields from DDT variable of type, to . Each field is added as a VarDDT. + Note: By default, it is an error to try to add a duplicate + field to (i.e., the field already exists in + or one of its parents). To simply skip duplicate + fields, set to True. """ if ddt is None: vtype = var.get_prop_value('type') @@ -259,8 +264,8 @@ def collect_ddt_fields(self, var_dict, var, run_env, ddt=None): ctx = context_string(var.context) errmsg = "Variable, {}, is not a known DDT{}" raise ParseInternalError(errmsg.format(lname, ctx)) - # End if - # End if + # end if + # end if for dvar in ddt.variable_list(): subvar = VarDDT(dvar, var, self.run_env) dvtype = dvar.get_prop_value('type') @@ -268,22 +273,25 @@ def collect_ddt_fields(self, var_dict, var, run_env, ddt=None): # If DDT in our library, we need to add sub-fields recursively. subddt = self[dvtype] self.collect_ddt_fields(var_dict, subvar, run_env, ddt=subddt) - else: - # add_variable only checks the current dictionary. For a - # DDT, the variable also cannot be in our parent dictionaries. - stdname = dvar.get_prop_value('standard_name') - pvar = var_dict.find_variable(standard_name=stdname, + # end if + # add_variable only checks the current dictionary. By default, + # for a DDT, the variable also cannot be in our parent + # dictionaries. + stdname = dvar.get_prop_value('standard_name') + pvar = var_dict.find_variable(standard_name=stdname, any_scope=True) - if pvar: - emsg = "Attempt to add duplicate DDT sub-variable, {}{}." - emsg += "\nVariable originally defined{}" - ntx = context_string(dvar.context) - ctx = context_string(pvar.context) - raise CCPPError(emsg.format(stdname, ntx, ctx)) - # end if - # Add this intrinsic to + if pvar and (not skip_duplicates): + emsg = "Attempt to add duplicate DDT sub-variable, {}{}." + emsg += "\nVariable originally defined{}" + ntx = context_string(dvar.context) + ctx = context_string(pvar.context) + raise CCPPError(emsg.format(stdname, ntx, ctx)) + # end if + # Add this intrinsic to + if not pvar: var_dict.add_variable(subvar, run_env) - # End for + # end if + # end for def ddt_modules(self, variable_list, ddt_mods=None): """Collect information for module use statements. @@ -292,14 +300,14 @@ def ddt_modules(self, variable_list, ddt_mods=None): """ if ddt_mods is None: ddt_mods = set() # Need a new set for every call - # End if + # end if for var in variable_list: vtype = var.get_prop_value('type') if vtype in self: module = self[vtype].module ddt_mods.add((module, vtype)) - # End if - # End for + # end if + # end for return ddt_mods def write_ddt_use_statements(self, variable_list, outfile, indent, pad=0): @@ -313,7 +321,7 @@ def write_ddt_use_statements(self, variable_list, outfile, indent, pad=0): slen = ' '*(pad - len(dmod)) ustring = 'use {},{} only: {}' outfile.write(ustring.format(dmod, slen, dtype), indent) - # End for + # end for @property def name(self): diff --git a/scripts/file_utils.py b/scripts/file_utils.py index 8cdfd023..669922b9 100644 --- a/scripts/file_utils.py +++ b/scripts/file_utils.py @@ -13,11 +13,6 @@ import os # CCPP framework imports from parse_tools import CCPPError, ParseInternalError -#XXgoldyXX: v Crap required to support python 2 -import sys -# Find python version -PY3 = sys.version_info[0] > 2 -#XXgoldyXX: ^ Crap required to support python 2 # Standardize name of generated kinds file and module KINDS_MODULE = 'ccpp_kinds' @@ -300,13 +295,7 @@ def move_modified_files(src_dir, dest_dir, overwrite=False, remove_src=False): fmove = True # end if if fmove: -#XXgoldyXX: v Crap required to support python 2 - if PY3: - os.replace(src_path, dest_path) - else: - os.rename(src_path, dest_path) - # end if -#XXgoldyXX: ^ Crap required to support python 2 + os.replace(src_path, dest_path) else: os.remove(src_path) # end if diff --git a/scripts/fortran_tools/fortran_write.py b/scripts/fortran_tools/fortran_write.py index a238dcc5..f9bcfa3f 100644 --- a/scripts/fortran_tools/fortran_write.py +++ b/scripts/fortran_tools/fortran_write.py @@ -4,7 +4,9 @@ """Code to write Fortran code """ -class FortranWriter(object): +import math + +class FortranWriter: """Class to turn output into properly continued and indented Fortran code >>> FortranWriter("foo.F90", 'r', 'test', 'mod_name') #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): @@ -59,9 +61,9 @@ class FortranWriter(object): def indent(self, level=0, continue_line=False): 'Return an indent string for any level' - indent = self._indent * level + indent = self.indent_size * level if continue_line: - indent = indent + self._continue_indent + indent = indent + self.__continue_indent # End if return indent*' ' @@ -71,22 +73,55 @@ def find_best_break(self, choices, last=None): """Find the best line break point given . If is present, use it as a target line length.""" if last is None: - last = self._line_fill + last = self.__line_fill # End if # Find largest good break - possible = [x for x in choices if x < last] + possible = [x for x in choices if 0 < x < last] if not possible: - best = self._line_max + 1 + best = self.__line_max + 1 else: best = max(possible) # End if - if (best > self._line_max) and (last < self._line_max): - best = self.find_best_break(choices, last=self._line_max) + if (best > self.__line_max) and (last < self.__line_max): + best = self.find_best_break(choices, last=self.__line_max) # End if return best ########################################################################### + @staticmethod + def _in_quote(test_str): + """Return True if ends in a character context. + >>> FortranWriter._in_quote("hi'mom") + True + >>> FortranWriter._in_quote("hi mom") + False + >>> FortranWriter._in_quote("'hi mom'") + False + >>> FortranWriter._in_quote("'hi"" mom'") + False + """ + in_single_char = False + in_double_char = False + for char in test_str: + if in_single_char: + if char == "'": + in_single_char = False + # end if + elif in_double_char: + if char == '"': + in_double_char = False + # end if + elif char == "'": + in_single_char = True + elif char == '"': + in_double_char = True + # end if + # end for + return in_single_char or in_double_char + + ########################################################################### + def write(self, statement, indent_level, continue_line=False): """Write to the open file, indenting to (see self.indent). @@ -102,9 +137,18 @@ def write(self, statement, indent_level, continue_line=False): # End for else: istr = self.indent(indent_level, continue_line) - outstr = istr + statement.strip() + ostmt = statement.strip() + is_comment_stmt = ostmt and (ostmt[0] == '!') + in_comment = "" + if ostmt and (ostmt[0] != '&'): + # Skip indent for continue that is in the middle of a + # token or a quoted region + outstr = istr + ostmt + else: + outstr = ostmt + # end if line_len = len(outstr) - if line_len > self._line_fill: + if line_len > self.__line_fill: # Collect pretty break points spaces = list() commas = list() @@ -125,9 +169,14 @@ def write(self, statement, indent_level, continue_line=False): elif outstr[sptr] == '"': in_double_char = True elif outstr[sptr] == '!': - # Comment in non-character context, suck in rest of line + # Comment in non-character context spaces.append(sptr-1) - sptr = line_len - 1 + in_comment = "! " # No continue for comment + if ((not is_comment_stmt) and + (sptr >= self.__max_comment_start)): + # suck in rest of line + sptr = line_len - 1 + # end if elif outstr[sptr] == ' ': # Non-quote spaces are where we can break spaces.append(sptr) @@ -140,31 +189,62 @@ def write(self, statement, indent_level, continue_line=False): # End if (no else, other characters will be ignored) sptr = sptr + 1 # End while + # Before looking for best space, reject any that are on a + # comment line but before any significant characters + if outstr.lstrip()[0] == '!': + first_space = outstr.index('!') + 1 + while ((outstr[first_space] == '!' or + outstr[first_space] == ' ') and + (first_space < line_len)): + first_space += 1 + # end while + if min(spaces) < first_space: + spaces = [x for x in spaces if x >= first_space] + # end if best = self.find_best_break(spaces) - if best >= self._line_fill: - best = self.find_best_break(commas) + if best >= self.__line_fill: + best = min(best, self.find_best_break(commas)) # End if - if best > self._line_max: - # This is probably a bad situation that might not - # compile, just write the line and hope for the best. - line_continue = False - elif len(outstr) > best: - # If next line is just comment, do not use continue - # NB: Is this a Fortran issue or just a gfortran issue? - line_continue = outstr[best+1:].lstrip()[0] != '!' - else: - line_continue = True + line_continue = False + if best >= self.__line_max: + # This is probably a bad situation so we have to break + # in an ugly spot + best = self.__line_max - 1 + if len(outstr) > best: + line_continue = '&' + # end if + # end if + if len(outstr) > best: + if self._in_quote(outstr[0:best+1]): + line_continue = '&' + else: + # If next line is just comment, do not use continue + line_continue = outstr[best+1:].lstrip()[0] != '!' + # end if + elif not line_continue: + line_continue = len(outstr) > best # End if + if in_comment or is_comment_stmt: + line_continue = False + # end if if line_continue: - fill = "{}&".format((self._line_fill - best)*' ') + fill = "{}&".format((self.__line_fill - best)*' ') else: - fill = '' + fill = "" # End if - self._file.write("{}{}\n".format(outstr[0:best+1], fill)) - statement = outstr[best+1:] + outline = f"{outstr[0:best+1]}{fill}".rstrip() + self.__file.write(f"{outline}\n") + if best <= 0: + imsg = "Internal ERROR: Unable to break line" + raise ValueError(f"{imsg}, '{statement}'") + # end if + statement = in_comment + outstr[best+1:] + if isinstance(line_continue, str) and statement: + statement = line_continue + statement + # end if self.write(statement, indent_level, continue_line=line_continue) else: - self._file.write("{}\n".format(outstr)) + self.__file.write("{}\n".format(outstr)) # End if # End if @@ -175,7 +255,7 @@ def __init__(self, filename, mode, file_description, module_name, line_fill=None, line_max=None): """Initialize thie FortranWriter object. Some boilerplate is written automatically.""" - self.__file_desc = file_description + self.__file_desc = file_description.replace('\n', '\n!! ') self.__module = module_name # We only handle writing situations (for now) and only text if 'r' in mode: @@ -184,26 +264,27 @@ def __init__(self, filename, mode, file_description, module_name, if 'b' in mode: raise ValueError('Binary mode not allowed in FortranWriter object') # End if - self._file = open(filename, mode) + self.__file = open(filename, mode) if indent is None: - self._indent = FortranWriter.__INDENT + self.__indent = FortranWriter.__INDENT else: - self._indent = indent + self.__indent = indent # End if if continue_indent is None: - self._continue_indent = FortranWriter.__CONTINUE_INDENT + self.__continue_indent = FortranWriter.__CONTINUE_INDENT else: - self._continue_indent = continue_indent + self.__continue_indent = continue_indent # End if if line_fill is None: - self._line_fill = FortranWriter.__LINE_FILL + self.__line_fill = FortranWriter.__LINE_FILL else: - self._line_fill = line_fill + self.__line_fill = line_fill # End if + self.__max_comment_start = math.ceil(self.__line_fill * 3 / 4) if line_max is None: - self._line_max = FortranWriter.__LINE_MAX + self.__line_max = FortranWriter.__LINE_MAX else: - self._line_max = line_max + self.__line_max = line_max # End if ########################################################################### @@ -234,7 +315,7 @@ def __enter__(self, *args): def __exit__(self, *args): self.write(FortranWriter.__MOD_FOOTER.format(module=self.__module), 0) - self._file.close() + self.__file.close() return False ########################################################################### @@ -247,6 +328,45 @@ def module_header(self): ########################################################################### + def comment(self, comment, indent): + """Write a Fortran comment with contents, """ + mlcomment = comment.replace('\n', '\n! ') # No backslash in f string + self.write(f"! {mlcomment}", indent) + + ########################################################################### + + def blank_line(self): + """Write a blank line""" + self.write("", 0) + + ########################################################################### + + def include(self, filename): + """Insert the contents of verbatim.""" + with open(filename, 'r') as infile: + for line in infile: + self.__file.write(line) + # end for + # end with + + ########################################################################### + + @property + def line_fill(self): + """Return the target line length for this Fortran file""" + return self.__line_fill + + ########################################################################### + + @property + def indent_size(self): + """Return the number of spaces for each indent level for this + Fortran file + """ + return self.__indent + + ########################################################################### + @classmethod def copyright(cls): """Return the standard Fortran file copyright string""" diff --git a/scripts/fortran_tools/parse_fortran.py b/scripts/fortran_tools/parse_fortran.py index 624f02cb..2310e13f 100644 --- a/scripts/fortran_tools/parse_fortran.py +++ b/scripts/fortran_tools/parse_fortran.py @@ -532,7 +532,8 @@ def class_match(cls, line): @classmethod def type_def_line(cls, line): """Return a type information if represents the start - of a type definition""" + of a type definition. + Otherwise, return None""" type_def = None if not cls.type_match(line): if '!' in line: @@ -629,7 +630,8 @@ def ftype_factory(line, context): def fortran_type_definition(line): ######################################################################## """Return a type information if represents the start - of a type definition""" + of a type definition. + Otherwise, return None.""" return FtypeTypeDecl.type_def_line(line) ######################################################################## @@ -720,8 +722,8 @@ def parse_fortran_var_decl(line, source, run_env): varprops = Ftype.parse_attr_specs(elements[0].strip(), context) for prop in varprops: if prop[0:6] == 'intent': - if source.type != 'scheme': - typ = source.type + if source.ptype != 'scheme': + typ = source.ptype errmsg = 'Invalid variable declaration, {}, intent' errmsg = errmsg + ' not allowed in {} variable' if run_env.logger is not None: diff --git a/scripts/fortran_tools/parse_fortran_file.py b/scripts/fortran_tools/parse_fortran_file.py index f37b3377..7c6493e7 100755 --- a/scripts/fortran_tools/parse_fortran_file.py +++ b/scripts/fortran_tools/parse_fortran_file.py @@ -563,29 +563,39 @@ def parse_preamble_data(statements, pobj, spec_name, endmatch, run_env): module=spec_name, var_dict=var_dict) mheaders.append(mheader) - if run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG): + if (run_env.logger and + run_env.logger.isEnabledFor(logging.DEBUG)): ctx = context_string(pobj, nodir=True) msg = 'Adding header {}{}' run_env.logger.debug(msg.format(mheader.table_name, ctx)) + # end if break - elif ((type_def is not None) and (active_table is not None) and - (type_def[0].lower() == active_table.lower())): + elif type_def is not None: # Put statement back so caller knows where we are statements.insert(0, statement) - statements, ddt = parse_type_def(statements, type_def, - spec_name, pobj, run_env) - if ddt is None: - ctx = context_string(pobj, nodir=True) - msg = "No DDT found at '{}'{}" - raise CCPPError(msg.format(statement, ctx)) - # End if - mheaders.append(ddt) - if run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG): - ctx = context_string(pobj, nodir=True) - msg = 'Adding DDT {}{}' - run_env.logger.debug(msg.format(ddt.table_name, ctx)) - # End if - active_table = None + if ((active_table is not None) and + (type_def[0].lower() == active_table.lower())): + statements, ddt = parse_type_def(statements, type_def, + spec_name, pobj, run_env) + if ddt is None: + ctx = context_string(pobj, nodir=True) + msg = "No DDT found at '{}'{}" + raise CCPPError(msg.format(statement, ctx)) + # End if + mheaders.append(ddt) + if (run_env.logger and + run_env.logger.isEnabledFor(logging.DEBUG)): + ctx = context_string(pobj, nodir=True) + msg = 'Adding DDT {}{}' + run_env.logger.debug(msg.format(ddt.table_name, ctx)) + # End if + active_table = None + else: + # We found a type definition but it is not one with + # metadata. Just parse it and throw away what is found. + _ = parse_type_def(statements, type_def, + spec_name, pobj, run_env) + # end if elif active_table is not None: # We should have a variable definition to add if ((not is_comment_statement(statement)) and @@ -786,6 +796,7 @@ def parse_specification(pobj, statements, run_env, mod_name=None, # End program or module pmatch = endmatch.match(statement) asmatch = _ARG_TABLE_START_RE.match(statement) + type_def = fortran_type_definition(statement) if pmatch is not None: # We never found a contains statement inspec = False @@ -812,6 +823,13 @@ def parse_specification(pobj, statements, run_env, mod_name=None, # End if inspec = pobj.in_region('MODULE', region_name=mod_name) break + elif type_def: + # We have a type definition without metadata + # Just parse it and throw away what is found. + # Put statement back so caller knows where we are + statements.insert(0, statement) + _ = parse_type_def(statements, type_def, + spec_name, pobj, run_env) elif is_contains_statement(statement, inmod): inspec = False break diff --git a/scripts/framework_env.py b/scripts/framework_env.py index 8ee553f4..b5550ca6 100644 --- a/scripts/framework_env.py +++ b/scripts/framework_env.py @@ -103,8 +103,8 @@ def __init__(self, logger, ndict=None, verbose=0, clean=False, # String of definitions, separated by spaces preproc_list = [x.strip() for x in preproc_defs.split(' ') if x] else: - wmsg = "Error: Bad preproc list type, '{}'" - emsg += esep + wmsg.format(type(preproc_defs)) + wmsg = f"Error: Bad preproc list type, '{type_name{preproc_defs)}'" + emsg += esep + wmsg esep = '\n' # end if # Turn the list into a dictionary diff --git a/scripts/metadata_table.py b/scripts/metadata_table.py index 17e8dfdb..3501e40e 100644 --- a/scripts/metadata_table.py +++ b/scripts/metadata_table.py @@ -158,7 +158,7 @@ def blank_metadata_line(line): def _parse_config_line(line, context): """Parse a config line and return a list of keyword value pairs.""" - parse_items = list() + parse_items = [] if line is None: pass # No properties on this line elif blank_metadata_line(line): @@ -182,8 +182,8 @@ def _parse_config_line(line, context): def parse_metadata_file(filename, known_ddts, run_env): """Parse and return list of parsed metadata tables""" # Read all lines of the file at once - meta_tables = list() - table_titles = list() # Keep track of names in file + meta_tables = [] + table_titles = [] # Keep track of names in file with open(filename, 'r') as infile: fin_lines = infile.readlines() for index, fin_line in enumerate(fin_lines): @@ -225,7 +225,7 @@ def find_scheme_names(filename): """Find and return a list of all the physics scheme names in . A scheme is identified by its ccpp-table-properties name. """ - scheme_names = list() + scheme_names = [] with open(filename, 'r') as infile: fin_lines = infile.readlines() # end with @@ -283,7 +283,7 @@ def __init__(self, run_env, table_name_in=None, table_type_in=None, self.__pobj = parse_object self.__dependencies = dependencies self.__relative_path = relative_path - self.__sections = list() + self.__sections = [] self.__run_env = run_env if parse_object is None: if table_name_in is not None: @@ -339,7 +339,7 @@ def __init__(self, run_env, table_name_in=None, table_type_in=None, raise ParseInternalError(perr) # end if if known_ddts is None: - known_ddts = list() + known_ddts = [] # end if self.__start_context = ParseContext(context=self.__pobj) self.__init_from_file(known_ddts, self.__run_env) @@ -351,7 +351,7 @@ def __init_from_file(self, known_ddts, run_env): curr_line, _ = self.__pobj.next_line() in_properties_header = True skip_rest_of_section = False - self.__dependencies = list() # Default is no dependencies + self.__dependencies = [] # Default is no dependencies # Process lines until the end of the file or start of the next table. while ((curr_line is not None) and (not MetadataTable.table_start(curr_line))): @@ -440,7 +440,7 @@ def __init_from_file(self, known_ddts, run_env): known_ddts.append(self.table_name) # end if if self.__dependencies is None: - self.__dependencies = list() + self.__dependencies = [] # end if def start_context(self, with_comma=True, nodir=True): @@ -673,7 +673,7 @@ def __init__(self, table_name, table_type, run_env, parse_object=None, self.__start_context = None else: if known_ddts is None: - known_ddts = list() + known_ddts = [] # end if self.__start_context = ParseContext(context=self.__pobj) self.__init_from_file(table_name, table_type, known_ddts, run_env) @@ -683,7 +683,7 @@ def __init__(self, table_name, table_type, run_env, parse_object=None, register_fortran_ddt_name(self.title) # end if # Categorize the variables - self._var_intents = {'in' : list(), 'out' : list(), 'inout' : list()} + self._var_intents = {'in' : [], 'out' : [], 'inout' : [])} for var in self.variable_list(): intent = var.get_prop_value('intent') if intent is not None: @@ -693,7 +693,7 @@ def __init__(self, table_name, table_type, run_env, parse_object=None, def _default_module(self): """Set a default module for this header""" - mfile = self.__pobj.file_name + mfile = self.__pobj.filename if mfile[-5:] == '.meta': # Default value is a Fortran module that matches the filename def_mod = os.path.basename(mfile)[:-5] @@ -888,7 +888,7 @@ def parse_variable(self, curr_line, known_ddts): # Special case for dimensions, turn them into ranges if pname == 'dimensions': porig = pval - pval = list() + pval = [] for dim in porig: if ':' in dim: pval.append(dim) @@ -975,7 +975,7 @@ def check_array_reference(local_name, var_dict, context): local_name, colon_rank, ctx)) # end if - sub_dims = list() + sub_dims = [] sindex = 0 for rind in rdims: if rind == ':': @@ -1010,7 +1010,7 @@ def convert_dims_to_standard_names(self, var, logger=None, context=None): """Convert the dimension elements in to standard names by by using other variables in this header. """ - std_dims = list() + std_dims = [] vdims = var.get_dimensions() # Check for bad dimensions if vdims is None: @@ -1036,7 +1036,7 @@ def convert_dims_to_standard_names(self, var, logger=None, context=None): raise CCPPError("{}".format(errmsg)) # end if for dim in vdims: - std_dim = list() + std_dim = [] if ':' not in dim: # Metadata dimensions always have an explicit start var_one = CCPP_CONSTANT_VARS.find_local_name('1') @@ -1056,7 +1056,7 @@ def convert_dims_to_standard_names(self, var, logger=None, context=None): # Some non-standard integer value dname = item # end if - except ValueError: + except ValueError as verr: # Not an integer, try to find the standard_name if not item: # Naked colons are okay @@ -1070,15 +1070,15 @@ def convert_dims_to_standard_names(self, var, logger=None, context=None): # end if # end if if dname is None: - errmsg = "Unknown dimension element, {}, in {}{}" std = var.get_prop_value('local_name') - ctx = context_string(context) + errmsg = f"Unknown dimension element, {item}, in {std}" + errmsg += context_string(context) if logger is not None: errmsg = "ERROR: " + errmsg logger.error(errmsg.format(item, std, ctx)) dname = unique_standard_name() else: - raise CCPPError(errmsg.format(item, std, ctx)) + raise CCPPError(errmsg) from verr # end if # end if # end try diff --git a/scripts/metavar.py b/scripts/metavar.py index 71609580..7f852c4c 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -20,7 +20,7 @@ from parse_tools import check_units, check_dimensions, check_cf_standard_name from parse_tools import check_diagnostic_id, check_diagnostic_fixed from parse_tools import check_default_value, check_valid_values -from parse_tools import ParseContext, ParseSource +from parse_tools import ParseContext, ParseSource, type_name from parse_tools import ParseInternalError, ParseSyntaxError, CCPPError from var_props import CCPP_LOOP_DIM_SUBSTS, VariableProperty, VarCompatObj from var_props import find_horizontal_dimension, find_vertical_dimension @@ -261,7 +261,7 @@ def __init__(self, prop_dict, source, run_env, context=None, if isinstance(prop_dict, Var): prop_dict = prop_dict.copy_prop_dict() # end if - if source.type == 'scheme': + if source.ptype == 'scheme': self.__required_props = Var.__required_var_props # XXgoldyXX: v don't fill in default properties? # mstr_propdict = Var.__var_propdict @@ -272,7 +272,7 @@ def __init__(self, prop_dict, source, run_env, context=None, mstr_propdict = Var.__spec_propdict # XXgoldyXX: ^ don't fill in default properties? # end if - self._source = source + self.__source = source # Grab a frozen copy of the context if context is None: self._context = ParseContext(context=source.context) @@ -476,7 +476,7 @@ def clone(self, subst_dict=None, remove_intent=False, source_name = self.source.name # end if if source_type is None: - source_type = self.source.type + source_type = self.source.ptype # end if if context is None: context = self._context @@ -822,7 +822,7 @@ def parent(self, parent_var): else: emsg = 'Attempting to set parent for {}, bad parent type, {}' lname = self.get_prop_value('local_name') - raise ParseInternalError(emsg.format(lname, type(parent_var))) + raise ParseInternalError(emsg.format(lname, type_name(parent_var))) # end if def add_child(self, cvar): @@ -852,13 +852,13 @@ def context(self): @property def source(self): """Return the source object for this variable""" - return self._source + return self.__source @source.setter def source(self, new_source): """Reset this Var's source if seems legit""" if isinstance(new_source, ParseSource): - self._source = new_source + self.__source = new_source else: errmsg = 'Attemping to set source of {} ({}) to "{}"' stdname = self.get_prop_value('standard_name') @@ -874,7 +874,7 @@ def clone_source(self): @property def host_interface_var(self): """True iff self is included in the host model interface calls""" - return self.source.type == 'host' + return self.source.ptype == 'host' @property def run_env(self): @@ -1427,7 +1427,7 @@ def __init__(self, name, run_env, variables=None, self[stdname] = variables[key] # end for elif variables is not None: - raise ParseInternalError('Illegal type for variables, {} in {}'.format(type(variables), self.name)) + raise ParseInternalError('Illegal type for variables, {} in {}'.format(type_name(variables), self.name)) # end if @property @@ -1608,7 +1608,7 @@ def add_variable_dimensions(self, var, ignore_sources, to_dict=None, # end if if not present: dvar = self.find_variable(standard_name=dimname, any_scope=True) - if dvar and (dvar.source.type not in ignore_sources): + if dvar and (dvar.source.ptype not in ignore_sources): if to_dict: to_dict.add_variable(dvar, self.__run_env, exists_ok=True, @@ -1631,7 +1631,7 @@ def add_variable_dimensions(self, var, ignore_sources, to_dict=None, err_ret += "\nFound {} from excluded source, '{}'{}" lname = dvar.get_prop_value('local_name') dctx = context_string(dvar.context) - err_ret = err_ret.format(lname, dvar.source.type, dctx) + err_ret = err_ret.format(lname, dvar.source.ptype, dctx) # end if # end if # end if diff --git a/scripts/parse_tools/__init__.py b/scripts/parse_tools/__init__.py index 03dd0429..4f888fb1 100644 --- a/scripts/parse_tools/__init__.py +++ b/scripts/parse_tools/__init__.py @@ -8,7 +8,7 @@ # pylint: disable=wrong-import-position from parse_source import ParseContext, ParseSource from parse_source import ParseSyntaxError, ParseInternalError -from parse_source import CCPPError, context_string +from parse_source import CCPPError, context_string, type_name from parse_source import unique_standard_name, reset_standard_name_counter from parse_object import ParseObject from parse_checkers import check_fortran_id, FORTRAN_ID @@ -71,6 +71,7 @@ 'set_log_to_file', 'set_log_to_null', 'set_log_to_stdout', + 'type_name', 'unique_standard_name', 'validate_xml_file' ] diff --git a/scripts/parse_tools/parse_checkers.py b/scripts/parse_tools/parse_checkers.py index 487478e6..ca5f1f51 100755 --- a/scripts/parse_tools/parse_checkers.py +++ b/scripts/parse_tools/parse_checkers.py @@ -12,7 +12,7 @@ ######################################################################## -_UNITS_RE = re.compile(r"^[^/@#$%^&*()\|<>\[\]{}?,.]+$") +_UNITS_RE = re.compile(r"^[^/!@#$%^&*=()\|<>\[\]{}?,.]+$") def check_units(test_val, prop_dict, error): """Return if a valid unit, otherwise, None diff --git a/scripts/parse_tools/parse_object.py b/scripts/parse_tools/parse_object.py index 4518d75c..f141b298 100644 --- a/scripts/parse_tools/parse_object.py +++ b/scripts/parse_tools/parse_object.py @@ -35,7 +35,6 @@ class ParseObject(ParseContext): def __init__(self, filename, lines_in, line_start=0): """Initialize this ParseObject""" - self.__filename = filename self.__lines = lines_in self.__line_start = line_start self.__line_end = line_start @@ -43,7 +42,7 @@ def __init__(self, filename, lines_in, line_start=0): self.__num_lines = len(self.__lines) self.__error_message = "" self.__num_errors = 0 - super(ParseObject, self).__init__(linenum=line_start, filename=filename) + super().__init__(linenum=line_start, filename=filename) @property def first_line_num(self): @@ -59,11 +58,6 @@ def valid_line(self): """Return True if the current line is valid""" return (self.line_num >= 0) and (self.line_num < self.__num_lines) - @property - def file_name(self): - """Return this object's filename""" - return self.__filename - @property def error_message(self): """Return this object's error message""" diff --git a/scripts/parse_tools/parse_source.py b/scripts/parse_tools/parse_source.py index 6d28b694..f96bbb2d 100644 --- a/scripts/parse_tools/parse_source.py +++ b/scripts/parse_tools/parse_source.py @@ -2,24 +2,18 @@ """Classes to aid the parsing process""" -import sys -# Find python version -PY3 = sys.version_info[0] > 2 -# pylint: disable=wrong-import-position # Python library imports -if PY3: - from collections.abc import Iterable -else: - from collections import Iterable +from collections.abc import Iterable # end if import copy +import sys import os.path import logging # CCPP framework imports # pylint: enable=wrong-import-position -class _StdNameCounter(object): +class _StdNameCounter(): """Class to hold a global counter to avoid using global keyword""" __SNAME_NUM = 0 # Counter for unique standard names @@ -117,6 +111,12 @@ def context_string(context=None, with_comma=True, nodir=False): # End if return cstr.format(comma=comma, where_str=where_str, ctx=context) +############################################################################### +def type_name(obj): +############################################################################### + """Return the name of the type of """ + return type(obj).__name__ + ############################################################################### class CCPPError(ValueError): """Class so programs can log user errors without backtrace""" @@ -198,7 +198,7 @@ def __getitem__(self, index): ######################################################################## -class ParseContext(object): +class ParseContext(): """A class for keeping track of a parsing position >>> ParseContext(32, "source.F90") #doctest: +ELLIPSIS <__main__.ParseContext object at 0x...> @@ -216,13 +216,6 @@ class ParseContext(object): """ - # python 2/3 difference - try: - __pstr_types = (str, unicode) - except NameError: - __pstr_types = (str,) - # End try - def __init__(self, linenum=None, filename=None, context=None): """Initialize this ParseContext""" # Set regions first in case of exception @@ -245,27 +238,27 @@ def __init__(self, linenum=None, filename=None, context=None): filename = context.filename elif filename is None: filename = "" - elif not isinstance(filename, ParseContext.__pstr_types): + elif not isinstance(filename, str): raise CCPPError('ParseContext filename must be a string') # No else, everything is okay # End if - self._linenum = linenum - self._filename = filename + self.__linenum = linenum + self.__filename = filename @property def line_num(self): """Return the current line""" - return self._linenum + return self.__linenum @line_num.setter def line_num(self, newnum): """Set a new line number for this context""" - self._linenum = newnum + self.__linenum = newnum @property def filename(self): """Return the object's filename""" - return self._filename + return self.__filename @property def regions(self): @@ -274,19 +267,19 @@ def regions(self): def __format__(self, spec): """Return a string representing the location in a file - Note that self._linenum is zero based. + Note that self.__linenum is zero based. can be 'dir' (show filename directory) or 'nodir' filename only. Any other spec entry is ignored. """ if spec == 'dir': - fname = self._filename + fname = self.__filename elif spec == 'nodir': - fname = os.path.basename(self._filename) + fname = os.path.basename(self.__filename) else: - fname = self._filename + fname = self.__filename # End if - if self._linenum >= 0: - fmt_str = "{}:{}".format(fname, self._linenum+1) + if self.__linenum >= 0: + fmt_str = "{}:{}".format(fname, self.__linenum+1) else: fmt_str = "{}".format(fname) # End if @@ -294,21 +287,21 @@ def __format__(self, spec): def __str__(self): """Return a string representing the location in a file - Note that self._linenum is zero based. + Note that self.__linenum is zero based. """ - if self._linenum >= 0: - retstr = "{}:{}".format(self._filename, self._linenum+1) + if self.__linenum >= 0: + retstr = "{}:{}".format(self.__filename, self.__linenum+1) else: - retstr = "{}".format(self._filename) + retstr = "{}".format(self.__filename) # End if return retstr def increment(self, inc=1): """Increment the location within a file""" - if self._linenum < 0: - self._linenum = 0 + if self.__linenum < 0: + self.__linenum = 0 # End if - self._linenum = self._linenum + inc + self.__linenum = self.__linenum + inc def enter_region(self, region_type, region_name=None, nested_ok=True): """Mark the entry of a region (e.g., DDT, module, function). @@ -378,12 +371,12 @@ def region_str(self): ######################################################################## -class ParseSource(object): +class ParseSource(): """ A simple object for providing source information >>> ParseSource("myname", "mytype", ParseContext(13, "foo.F90")) #doctest: +ELLIPSIS <__main__.ParseSource object at 0x...> - >>> ParseSource("myname", "mytype", ParseContext(13, "foo.F90")).type + >>> ParseSource("myname", "mytype", ParseContext(13, "foo.F90")).ptype 'mytype' >>> ParseSource("myname", "mytype", ParseContext(13, "foo.F90")).name 'myname' @@ -393,24 +386,24 @@ class ParseSource(object): def __init__(self, name_in, type_in, context_in): """Initialize this ParseSource object.""" - self._name = name_in - self._type = type_in - self._context = context_in + self.__name = name_in + self.__type = type_in + self.__context = context_in @property - def type(self): + def ptype(self): """Return this source's type""" - return self._type + return self.__type @property def name(self): """Return this source's name""" - return self._name + return self.__name @property def context(self): """Return this source's context""" - return self._context + return self.__context ######################################################################## diff --git a/scripts/parse_tools/xml_tools.py b/scripts/parse_tools/xml_tools.py index 414ffc5a..0886356d 100644 --- a/scripts/parse_tools/xml_tools.py +++ b/scripts/parse_tools/xml_tools.py @@ -7,26 +7,19 @@ # Python library imports from __future__ import print_function import os -import os.path import re +import shutil import subprocess import sys import xml.etree.ElementTree as ET sys.path.insert(0, os.path.dirname(__file__)) -# pylint: disable=wrong-import-position -try: - from distutils.spawn import find_executable - _XMLLINT = find_executable('xmllint') -except ImportError: - _XMLLINT = None -# End try # CCPP framework imports from parse_source import CCPPError from parse_log import init_log, set_log_to_null -# pylint: enable=wrong-import-position # Global data _INDENT_STR = " " +_XMLLINT = shutil.which('xmllint') # Blank if not installed beg_tag_re = re.compile(r"([<][^/][^<>]*[^/][>])") end_tag_re = re.compile(r"([<][/][^<>/]+[>])") simple_tag_re = re.compile(r"([<][^/][^<>/]+[/][>])") @@ -36,6 +29,14 @@ PYSUBVER = sys.version_info[1] _LOGGER = None +############################################################################### +class XMLToolsInternalError(ValueError): +############################################################################### + """Error class for reporting internal errors""" + def __init__(self, message): + """Initialize this exception""" + super().__init__(message) + ############################################################################### def call_command(commands, logger, silent=False): ############################################################################### @@ -53,35 +54,12 @@ def call_command(commands, logger, silent=False): result = False outstr = '' try: - if PY3: - if PYSUBVER > 6: - cproc = subprocess.run(commands, check=True, - capture_output=True) - if not silent: - logger.debug(cproc.stdout) - # End if - result = cproc.returncode == 0 - elif PYSUBVER >= 5: - cproc = subprocess.run(commands, check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if not silent: - logger.debug(cproc.stdout) - # End if - result = cproc.returncode == 0 - else: - raise ValueError("Python 3 must be at least version 3.5") - # End if - else: - pproc = subprocess.Popen(commands, stdin=None, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, _ = pproc.communicate() - if not silent: - logger.debug(output) - # End if - result = pproc.returncode == 0 - # End if + cproc = subprocess.run(commands, check=True, + capture_output=True) + if not silent: + logger.debug(cproc.stdout) + # end if + result = cproc.returncode == 0 except (OSError, CCPPError, subprocess.CalledProcessError) as err: if silent: result = False @@ -90,9 +68,9 @@ def call_command(commands, logger, silent=False): emsg = "Execution of '{}' failed with code:\n" outstr = emsg.format(cmd, err.returncode) outstr += "{}".format(err.output) - raise CCPPError(outstr) - # End if - # End of try + raise CCPPError(outstr) from err + # end if + # end of try return result ############################################################################### @@ -102,6 +80,8 @@ def find_schema_version(root): Find the version of the host registry file represented by root >>> find_schema_version(ET.fromstring('')) [1, 0] + >>> find_schema_version(ET.fromstring('')) + [2, 0] >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: Illegal version string, '1.a' @@ -118,33 +98,33 @@ def find_schema_version(root): verbits = None if 'version' not in root.attrib: raise CCPPError("version attribute required") - # End if + # end if version = root.attrib['version'] versplit = version.split('.') try: if len(versplit) != 2: raise CCPPError('oops') - # End if (no else needed) + # end if (no else needed) try: verbits = [int(x) for x in versplit] except ValueError as verr: - raise CCPPError(verr) - # End try + raise CCPPError(verr) from verr + # end try if verbits[0] < 1: raise CCPPError('Major version must be at least 1') - # End if + # end if if verbits[1] < 0: raise CCPPError('Minor version must be non-negative') - # End if + # end if except CCPPError as verr: errstr = """Illegal version string, '{}' Format must be .""" ve_str = str(verr) if ve_str: errstr = ve_str + '\n' + errstr - # End if - raise CCPPError(errstr.format(version)) - # End try + # end if + raise CCPPError(errstr.format(version)) from verr + # end try return verbits ############################################################################### @@ -161,10 +141,10 @@ def find_schema_file(schema_root, version, schema_path=None): schema_file = os.path.join(schema_path, schema_filename) else: schema_file = schema_filename - # End if + # end if if os.path.exists(schema_file): return schema_file - # End if + # end if return None ############################################################################### @@ -178,38 +158,42 @@ def validate_xml_file(filename, schema_root, version, logger, # Check the filename if not os.path.isfile(filename): raise CCPPError("validate_xml_file: Filename, '{}', does not exist".format(filename)) - # End if + # end if if not os.access(filename, os.R_OK): raise CCPPError("validate_xml_file: Cannot open '{}'".format(filename)) - # End if - if not schema_path: - # Find the schema, based on the model version - thispath = os.path.abspath(__file__) - pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) - schema_path = os.path.join(pdir, 'schema') - # End if - schema_file = find_schema_file(schema_root, version, schema_path) - if not (schema_file and os.path.isfile(schema_file)): - verstring = '.'.join([str(x) for x in version]) - emsg = """validate_xml_file: Cannot find schema for version {}, - {} does not exist""" - raise CCPPError(emsg.format(verstring, schema_file)) - # End if + # end if + if os.path.isfile(schema_root): + # We already have a file, just use it + schema_file = schema_root + else: + if not schema_path: + # Find the schema, based on the model version + thispath = os.path.abspath(__file__) + pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) + schema_path = os.path.join(pdir, 'schema') + # end if + schema_file = find_schema_file(schema_root, version, schema_path) + if not (schema_file and os.path.isfile(schema_file)): + verstring = '.'.join([str(x) for x in version]) + emsg = """validate_xml_file: Cannot find schema for version {}, {} does not exist""" + raise CCPPError(emsg.format(verstring, schema_file)) + # end if + # end if if not os.access(schema_file, os.R_OK): emsg = "validate_xml_file: Cannot open schema, '{}'" raise CCPPError(emsg.format(schema_file)) - # End if - if _XMLLINT is not None: + # end if + if _XMLLINT: logger.debug("Checking file {} against schema {}".format(filename, schema_file)) cmd = [_XMLLINT, '--noout', '--schema', schema_file, filename] result = call_command(cmd, logger) return result - # End if + # end if lmsg = "xmllint not found, could not validate file {}" if error_on_noxmllint: raise CCPPError("validate_xml_file: " + lmsg.format(filename)) - # End if + # end if logger.warning(lmsg.format(filename)) return True # We could not check but still need to proceed @@ -229,13 +213,13 @@ def read_xml_file(filename, logger=None): root = tree.getroot() except ET.ParseError as perr: emsg = "read_xml_file: Cannot read {}, {}" - raise CCPPError(emsg.format(filename, perr)) + raise CCPPError(emsg.format(filename, perr)) from perr elif not os.access(filename, os.R_OK): raise CCPPError("read_xml_file: Cannot open '{}'".format(filename)) else: emsg = "read_xml_file: Filename, '{}', does not exist" raise CCPPError(emsg.format(filename)) - # End if + # end if if logger: logger.debug("Read XML file, '{}'".format(filename)) # End if @@ -248,7 +232,7 @@ class PrettyElementTree(ET.ElementTree): def __init__(self, element=None, file=None): """Initialize a PrettyElementTree object""" - super(PrettyElementTree, self).__init__(element, file) + super().__init__(element, file) def _write(self, outfile, line, indent, eol=os.linesep): """Write as an ASCII string to """ @@ -268,7 +252,7 @@ def _inc_pos(outstr, text, txt_beg): # end if emsg = "No output at {} of {}\n{}".format(txt_beg, len(text), text[txt_beg:txt_end]) - raise DatatableInternalError(emsg) + raise XMLToolsInternalError(emsg) def write(self, file, encoding="us-ascii", xml_declaration=None, default_namespace=None, method="xml", @@ -276,26 +260,26 @@ def write(self, file, encoding="us-ascii", xml_declaration=None, """Subclassed write method to format output.""" if PY3 and (PYSUBVER >= 4): if PYSUBVER >= 8: - input = ET.tostring(self.getroot(), - encoding=encoding, method=method, - xml_declaration=xml_declaration, - default_namespace=default_namespace, - short_empty_elements=short_empty_elements) + et_str = ET.tostring(self.getroot(), + encoding=encoding, method=method, + xml_declaration=xml_declaration, + default_namespace=default_namespace, + short_empty_elements=short_empty_elements) else: - input = ET.tostring(self.getroot(), - encoding=encoding, method=method, - short_empty_elements=short_empty_elements) + et_str = ET.tostring(self.getroot(), + encoding=encoding, method=method, + short_empty_elements=short_empty_elements) # end if else: - input = ET.tostring(self.getroot(), - encoding=encoding, method=method) + et_str = ET.tostring(self.getroot(), + encoding=encoding, method=method) # end if if PY3: fmode = 'wt' - root = str(input, encoding="utf-8") + root = str(et_str, encoding="utf-8") else: fmode = 'w' - root = input + root = et_str # end if indent = 0 last_write_text = False diff --git a/scripts/state_machine.py b/scripts/state_machine.py index 966ad04f..393e423d 100644 --- a/scripts/state_machine.py +++ b/scripts/state_machine.py @@ -30,7 +30,7 @@ class StateMachine: >>> StateMachine([('ab','a','b','a')]).final_state('ab') 'b' >>> StateMachine([('ab','a','b','a')]).transition_regex('ab') - re.compile('a$') + re.compile('a$, re.IGNORECASE') >>> StateMachine([('ab','a','b','a')]).function_match('foo_a', transition='ab') ('foo', 'a', 'ab') >>> StateMachine([('ab','a','b',r'ax?')]).function_match('foo_a', transition='ab') @@ -162,8 +162,9 @@ def __setitem__(self, key, value): if len(value) != 3: raise ValueError("Invalid transition ({}), should be of the form (inital_state, final_state, regex).".format(value)) # end if - regex = re.compile(value[2] + r"$") - function = re.compile(FORTRAN_ID + r"_(" + value[2] + r")$") + regex = re.compile(value[2] + r"$", re.IGNORECASE) + function = re.compile(FORTRAN_ID + r"_(" + value[2] + r")$", + re.IGNORECASE) self.__stt__[key] = (value[0], value[1], regex, function) def __delitem__(self, key): diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 472556cb..cc34a750 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -361,7 +361,7 @@ def register_action(self, vaction): @classmethod def is_suite_variable(cls, var): """Return True iff belongs to our Suite""" - return var and (var.source.type == _API_SUITE_VAR_NAME) + return var and (var.source.ptype == _API_SUITE_VAR_NAME) def is_local_variable(self, var): """Return the local variable matching if one is found belonging @@ -433,7 +433,7 @@ def add_call_list_variable(self, newvar, exists_ok=False, if dvar is None: emsg = "{}: Could not find dimension {} in {}" raise ParseInternalError(emsg.format(self.name, - stdname, vardim)) + vardim, stdname)) # end if elif self.parent is None: errmsg = 'No call_list found for {}'.format(newvar) @@ -831,7 +831,7 @@ def match_variable(self, var, vstdname=None, vdims=None): found_var = self.parent.add_variable_to_call_tree(dict_var, vmatch=vmatch) new_vdims = vdims - elif dict_var.source.type in _API_LOCAL_VAR_TYPES: + elif dict_var.source.ptype in _API_LOCAL_VAR_TYPES: # We cannot change the dimensions of locally-declared variables # Using a loop substitution is invalid because the loop variable # value has not yet been set. @@ -1716,7 +1716,7 @@ def find_variable(self, standard_name=None, source_var=None, search_call_list=search_call_list, loop_subst=loop_subst) if fvar and fvar.is_constituent(): - if fvar.source.type == ConstituentVarDict.constitutent_source_type(): + if fvar.source.ptype == ConstituentVarDict.constitutent_source_type(): # We found this variable in the constituent dictionary, # add it to our call list self.add_call_list_variable(fvar, exists_ok=True) From 3d75c77a5f6404a8b8500ac73217ad08e908f1b9 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Mon, 28 Aug 2023 03:18:40 -0600 Subject: [PATCH 2/8] cleanup and fixes --- scripts/constituents.py | 2 +- scripts/ddt_library.py | 104 +++++----- scripts/fortran_tools/fortran_write.py | 200 ++++---------------- scripts/fortran_tools/parse_fortran_file.py | 52 ++--- scripts/host_cap.py | 10 +- scripts/parse_tools/xml_tools.py | 158 +++++++++------- 6 files changed, 197 insertions(+), 329 deletions(-) diff --git a/scripts/constituents.py b/scripts/constituents.py index be8d4774..659f997f 100644 --- a/scripts/constituents.py +++ b/scripts/constituents.py @@ -398,7 +398,7 @@ def constituent_module_name(self): if not ((self.parent is not None) and hasattr(self.parent.parent, "constituent_module")): emsg = "ConstituentVarDict parent not HostModel?" - emsg += "\nparent is '{}'".format(type(self.parent.parent)) + emsg += f"\nparent is '{type_name(self.parent.parent)}'" raise ParseInternalError(emsg) # end if return self.parent.parent.constituent_module diff --git a/scripts/ddt_library.py b/scripts/ddt_library.py index 4be53963..30614226 100644 --- a/scripts/ddt_library.py +++ b/scripts/ddt_library.py @@ -44,11 +44,11 @@ def __init__(self, new_field, var_ref, run_env, recur=False): else: # Recurse to find correct (tail) location for self.__field = VarDDT(new_field, var_ref.field, run_env, recur=True) - # end if + # End if if ((not recur) and run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG)): run_env.logger.debug('Adding DDT field, {}'.format(self)) - # end if + # End if def is_ddt(self): """Return True iff is a DDT type.""" @@ -66,18 +66,18 @@ def get_prop_value(self, name): pvalue = super().get_prop_value(name) else: pvalue = self.field.get_prop_value(name) - # end if + # End if return pvalue def intrinsic_elements(self, check_dict=None): """Return the Var intrinsic elements for the leaf Var object. - See Var.intrinsic_elements for details + See Var.intrinsic_elem for details """ if self.field is None: pvalue = super().intrinsic_elements(check_dict=check_dict) else: pvalue = self.field.intrinsic_elements(check_dict=check_dict) - # end if + # End if return pvalue def clone(self, subst_dict, source_name=None, source_type=None, @@ -98,7 +98,7 @@ def clone(self, subst_dict, source_name=None, source_type=None, source_name=source_name, source_type=source_type, context=context) - # end if + # End if return clone_var def call_string(self, var_dict, loop_vars=None): @@ -109,7 +109,7 @@ def call_string(self, var_dict, loop_vars=None): if self.field is not None: call_str += '%' + self.field.call_string(var_dict, loop_vars=loop_vars) - # end if + # End if return call_str def write_def(self, outfile, indent, ddict, allocatable=False, dummy=False): @@ -122,7 +122,7 @@ def write_def(self, outfile, indent, ddict, allocatable=False, dummy=False): else: self.field.write_def(outfile, indent, ddict, allocatable=allocatable, dummy=dummy) - # end if + # End if @staticmethod def __var_rep(var, prefix=""): @@ -137,14 +137,14 @@ def __var_rep(var, prefix=""): lstr = '{}%{}({})'.format(prefix, lname, ', '.join(ldims)) else: lstr = '{}({})'.format(lname, ', '.join(ldims)) - # end if + # End if else: if prefix: lstr = '{}%{}'.format(prefix, lname) else: lstr = '{}'.format(lname) - # end if - # end if + # End if + # End if return lstr def __repr__(self): @@ -160,9 +160,9 @@ def __repr__(self): elif isinstance(field, Var): lstr = self.__var_rep(field, prefix=lstr) field = None - # end if + # End if sep = '%' - # end while + # End while return "".format(lstr) def __str__(self): @@ -201,33 +201,33 @@ def __init__(self, name, run_env, ddts=None, logger=None): ddts = list() elif not isinstance(ddts, list): ddts = [ddts] - # end if + # End if # Add all the DDT headers, then process for ddt in ddts: if not isinstance(ddt, MetadataSection): errmsg = 'Invalid DDT metadata type, {}' - raise ParseInternalError(errmsg.format(type(ddt).__name__)) - # end if + raise ParseInternalError(errmsg.format(type(ddt))) + # End if if not ddt.header_type == 'ddt': errmsg = 'Metadata table header is for a {}, should be DDT' raise ParseInternalError(errmsg.format(ddt.header_type)) - # end if + # End if if ddt.title in self: errmsg = "Duplicate DDT, {}, found{}, original{}" ctx = context_string(ddt.source.context) octx = context_string(self[ddt.title].source.context) raise CCPPError(errmsg.format(ddt.title, ctx, octx)) - # end if - if logger and logger.isEnabledFor(logging.DEBUG): - lmsg = f"Adding DDT {ddt.title} to {self.name}" - logger.debug(lmsg) - # end if + # End if + if logger is not None: + lmsg = 'Adding DDT {} to {}' + logger.debug(lmsg.format(ddt.title, self.name)) + # End if self[ddt.title] = ddt dlen = len(ddt.module) if dlen > self.__max_mod_name_len: self.__max_mod_name_len = dlen - # end if - # end for + # End if + # End for def check_ddt_type(self, var, header, lname=None): """If is a DDT, check to make sure it is in this DDT library. @@ -239,21 +239,16 @@ def check_ddt_type(self, var, header, lname=None): if vtype not in self: if lname is None: lname = var.get_prop_value('local_name') - # end if + # End if errmsg = 'Variable {} is of unknown type ({}) in {}' ctx = context_string(var.context) raise CCPPError(errmsg.format(lname, vtype, header.title, ctx)) - # end if - # end if (no else needed) + # End if + # End if (no else needed) - def collect_ddt_fields(self, var_dict, var, run_env, - ddt=None, skip_duplicates=False): + def collect_ddt_fields(self, var_dict, var, run_env, ddt=None): """Add all the reachable fields from DDT variable of type, to . Each field is added as a VarDDT. - Note: By default, it is an error to try to add a duplicate - field to (i.e., the field already exists in - or one of its parents). To simply skip duplicate - fields, set to True. """ if ddt is None: vtype = var.get_prop_value('type') @@ -264,8 +259,8 @@ def collect_ddt_fields(self, var_dict, var, run_env, ctx = context_string(var.context) errmsg = "Variable, {}, is not a known DDT{}" raise ParseInternalError(errmsg.format(lname, ctx)) - # end if - # end if + # End if + # End if for dvar in ddt.variable_list(): subvar = VarDDT(dvar, var, self.run_env) dvtype = dvar.get_prop_value('type') @@ -273,25 +268,22 @@ def collect_ddt_fields(self, var_dict, var, run_env, # If DDT in our library, we need to add sub-fields recursively. subddt = self[dvtype] self.collect_ddt_fields(var_dict, subvar, run_env, ddt=subddt) - # end if - # add_variable only checks the current dictionary. By default, - # for a DDT, the variable also cannot be in our parent - # dictionaries. - stdname = dvar.get_prop_value('standard_name') - pvar = var_dict.find_variable(standard_name=stdname, + else: + # add_variable only checks the current dictionary. For a + # DDT, the variable also cannot be in our parent dictionaries. + stdname = dvar.get_prop_value('standard_name') + pvar = var_dict.find_variable(standard_name=stdname, any_scope=True) - if pvar and (not skip_duplicates): - emsg = "Attempt to add duplicate DDT sub-variable, {}{}." - emsg += "\nVariable originally defined{}" - ntx = context_string(dvar.context) - ctx = context_string(pvar.context) - raise CCPPError(emsg.format(stdname, ntx, ctx)) - # end if - # Add this intrinsic to - if not pvar: + if pvar: + emsg = "Attempt to add duplicate DDT sub-variable, {}{}." + emsg += "\nVariable originally defined{}" + ntx = context_string(dvar.context) + ctx = context_string(pvar.context) + raise CCPPError(emsg.format(stdname, ntx, ctx)) + # end if + # Add this intrinsic to var_dict.add_variable(subvar, run_env) - # end if - # end for + # End for def ddt_modules(self, variable_list, ddt_mods=None): """Collect information for module use statements. @@ -300,14 +292,14 @@ def ddt_modules(self, variable_list, ddt_mods=None): """ if ddt_mods is None: ddt_mods = set() # Need a new set for every call - # end if + # End if for var in variable_list: vtype = var.get_prop_value('type') if vtype in self: module = self[vtype].module ddt_mods.add((module, vtype)) - # end if - # end for + # End if + # End for return ddt_mods def write_ddt_use_statements(self, variable_list, outfile, indent, pad=0): @@ -321,7 +313,7 @@ def write_ddt_use_statements(self, variable_list, outfile, indent, pad=0): slen = ' '*(pad - len(dmod)) ustring = 'use {},{} only: {}' outfile.write(ustring.format(dmod, slen, dtype), indent) - # end for + # End for @property def name(self): diff --git a/scripts/fortran_tools/fortran_write.py b/scripts/fortran_tools/fortran_write.py index f9bcfa3f..a238dcc5 100644 --- a/scripts/fortran_tools/fortran_write.py +++ b/scripts/fortran_tools/fortran_write.py @@ -4,9 +4,7 @@ """Code to write Fortran code """ -import math - -class FortranWriter: +class FortranWriter(object): """Class to turn output into properly continued and indented Fortran code >>> FortranWriter("foo.F90", 'r', 'test', 'mod_name') #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): @@ -61,9 +59,9 @@ class FortranWriter: def indent(self, level=0, continue_line=False): 'Return an indent string for any level' - indent = self.indent_size * level + indent = self._indent * level if continue_line: - indent = indent + self.__continue_indent + indent = indent + self._continue_indent # End if return indent*' ' @@ -73,55 +71,22 @@ def find_best_break(self, choices, last=None): """Find the best line break point given . If is present, use it as a target line length.""" if last is None: - last = self.__line_fill + last = self._line_fill # End if # Find largest good break - possible = [x for x in choices if 0 < x < last] + possible = [x for x in choices if x < last] if not possible: - best = self.__line_max + 1 + best = self._line_max + 1 else: best = max(possible) # End if - if (best > self.__line_max) and (last < self.__line_max): - best = self.find_best_break(choices, last=self.__line_max) + if (best > self._line_max) and (last < self._line_max): + best = self.find_best_break(choices, last=self._line_max) # End if return best ########################################################################### - @staticmethod - def _in_quote(test_str): - """Return True if ends in a character context. - >>> FortranWriter._in_quote("hi'mom") - True - >>> FortranWriter._in_quote("hi mom") - False - >>> FortranWriter._in_quote("'hi mom'") - False - >>> FortranWriter._in_quote("'hi"" mom'") - False - """ - in_single_char = False - in_double_char = False - for char in test_str: - if in_single_char: - if char == "'": - in_single_char = False - # end if - elif in_double_char: - if char == '"': - in_double_char = False - # end if - elif char == "'": - in_single_char = True - elif char == '"': - in_double_char = True - # end if - # end for - return in_single_char or in_double_char - - ########################################################################### - def write(self, statement, indent_level, continue_line=False): """Write to the open file, indenting to (see self.indent). @@ -137,18 +102,9 @@ def write(self, statement, indent_level, continue_line=False): # End for else: istr = self.indent(indent_level, continue_line) - ostmt = statement.strip() - is_comment_stmt = ostmt and (ostmt[0] == '!') - in_comment = "" - if ostmt and (ostmt[0] != '&'): - # Skip indent for continue that is in the middle of a - # token or a quoted region - outstr = istr + ostmt - else: - outstr = ostmt - # end if + outstr = istr + statement.strip() line_len = len(outstr) - if line_len > self.__line_fill: + if line_len > self._line_fill: # Collect pretty break points spaces = list() commas = list() @@ -169,14 +125,9 @@ def write(self, statement, indent_level, continue_line=False): elif outstr[sptr] == '"': in_double_char = True elif outstr[sptr] == '!': - # Comment in non-character context + # Comment in non-character context, suck in rest of line spaces.append(sptr-1) - in_comment = "! " # No continue for comment - if ((not is_comment_stmt) and - (sptr >= self.__max_comment_start)): - # suck in rest of line - sptr = line_len - 1 - # end if + sptr = line_len - 1 elif outstr[sptr] == ' ': # Non-quote spaces are where we can break spaces.append(sptr) @@ -189,62 +140,31 @@ def write(self, statement, indent_level, continue_line=False): # End if (no else, other characters will be ignored) sptr = sptr + 1 # End while - # Before looking for best space, reject any that are on a - # comment line but before any significant characters - if outstr.lstrip()[0] == '!': - first_space = outstr.index('!') + 1 - while ((outstr[first_space] == '!' or - outstr[first_space] == ' ') and - (first_space < line_len)): - first_space += 1 - # end while - if min(spaces) < first_space: - spaces = [x for x in spaces if x >= first_space] - # end if best = self.find_best_break(spaces) - if best >= self.__line_fill: - best = min(best, self.find_best_break(commas)) + if best >= self._line_fill: + best = self.find_best_break(commas) # End if - line_continue = False - if best >= self.__line_max: - # This is probably a bad situation so we have to break - # in an ugly spot - best = self.__line_max - 1 - if len(outstr) > best: - line_continue = '&' - # end if - # end if - if len(outstr) > best: - if self._in_quote(outstr[0:best+1]): - line_continue = '&' - else: - # If next line is just comment, do not use continue - line_continue = outstr[best+1:].lstrip()[0] != '!' - # end if - elif not line_continue: - line_continue = len(outstr) > best - # End if - if in_comment or is_comment_stmt: + if best > self._line_max: + # This is probably a bad situation that might not + # compile, just write the line and hope for the best. line_continue = False - # end if + elif len(outstr) > best: + # If next line is just comment, do not use continue + # NB: Is this a Fortran issue or just a gfortran issue? + line_continue = outstr[best+1:].lstrip()[0] != '!' + else: + line_continue = True + # End if if line_continue: - fill = "{}&".format((self.__line_fill - best)*' ') + fill = "{}&".format((self._line_fill - best)*' ') else: - fill = "" + fill = '' # End if - outline = f"{outstr[0:best+1]}{fill}".rstrip() - self.__file.write(f"{outline}\n") - if best <= 0: - imsg = "Internal ERROR: Unable to break line" - raise ValueError(f"{imsg}, '{statement}'") - # end if - statement = in_comment + outstr[best+1:] - if isinstance(line_continue, str) and statement: - statement = line_continue + statement - # end if + self._file.write("{}{}\n".format(outstr[0:best+1], fill)) + statement = outstr[best+1:] self.write(statement, indent_level, continue_line=line_continue) else: - self.__file.write("{}\n".format(outstr)) + self._file.write("{}\n".format(outstr)) # End if # End if @@ -255,7 +175,7 @@ def __init__(self, filename, mode, file_description, module_name, line_fill=None, line_max=None): """Initialize thie FortranWriter object. Some boilerplate is written automatically.""" - self.__file_desc = file_description.replace('\n', '\n!! ') + self.__file_desc = file_description self.__module = module_name # We only handle writing situations (for now) and only text if 'r' in mode: @@ -264,27 +184,26 @@ def __init__(self, filename, mode, file_description, module_name, if 'b' in mode: raise ValueError('Binary mode not allowed in FortranWriter object') # End if - self.__file = open(filename, mode) + self._file = open(filename, mode) if indent is None: - self.__indent = FortranWriter.__INDENT + self._indent = FortranWriter.__INDENT else: - self.__indent = indent + self._indent = indent # End if if continue_indent is None: - self.__continue_indent = FortranWriter.__CONTINUE_INDENT + self._continue_indent = FortranWriter.__CONTINUE_INDENT else: - self.__continue_indent = continue_indent + self._continue_indent = continue_indent # End if if line_fill is None: - self.__line_fill = FortranWriter.__LINE_FILL + self._line_fill = FortranWriter.__LINE_FILL else: - self.__line_fill = line_fill + self._line_fill = line_fill # End if - self.__max_comment_start = math.ceil(self.__line_fill * 3 / 4) if line_max is None: - self.__line_max = FortranWriter.__LINE_MAX + self._line_max = FortranWriter.__LINE_MAX else: - self.__line_max = line_max + self._line_max = line_max # End if ########################################################################### @@ -315,7 +234,7 @@ def __enter__(self, *args): def __exit__(self, *args): self.write(FortranWriter.__MOD_FOOTER.format(module=self.__module), 0) - self.__file.close() + self._file.close() return False ########################################################################### @@ -328,45 +247,6 @@ def module_header(self): ########################################################################### - def comment(self, comment, indent): - """Write a Fortran comment with contents, """ - mlcomment = comment.replace('\n', '\n! ') # No backslash in f string - self.write(f"! {mlcomment}", indent) - - ########################################################################### - - def blank_line(self): - """Write a blank line""" - self.write("", 0) - - ########################################################################### - - def include(self, filename): - """Insert the contents of verbatim.""" - with open(filename, 'r') as infile: - for line in infile: - self.__file.write(line) - # end for - # end with - - ########################################################################### - - @property - def line_fill(self): - """Return the target line length for this Fortran file""" - return self.__line_fill - - ########################################################################### - - @property - def indent_size(self): - """Return the number of spaces for each indent level for this - Fortran file - """ - return self.__indent - - ########################################################################### - @classmethod def copyright(cls): """Return the standard Fortran file copyright string""" diff --git a/scripts/fortran_tools/parse_fortran_file.py b/scripts/fortran_tools/parse_fortran_file.py index 7c6493e7..f37b3377 100755 --- a/scripts/fortran_tools/parse_fortran_file.py +++ b/scripts/fortran_tools/parse_fortran_file.py @@ -563,39 +563,29 @@ def parse_preamble_data(statements, pobj, spec_name, endmatch, run_env): module=spec_name, var_dict=var_dict) mheaders.append(mheader) - if (run_env.logger and - run_env.logger.isEnabledFor(logging.DEBUG)): + if run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG): ctx = context_string(pobj, nodir=True) msg = 'Adding header {}{}' run_env.logger.debug(msg.format(mheader.table_name, ctx)) - # end if break - elif type_def is not None: + elif ((type_def is not None) and (active_table is not None) and + (type_def[0].lower() == active_table.lower())): # Put statement back so caller knows where we are statements.insert(0, statement) - if ((active_table is not None) and - (type_def[0].lower() == active_table.lower())): - statements, ddt = parse_type_def(statements, type_def, - spec_name, pobj, run_env) - if ddt is None: - ctx = context_string(pobj, nodir=True) - msg = "No DDT found at '{}'{}" - raise CCPPError(msg.format(statement, ctx)) - # End if - mheaders.append(ddt) - if (run_env.logger and - run_env.logger.isEnabledFor(logging.DEBUG)): - ctx = context_string(pobj, nodir=True) - msg = 'Adding DDT {}{}' - run_env.logger.debug(msg.format(ddt.table_name, ctx)) - # End if - active_table = None - else: - # We found a type definition but it is not one with - # metadata. Just parse it and throw away what is found. - _ = parse_type_def(statements, type_def, - spec_name, pobj, run_env) - # end if + statements, ddt = parse_type_def(statements, type_def, + spec_name, pobj, run_env) + if ddt is None: + ctx = context_string(pobj, nodir=True) + msg = "No DDT found at '{}'{}" + raise CCPPError(msg.format(statement, ctx)) + # End if + mheaders.append(ddt) + if run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG): + ctx = context_string(pobj, nodir=True) + msg = 'Adding DDT {}{}' + run_env.logger.debug(msg.format(ddt.table_name, ctx)) + # End if + active_table = None elif active_table is not None: # We should have a variable definition to add if ((not is_comment_statement(statement)) and @@ -796,7 +786,6 @@ def parse_specification(pobj, statements, run_env, mod_name=None, # End program or module pmatch = endmatch.match(statement) asmatch = _ARG_TABLE_START_RE.match(statement) - type_def = fortran_type_definition(statement) if pmatch is not None: # We never found a contains statement inspec = False @@ -823,13 +812,6 @@ def parse_specification(pobj, statements, run_env, mod_name=None, # End if inspec = pobj.in_region('MODULE', region_name=mod_name) break - elif type_def: - # We have a type definition without metadata - # Just parse it and throw away what is found. - # Put statement back so caller knows where we are - statements.insert(0, statement) - _ = parse_type_def(statements, type_def, - spec_name, pobj, run_env) elif is_contains_statement(statement, inmod): inspec = False break diff --git a/scripts/host_cap.py b/scripts/host_cap.py index 9c88cf34..f7447de2 100644 --- a/scripts/host_cap.py +++ b/scripts/host_cap.py @@ -8,7 +8,7 @@ import logging import os # CCPP framework imports -from ccpp_suite import API +from ccpp_suite import API, API_SOURCE_NAME from ccpp_state_machine import CCPP_STATE_MACH from constituents import ConstituentVarDict, CONST_DDT_NAME, CONST_DDT_MOD from ddt_library import DDTLibrary @@ -32,9 +32,7 @@ end subroutine {host_model}_ccpp_physics_{stage} ''' -_API_SRC_NAME = "CCPP_API" - -_API_SOURCE = ParseSource(_API_SRC_NAME, "MODULE", +_API_SOURCE = ParseSource(API_SOURCE_NAME, "MODULE", ParseContext(filename="host_cap.F90")) _API_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', @@ -60,7 +58,7 @@ 'suites':''}) # Used to prevent loop substitution lookups -_BLANK_DICT = VarDictionary(_API_SRC_NAME, _MVAR_DUMMY_RUN_ENV) +_BLANK_DICT = VarDictionary(API_SOURCE_NAME, _MVAR_DUMMY_RUN_ENV) ############################################################################### def suite_part_list(suite, stage): @@ -475,7 +473,7 @@ def write_host_cap(host_model, api, output_dir, run_env): subst_dict['intent'] = 'inout' # End if hdvars.append(hvar.clone(subst_dict, - source_name=_API_SRC_NAME)) + source_name=API_SOURCE_NAME)) # End for lnames = [x.get_prop_value('local_name') for x in apivars + hdvars] api_vlist = ", ".join(lnames) diff --git a/scripts/parse_tools/xml_tools.py b/scripts/parse_tools/xml_tools.py index 0886356d..414ffc5a 100644 --- a/scripts/parse_tools/xml_tools.py +++ b/scripts/parse_tools/xml_tools.py @@ -7,19 +7,26 @@ # Python library imports from __future__ import print_function import os +import os.path import re -import shutil import subprocess import sys import xml.etree.ElementTree as ET sys.path.insert(0, os.path.dirname(__file__)) +# pylint: disable=wrong-import-position +try: + from distutils.spawn import find_executable + _XMLLINT = find_executable('xmllint') +except ImportError: + _XMLLINT = None +# End try # CCPP framework imports from parse_source import CCPPError from parse_log import init_log, set_log_to_null +# pylint: enable=wrong-import-position # Global data _INDENT_STR = " " -_XMLLINT = shutil.which('xmllint') # Blank if not installed beg_tag_re = re.compile(r"([<][^/][^<>]*[^/][>])") end_tag_re = re.compile(r"([<][/][^<>/]+[>])") simple_tag_re = re.compile(r"([<][^/][^<>/]+[/][>])") @@ -29,14 +36,6 @@ PYSUBVER = sys.version_info[1] _LOGGER = None -############################################################################### -class XMLToolsInternalError(ValueError): -############################################################################### - """Error class for reporting internal errors""" - def __init__(self, message): - """Initialize this exception""" - super().__init__(message) - ############################################################################### def call_command(commands, logger, silent=False): ############################################################################### @@ -54,12 +53,35 @@ def call_command(commands, logger, silent=False): result = False outstr = '' try: - cproc = subprocess.run(commands, check=True, - capture_output=True) - if not silent: - logger.debug(cproc.stdout) - # end if - result = cproc.returncode == 0 + if PY3: + if PYSUBVER > 6: + cproc = subprocess.run(commands, check=True, + capture_output=True) + if not silent: + logger.debug(cproc.stdout) + # End if + result = cproc.returncode == 0 + elif PYSUBVER >= 5: + cproc = subprocess.run(commands, check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if not silent: + logger.debug(cproc.stdout) + # End if + result = cproc.returncode == 0 + else: + raise ValueError("Python 3 must be at least version 3.5") + # End if + else: + pproc = subprocess.Popen(commands, stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, _ = pproc.communicate() + if not silent: + logger.debug(output) + # End if + result = pproc.returncode == 0 + # End if except (OSError, CCPPError, subprocess.CalledProcessError) as err: if silent: result = False @@ -68,9 +90,9 @@ def call_command(commands, logger, silent=False): emsg = "Execution of '{}' failed with code:\n" outstr = emsg.format(cmd, err.returncode) outstr += "{}".format(err.output) - raise CCPPError(outstr) from err - # end if - # end of try + raise CCPPError(outstr) + # End if + # End of try return result ############################################################################### @@ -80,8 +102,6 @@ def find_schema_version(root): Find the version of the host registry file represented by root >>> find_schema_version(ET.fromstring('')) [1, 0] - >>> find_schema_version(ET.fromstring('')) - [2, 0] >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: Illegal version string, '1.a' @@ -98,33 +118,33 @@ def find_schema_version(root): verbits = None if 'version' not in root.attrib: raise CCPPError("version attribute required") - # end if + # End if version = root.attrib['version'] versplit = version.split('.') try: if len(versplit) != 2: raise CCPPError('oops') - # end if (no else needed) + # End if (no else needed) try: verbits = [int(x) for x in versplit] except ValueError as verr: - raise CCPPError(verr) from verr - # end try + raise CCPPError(verr) + # End try if verbits[0] < 1: raise CCPPError('Major version must be at least 1') - # end if + # End if if verbits[1] < 0: raise CCPPError('Minor version must be non-negative') - # end if + # End if except CCPPError as verr: errstr = """Illegal version string, '{}' Format must be .""" ve_str = str(verr) if ve_str: errstr = ve_str + '\n' + errstr - # end if - raise CCPPError(errstr.format(version)) from verr - # end try + # End if + raise CCPPError(errstr.format(version)) + # End try return verbits ############################################################################### @@ -141,10 +161,10 @@ def find_schema_file(schema_root, version, schema_path=None): schema_file = os.path.join(schema_path, schema_filename) else: schema_file = schema_filename - # end if + # End if if os.path.exists(schema_file): return schema_file - # end if + # End if return None ############################################################################### @@ -158,42 +178,38 @@ def validate_xml_file(filename, schema_root, version, logger, # Check the filename if not os.path.isfile(filename): raise CCPPError("validate_xml_file: Filename, '{}', does not exist".format(filename)) - # end if + # End if if not os.access(filename, os.R_OK): raise CCPPError("validate_xml_file: Cannot open '{}'".format(filename)) - # end if - if os.path.isfile(schema_root): - # We already have a file, just use it - schema_file = schema_root - else: - if not schema_path: - # Find the schema, based on the model version - thispath = os.path.abspath(__file__) - pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) - schema_path = os.path.join(pdir, 'schema') - # end if - schema_file = find_schema_file(schema_root, version, schema_path) - if not (schema_file and os.path.isfile(schema_file)): - verstring = '.'.join([str(x) for x in version]) - emsg = """validate_xml_file: Cannot find schema for version {}, {} does not exist""" - raise CCPPError(emsg.format(verstring, schema_file)) - # end if - # end if + # End if + if not schema_path: + # Find the schema, based on the model version + thispath = os.path.abspath(__file__) + pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) + schema_path = os.path.join(pdir, 'schema') + # End if + schema_file = find_schema_file(schema_root, version, schema_path) + if not (schema_file and os.path.isfile(schema_file)): + verstring = '.'.join([str(x) for x in version]) + emsg = """validate_xml_file: Cannot find schema for version {}, + {} does not exist""" + raise CCPPError(emsg.format(verstring, schema_file)) + # End if if not os.access(schema_file, os.R_OK): emsg = "validate_xml_file: Cannot open schema, '{}'" raise CCPPError(emsg.format(schema_file)) - # end if - if _XMLLINT: + # End if + if _XMLLINT is not None: logger.debug("Checking file {} against schema {}".format(filename, schema_file)) cmd = [_XMLLINT, '--noout', '--schema', schema_file, filename] result = call_command(cmd, logger) return result - # end if + # End if lmsg = "xmllint not found, could not validate file {}" if error_on_noxmllint: raise CCPPError("validate_xml_file: " + lmsg.format(filename)) - # end if + # End if logger.warning(lmsg.format(filename)) return True # We could not check but still need to proceed @@ -213,13 +229,13 @@ def read_xml_file(filename, logger=None): root = tree.getroot() except ET.ParseError as perr: emsg = "read_xml_file: Cannot read {}, {}" - raise CCPPError(emsg.format(filename, perr)) from perr + raise CCPPError(emsg.format(filename, perr)) elif not os.access(filename, os.R_OK): raise CCPPError("read_xml_file: Cannot open '{}'".format(filename)) else: emsg = "read_xml_file: Filename, '{}', does not exist" raise CCPPError(emsg.format(filename)) - # end if + # End if if logger: logger.debug("Read XML file, '{}'".format(filename)) # End if @@ -232,7 +248,7 @@ class PrettyElementTree(ET.ElementTree): def __init__(self, element=None, file=None): """Initialize a PrettyElementTree object""" - super().__init__(element, file) + super(PrettyElementTree, self).__init__(element, file) def _write(self, outfile, line, indent, eol=os.linesep): """Write as an ASCII string to """ @@ -252,7 +268,7 @@ def _inc_pos(outstr, text, txt_beg): # end if emsg = "No output at {} of {}\n{}".format(txt_beg, len(text), text[txt_beg:txt_end]) - raise XMLToolsInternalError(emsg) + raise DatatableInternalError(emsg) def write(self, file, encoding="us-ascii", xml_declaration=None, default_namespace=None, method="xml", @@ -260,26 +276,26 @@ def write(self, file, encoding="us-ascii", xml_declaration=None, """Subclassed write method to format output.""" if PY3 and (PYSUBVER >= 4): if PYSUBVER >= 8: - et_str = ET.tostring(self.getroot(), - encoding=encoding, method=method, - xml_declaration=xml_declaration, - default_namespace=default_namespace, - short_empty_elements=short_empty_elements) + input = ET.tostring(self.getroot(), + encoding=encoding, method=method, + xml_declaration=xml_declaration, + default_namespace=default_namespace, + short_empty_elements=short_empty_elements) else: - et_str = ET.tostring(self.getroot(), - encoding=encoding, method=method, - short_empty_elements=short_empty_elements) + input = ET.tostring(self.getroot(), + encoding=encoding, method=method, + short_empty_elements=short_empty_elements) # end if else: - et_str = ET.tostring(self.getroot(), - encoding=encoding, method=method) + input = ET.tostring(self.getroot(), + encoding=encoding, method=method) # end if if PY3: fmode = 'wt' - root = str(et_str, encoding="utf-8") + root = str(input, encoding="utf-8") else: fmode = 'w' - root = et_str + root = input # end if indent = 0 last_write_text = False From 52626d09d2f8b89776bc4e58c51fc8aaee73d8df Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Mon, 28 Aug 2023 04:27:33 -0600 Subject: [PATCH 3/8] bugfixes --- scripts/ccpp_capgen.py | 5 ++--- scripts/ccpp_suite.py | 6 ++---- scripts/framework_env.py | 2 +- scripts/metadata_table.py | 2 +- scripts/state_machine.py | 2 +- test/advection_test/cld_ice.F90 | 12 ++++++------ test/advection_test/cld_ice.meta | 4 ++-- test/advection_test/cld_liq.F90 | 12 ++++++------ test/advection_test/cld_liq.meta | 4 ++-- test/capgen_test/test_host.F90 | 8 ++++---- 10 files changed, 27 insertions(+), 30 deletions(-) diff --git a/scripts/ccpp_capgen.py b/scripts/ccpp_capgen.py index d28bfd6a..dfa2e211 100755 --- a/scripts/ccpp_capgen.py +++ b/scripts/ccpp_capgen.py @@ -31,7 +31,7 @@ ## Capture the Framework root __SCRIPT_PATH = os.path.dirname(__file__) -__FRAMEWORK_ROOT = os.path.abspath(os.path.join(_SCRIPT_PATH, os.pardir)) +__FRAMEWORK_ROOT = os.path.abspath(os.path.join(__SCRIPT_PATH, os.pardir)) ## Init this now so that all Exceptions can be trapped _LOGGER = init_log(os.path.basename(__file__)) @@ -629,8 +629,7 @@ def capgen(run_env, return_db=False): cap_filenames = ccpp_api.write(outtemp_dir, run_env) if run_env.generate_host_cap: # Create a cap file - cap_module = host_model.ccpp_cap_name() - host_files = [write_host_cap(host_model, ccpp_api, cap_module, + host_files = [write_host_cap(host_model, ccpp_api, outtemp_dir, run_env)] else: host_files = list() diff --git a/scripts/ccpp_suite.py b/scripts/ccpp_suite.py index ba8f8609..c4d7fbab 100644 --- a/scripts/ccpp_suite.py +++ b/scripts/ccpp_suite.py @@ -729,7 +729,6 @@ def write_var_set_loop(ofile, varlist_name, var_list, indent, def write_suite_part_list_sub(self, ofile, errmsg_name, errcode_name): """Write the suite-part list subroutine""" - ofile.blank_line() inargs = f"suite_name, part_list, {errmsg_name}, {errcode_name}" ofile.write(f"subroutine {API.__part_fname}({inargs})", 1) oline = "character(len=*), intent(in) :: suite_name" @@ -740,7 +739,7 @@ def write_suite_part_list_sub(self, ofile, errmsg_name, errcode_name): self._errcode_var.write_def(ofile, 2, self) else_str = '' ename = self._errcode_var.get_prop_value('local_name') - ofile.write(f"{ename} = ''", 2) + ofile.write(f"{ename} = 0", 2) ename = self._errmsg_var.get_prop_value('local_name') ofile.write(f"{ename} = ''", 2) for suite in self.suites: @@ -820,8 +819,7 @@ def write_req_vars_sub(self, ofile, errmsg_name, errcode_name): protected = pvar.get_prop_value("protected") # end if # end if - elements = var.intrinsic_elements(check_dict=self.parent, - ddt_lib=self.__ddt_lib) + elements = var.intrinsic_elements(check_dict=self.parent) if (intent == 'in') and (not protected): if isinstance(elements, list): input_vars[1].add(stdname) diff --git a/scripts/framework_env.py b/scripts/framework_env.py index b5550ca6..6456134e 100644 --- a/scripts/framework_env.py +++ b/scripts/framework_env.py @@ -103,7 +103,7 @@ def __init__(self, logger, ndict=None, verbose=0, clean=False, # String of definitions, separated by spaces preproc_list = [x.strip() for x in preproc_defs.split(' ') if x] else: - wmsg = f"Error: Bad preproc list type, '{type_name{preproc_defs)}'" + wmsg = f"Error: Bad preproc list type, '{type_name(preproc_defs)}'" emsg += esep + wmsg esep = '\n' # end if diff --git a/scripts/metadata_table.py b/scripts/metadata_table.py index 3501e40e..5e9820d7 100644 --- a/scripts/metadata_table.py +++ b/scripts/metadata_table.py @@ -683,7 +683,7 @@ def __init__(self, table_name, table_type, run_env, parse_object=None, register_fortran_ddt_name(self.title) # end if # Categorize the variables - self._var_intents = {'in' : [], 'out' : [], 'inout' : [])} + self._var_intents = {'in' : [], 'out' : [], 'inout' : []} for var in self.variable_list(): intent = var.get_prop_value('intent') if intent is not None: diff --git a/scripts/state_machine.py b/scripts/state_machine.py index 393e423d..692da7a9 100644 --- a/scripts/state_machine.py +++ b/scripts/state_machine.py @@ -30,7 +30,7 @@ class StateMachine: >>> StateMachine([('ab','a','b','a')]).final_state('ab') 'b' >>> StateMachine([('ab','a','b','a')]).transition_regex('ab') - re.compile('a$, re.IGNORECASE') + re.compile('a$', re.IGNORECASE) >>> StateMachine([('ab','a','b','a')]).function_match('foo_a', transition='ab') ('foo', 'a', 'ab') >>> StateMachine([('ab','a','b',r'ax?')]).function_match('foo_a', transition='ab') diff --git a/test/advection_test/cld_ice.F90 b/test/advection_test/cld_ice.F90 index 5a91282f..a7da57e5 100644 --- a/test/advection_test/cld_ice.F90 +++ b/test/advection_test/cld_ice.F90 @@ -19,7 +19,7 @@ MODULE cld_ice !> \section arg_table_cld_ice_run Argument Table !! \htmlinclude arg_table_cld_ice_run.html !! - subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice, & + subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & errmsg, errflg) integer, intent(in) :: ncol @@ -27,7 +27,7 @@ subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice, & real(kind_phys), intent(inout) :: temp(:,:) real(kind_phys), intent(inout) :: qv(:,:) real(kind_phys), intent(in) :: ps(:) - REAL(kind_phys), intent(inout) :: cld_ice(:,:) + REAL(kind_phys), intent(inout) :: cld_ice_array(:,:) character(len=512), intent(out) :: errmsg integer, intent(out) :: errflg !---------------------------------------------------------------- @@ -44,7 +44,7 @@ subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice, & do ilev = 1, size(temp, 2) if (temp(icol, ilev) < tcld) then frz = MAX(qv(icol, ilev) - 0.5_kind_phys, 0.0_kind_phys) - cld_ice(icol, ilev) = cld_ice(icol, ilev) + frz + cld_ice_array(icol, ilev) = cld_ice_array(icol, ilev) + frz qv(icol, ilev) = qv(icol, ilev) - frz if (frz > 0.0_kind_phys) then temp(icol, ilev) = temp(icol, ilev) + 1.0_kind_phys @@ -58,16 +58,16 @@ END SUBROUTINE cld_ice_run !> \section arg_table_cld_ice_init Argument Table !! \htmlinclude arg_table_cld_ice_init.html !! - subroutine cld_ice_init(tfreeze, cld_ice, errmsg, errflg) + subroutine cld_ice_init(tfreeze, cld_ice_array, errmsg, errflg) real(kind_phys), intent(in) :: tfreeze - real(kind_phys), intent(inout) :: cld_ice(:,:) + real(kind_phys), intent(inout) :: cld_ice_array(:,:) character(len=512), intent(out) :: errmsg integer, intent(out) :: errflg errmsg = '' errflg = 0 - cld_ice = 0.0_kind_phys + cld_ice_array = 0.0_kind_phys tcld = tfreeze - 20.0_kind_phys end subroutine cld_ice_init diff --git a/test/advection_test/cld_ice.meta b/test/advection_test/cld_ice.meta index e4a961ab..f66888e0 100644 --- a/test/advection_test/cld_ice.meta +++ b/test/advection_test/cld_ice.meta @@ -41,7 +41,7 @@ units = Pa dimensions = (horizontal_loop_extent) intent = in -[ cld_ice ] +[ cld_ice_array ] standard_name = cloud_ice_dry_mixing_ratio advected = .true. units = kg kg-1 @@ -73,7 +73,7 @@ dimensions = () type = real | kind = kind_phys intent = in -[ cld_ice ] +[ cld_ice_array ] standard_name = cloud_ice_dry_mixing_ratio advected = .true. units = kg kg-1 diff --git a/test/advection_test/cld_liq.F90 b/test/advection_test/cld_liq.F90 index 2e1e5a57..9411e0cb 100644 --- a/test/advection_test/cld_liq.F90 +++ b/test/advection_test/cld_liq.F90 @@ -16,7 +16,7 @@ MODULE cld_liq !> \section arg_table_cld_liq_run Argument Table !! \htmlinclude arg_table_cld_liq_run.html !! - subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, cld_liq, & + subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, cld_liq_array, & errmsg, errflg) integer, intent(in) :: ncol @@ -25,7 +25,7 @@ subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, cld_liq, & real(kind_phys), intent(inout) :: temp(:,:) real(kind_phys), intent(inout) :: qv(:,:) real(kind_phys), intent(in) :: ps(:) - REAL(kind_phys), intent(inout) :: cld_liq(:,:) + REAL(kind_phys), intent(inout) :: cld_liq_array(:,:) character(len=512), intent(out) :: errmsg integer, intent(out) :: errflg !---------------------------------------------------------------- @@ -43,7 +43,7 @@ subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, cld_liq, & if ( (qv(icol, ilev) > 0.0_kind_phys) .and. & (temp(icol, ilev) <= tcld)) then cond = MIN(qv(icol, ilev), 0.1_kind_phys) - cld_liq(icol, ilev) = cld_liq(icol, ilev) + cond + cld_liq_array(icol, ilev) = cld_liq_array(icol, ilev) + cond qv(icol, ilev) = qv(icol, ilev) - cond if (cond > 0.0_kind_phys) then temp(icol, ilev) = temp(icol, ilev) + (cond * 5.0_kind_phys) @@ -57,10 +57,10 @@ END SUBROUTINE cld_liq_run !> \section arg_table_cld_liq_init Argument Table !! \htmlinclude arg_table_cld_liq_init.html !! - subroutine cld_liq_init(tfreeze, cld_liq, tcld, errmsg, errflg) + subroutine cld_liq_init(tfreeze, cld_liq_array, tcld, errmsg, errflg) real(kind_phys), intent(in) :: tfreeze - real(kind_phys), intent(out) :: cld_liq(:,:) + real(kind_phys), intent(out) :: cld_liq_array(:,:) real(kind_phys), intent(out) :: tcld character(len=512), intent(out) :: errmsg integer, intent(out) :: errflg @@ -69,7 +69,7 @@ subroutine cld_liq_init(tfreeze, cld_liq, tcld, errmsg, errflg) errmsg = '' errflg = 0 - cld_liq = 0.0_kind_phys + cld_liq_array = 0.0_kind_phys tcld = tfreeze - 20.0_kind_phys end subroutine cld_liq_init diff --git a/test/advection_test/cld_liq.meta b/test/advection_test/cld_liq.meta index 1186d741..4c87f4b7 100644 --- a/test/advection_test/cld_liq.meta +++ b/test/advection_test/cld_liq.meta @@ -47,7 +47,7 @@ units = Pa dimensions = (horizontal_loop_extent) intent = in -[ cld_liq ] +[ cld_liq_array ] standard_name = cloud_liquid_dry_mixing_ratio advected = .true. units = kg kg-1 @@ -79,7 +79,7 @@ dimensions = () type = real | kind = kind_phys intent = in -[ cld_liq ] +[ cld_liq_array ] standard_name = cloud_liquid_dry_mixing_ratio advected = .true. units = kg kg-1 diff --git a/test/capgen_test/test_host.F90 b/test/capgen_test/test_host.F90 index 55c7159e..2114cdac 100644 --- a/test/capgen_test/test_host.F90 +++ b/test/capgen_test/test_host.F90 @@ -130,7 +130,7 @@ logical function check_suite(test_suite) suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR 1 ', errflg, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -144,7 +144,7 @@ logical function check_suite(test_suite) 'input variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR 2 ', errflg, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -158,7 +158,7 @@ logical function check_suite(test_suite) 'output variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR 3 ', errflg, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -172,7 +172,7 @@ logical function check_suite(test_suite) 'required variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR 4 ', errflg, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then From 3c6a9e2fb2649cecf911e19a1e3e10a24895de63 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Fri, 1 Sep 2023 10:07:42 -0600 Subject: [PATCH 4/8] add code owners --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 97c9a2a7..22f6e1b0 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -3,7 +3,7 @@ # These owners will be the default owners for everything in the repo. #* @defunkt -* @climbfuji @gold2718 +* @climbfuji @gold2718 @dustinswales @mwaxmonsky @peverwhee # Order is important. The last matching pattern has the most precedence. # So if a pull request only touches javascript files, only these owners From 1be9b3d064fac5bb183eaee2dd4b9e9f4b37371c Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Fri, 1 Sep 2023 10:22:48 -0600 Subject: [PATCH 5/8] code cleanup --- test/capgen_test/test_host.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/capgen_test/test_host.F90 b/test/capgen_test/test_host.F90 index 2114cdac..55c7159e 100644 --- a/test/capgen_test/test_host.F90 +++ b/test/capgen_test/test_host.F90 @@ -130,7 +130,7 @@ logical function check_suite(test_suite) suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR 1 ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -144,7 +144,7 @@ logical function check_suite(test_suite) 'input variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR 2 ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -158,7 +158,7 @@ logical function check_suite(test_suite) 'output variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR 3 ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -172,7 +172,7 @@ logical function check_suite(test_suite) 'required variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR 4 ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then From c8ad82d587227f068e4a4205232374c5af8f2ee8 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Fri, 29 Sep 2023 16:55:54 -0600 Subject: [PATCH 6/8] generalize host --- scripts/ccpp_database_obj.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ccpp_database_obj.py b/scripts/ccpp_database_obj.py index 31aa450c..24579750 100644 --- a/scripts/ccpp_database_obj.py +++ b/scripts/ccpp_database_obj.py @@ -18,7 +18,7 @@ def __init__(self, message): super().__init__(message) class CCPPDatabaseObj: - """Ojbect with data and methods to provide information from a run of capgen. + """Object with data and methods to provide information from a run of capgen. """ def __init__(self, run_env, host_model=None, api=None, database_file=None): @@ -56,7 +56,7 @@ def db_from_file(self, run_env, database_file): datatable.xml file created by capgen. """ metadata_tables = {} - host_name = "cam" + host_name = "host" self.__host_model = HostModel(metadata_tables, host_name, run_env) self.__api = API(sdfs, host_model, scheme_headers, run_env) raise CCPPDatabaseObjError("ERROR: not supported") From 44ea46f73c7a226a3b78eb64e0c4b288ac26ffb6 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Fri, 29 Sep 2023 17:08:44 -0600 Subject: [PATCH 7/8] change to fstrings --- scripts/metavar.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/metavar.py b/scripts/metavar.py index 7f852c4c..64067dd1 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -1427,7 +1427,7 @@ def __init__(self, name, run_env, variables=None, self[stdname] = variables[key] # end for elif variables is not None: - raise ParseInternalError('Illegal type for variables, {} in {}'.format(type_name(variables), self.name)) + raise ParseInternalError(f'Illegal type for variables, {type_name(variables)} in {self.name}') # end if @property @@ -1623,15 +1623,13 @@ def add_variable_dimensions(self, var, ignore_sources, to_dict=None, else: ctx = context_string(var.context) # end if - err_ret += "{}: ".format(self.name) - err_ret += "Cannot find variable for dimension, {}, of {}{}" vstdname = var.get_prop_value('standard_name') - err_ret = err_ret.format(dimname, vstdname, ctx) + err_ret += f"{self.name}: " + err_ret += f"Cannot find variable for dimension, {dimname}, of {vstdname}{ctx}" if dvar: - err_ret += "\nFound {} from excluded source, '{}'{}" + err_ret += f"\nFound {lname} from excluded source, '{dvar.source.ptype}'{dctx}" lname = dvar.get_prop_value('local_name') dctx = context_string(dvar.context) - err_ret = err_ret.format(lname, dvar.source.ptype, dctx) # end if # end if # end if From 3f5c06f2a2661b1238973b0c0ef7f516aaf97bea Mon Sep 17 00:00:00 2001 From: Michael Waxmonsky Date: Fri, 29 Sep 2023 17:15:04 -0600 Subject: [PATCH 8/8] Cleaning up units regex. --- scripts/parse_tools/parse_checkers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/parse_tools/parse_checkers.py b/scripts/parse_tools/parse_checkers.py index ca5f1f51..ba8722d9 100755 --- a/scripts/parse_tools/parse_checkers.py +++ b/scripts/parse_tools/parse_checkers.py @@ -12,7 +12,13 @@ ######################################################################## -_UNITS_RE = re.compile(r"^[^/!@#$%^&*=()\|<>\[\]{}?,.]+$") +_UNITLESS_REGEX = "1" +_NON_LEADING_ZERO_NUM = "[1-9]\d*" +_NEGATIVE_NON_LEADING_ZERO_NUM = f"[-]{_NON_LEADING_ZERO_NUM}" +_UNIT_EXPONENT = f"({_NEGATIVE_NON_LEADING_ZERO_NUM}|{_NON_LEADING_ZERO_NUM})" +_UNIT_REGEX = f"[a-zA-Z]+{_UNIT_EXPONENT}?" +_UNITS_REGEX = f"^({_UNIT_REGEX}(\s{_UNIT_REGEX})*|{_UNITLESS_REGEX})$" +_UNITS_RE = re.compile(_UNITS_REGEX) def check_units(test_val, prop_dict, error): """Return if a valid unit, otherwise, None