Update master from gmtb/develop 2019/10/16

scripts/ccpp_prebuild.py

@@ -3,15 +3,16 @@
 # Standard modules
 import argparse
 import collections
+import importlib
 import itertools
 import logging
 import os
 import re
 import sys
 
-# Local modules
+# CCPP framework imports
 from common import encode_container, decode_container, decode_container_as_dict, execute
-from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE
+from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE
 from common import split_var_name_and_array_reference
 from metadata_parser import merge_dictionaries, parse_scheme_tables, parse_variable_tables
 from mkcap import Cap, CapsMakefile, CapsCMakefile, SchemesMakefile, SchemesCMakefile
@@ -32,15 +33,6 @@
 # BASEDIR is the current directory where this script is executed
 BASEDIR = os.getcwd()
 
-# SCRIPTDIR is the directory where the ccpp_prebuild.py and its Python modules are located
-SCRIPTDIR = os.path.abspath(os.path.split(__file__)[0])
-
-# SRCDIR is the directory where the CCPP framework source code (C, Fortran) is located
-SRCDIR = os.path.abspath(os.path.join(SCRIPTDIR, '..', 'src'))
-
-# Definition of variables (metadata tables) that are provided by CCPP
-CCPP_INTERNAL_VARIABLE_DEFINITON_FILE = os.path.join(SRCDIR, 'ccpp_types.F90')
-
 ###############################################################################
 # Functions and subroutines                                                   #
 ###############################################################################
@@ -71,10 +63,10 @@ def import_config(configfile):
 
     # Import the host-model specific CCPP prebuild config;
     # split into path and module name for import
-    configpath = os.path.abspath(os.path.split(configfile)[0])
-    configmodule = os.path.split(configfile)[1].rstrip('.py')
+    configpath = os.path.abspath(os.path.dirname(configfile))
+    configmodule = os.path.splitext(os.path.basename(configfile))[0]
     sys.path.append(configpath)
-    ccpp_prebuild_config = __import__(configmodule)
+    ccpp_prebuild_config = importlib.import_module(configmodule)
 
     # Definitions in host-model dependent CCPP prebuild config script
     config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES
@@ -99,7 +91,7 @@ def import_config(configfile):
     # Template code in host-model dependent CCPP prebuild config script
     config['ccpp_data_structure']            = ccpp_prebuild_config.CCPP_DATA_STRUCTURE
 
-    # Add model-intependent, CCPP-internal variable definition files
+    # Add model-independent, CCPP-internal variable definition files
     config['variable_definition_files'].append(CCPP_INTERNAL_VARIABLE_DEFINITON_FILE)
 
     # To handle new metadata: import DDT references (if exist)

scripts/common.py

@@ -2,6 +2,7 @@
 
 import keyword
 import logging
+import os
 import re
 import subprocess
 import sys
@@ -14,6 +15,16 @@
 
 CCPP_TYPE = 'ccpp_t'
 
+# SCRIPTDIR is the directory where ccpp_prebuild.py and its Python modules are located
+SCRIPTDIR = os.path.abspath(os.path.dirname(__file__))
+
+# SRCDIR is the directory where the CCPP framework source code (C, Fortran) is located
+SRCDIR = os.path.abspath(os.path.join(SCRIPTDIR, os.pardir, 'src'))
+
+# Definition of variables (metadata tables) that are provided by CCPP
+CCPP_INTERNAL_VARIABLE_DEFINITON_FILE = os.path.join(SRCDIR, 'ccpp_types.F90')
+
+# List of internal variables provided by the CCPP
 CCPP_INTERNAL_VARIABLES = {
     CCPP_ERROR_FLAG_VARIABLE : 'cdata%errflg',
     CCPP_ERROR_MSG_VARIABLE  : 'cdata%errmsg',

scripts/metadata2html.py

@@ -2,30 +2,89 @@
 
 import argparse
 import logging
+import importlib
 import os
 import sys
 
 # CCPP framework imports
+from common import CCPP_INTERNAL_VARIABLE_DEFINITON_FILE
 from parse_tools import init_log, set_log_level
 from metadata_table import MetadataHeader
 
+###############################################################################
+# Set up the command line argument parser and other global variables          #
+###############################################################################
+
 parser = argparse.ArgumentParser()
-parser.add_argument('--metafile', '-m', action='store',
-                    help='name of metadata file to convert',
-                    required=True)
+method = parser.add_mutually_exclusive_group(required=True)
+method.add_argument('--config', '-c', action='store',
+                    help='path to CCPP prebuild configuration file')
+method.add_argument('--metafile', '-m', action='store',
+                    help='name of metadata file to convert (requires -o)')
 parser.add_argument('--outputdir', '-o', action='store',
                     help='directory where to write the html files',
-                    required=True)
+                    required='--metafile' in sys.argv or '-m' in sys.argv)
 
-attributes = [ 'local_name', 'standard_name', 'long_name', 'units',
+# List and order of variable attributes to output to HTML
+ATTRIBUTES = [ 'local_name', 'standard_name', 'long_name', 'units',
                'type', 'dimensions', 'kind', 'intent', 'optional' ]
 
+###############################################################################
+# Functions and subroutines                                                   #
+###############################################################################
+
 def parse_arguments():
     """Parse command line arguments."""
     args = parser.parse_args()
+    config = args.config
     filename = args.metafile
     outdir = args.outputdir
-    return (filename, outdir)
+    return (config, filename, outdir)
+
+def import_config(configfile, logger):
+    """Import the configuration from a given configuration file"""
+
+    if not os.path.isfile(configfile):
+        raise Exception("Configuration file {0} not found".format(configfile))
+
+    # Import the host-model specific CCPP prebuild config;
+    # split into path and module name for import
+    configpath = os.path.abspath(os.path.dirname(configfile))
+    configmodule = os.path.splitext(os.path.basename(configfile))[0]
+    sys.path.append(configpath)
+    ccpp_prebuild_config = importlib.import_module(configmodule)
+
+    config = {}
+    # Definitions in host-model dependent CCPP prebuild config script
+    config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES
+    config['scheme_files']              = ccpp_prebuild_config.SCHEME_FILES
+    # Add model-independent, CCPP-internal variable definition files
+    config['variable_definition_files'].append(CCPP_INTERNAL_VARIABLE_DEFINITON_FILE)
+    # Output directory for converted metadata tables
+    config['metadata_html_output_dir'] = ccpp_prebuild_config.METADATA_HTML_OUTPUT_DIR
+
+    return config
+
+def get_metadata_files_from_config(config, logger):
+    """Create a list of metadata filenames for a CCPP prebuild configuration"""
+    filenames = []
+    for sourcefile in config['variable_definition_files'] + config['scheme_files'].keys():
+        metafile = os.path.splitext(sourcefile)[0]+'.meta'
+        if os.path.isfile(metafile):
+            filenames.append(metafile)
+        else:
+            # DH* Warn for now, raise exception later when
+            # old metadata format is no longer supported
+            logger.warn("Metadata file {} for source file {} not found, assuming old metadata format".format(
+                                                                                       metafile, sourcefile))
+    return filenames
+
+def get_output_directory_from_config(config, logger):
+    """Return the html output directory for a CCPP prebuild configuration"""
+    outdir = config['metadata_html_output_dir']
+    if not os.path.isdir(outdir):
+        raise Exception("Output directory {} for converted metadata tables does not exist".format(outdir))
+    return outdir
 
 def convert_to_html(filename_in, outdir, logger):
     """Convert a metadata file into html (one html file for each table)"""
@@ -34,7 +93,7 @@ def convert_to_html(filename_in, outdir, logger):
     logger.info("Converting file {} to HTML".format(filename_in))
     metadata_headers = MetadataHeader.parse_metadata_file(filename_in)
     for metadata_header in metadata_headers:
-        filename_out = metadata_header.to_html(outdir, attributes)
+        filename_out = metadata_header.to_html(outdir, ATTRIBUTES)
         if filename_out:
             logger.info("  ... wrote {}".format(filename_out))
 
@@ -43,8 +102,15 @@ def main():
     logger = init_log('metadata2html')
     set_log_level(logger, logging.INFO)
     # Convert metadata file
-    (filename, outdir) = parse_arguments()
-    convert_to_html(filename, outdir, logger)
+    (configfile, filename, outdir) = parse_arguments()
+    if configfile:
+        config = import_config(configfile, logger)
+        filenames = get_metadata_files_from_config(config, logger)
+        outdir = get_output_directory_from_config(config, logger)
+        for filename in filenames:
+            convert_to_html(filename, outdir, logger)
+    else:
+        convert_to_html(filename, outdir, logger)
 
 if __name__ == '__main__':
     main()

scripts/metavar.py

@@ -369,12 +369,7 @@ class Var(object):
                     VariableProperty('dimensions', list,
                                      check_fn_in=check_dimensions),
                     VariableProperty('type', str,
-                                     # DH* Avoid checking of DDTs being registered or not,
-                                     # this only works if all DDTs required in one metadata
-                                     # are defined in that file and/or parsed beforehand.
-                                     #check_fn_in=check_fortran_type),
-                                     # *DH
-                                     ),
+                                     check_fn_in=check_fortran_type),
                     VariableProperty('kind', str,
                                      optional_in=True,
                                      default_fn_in=default_kind_val),

scripts/parse_tools/parse_checkers.py

@@ -354,13 +354,15 @@ def check_fortran_type(typestr, error=False):
         match = registered_fortran_ddt_name(typestr)
         dt = " derived"
     # End if
-    if match is None:
-        if error:
-            raise CCPPError("'{}' is not a valid{} Fortran type".format(typestr, dt))
-        else:
-            typestr = None
-        # End if
-    # End if
+    # DH* 20190913 - skip checking if a DDT is registered at this time
+    #if match is None:
+    #    if error:
+    #        raise CCPPError("'{}' is not a valid{} Fortran type".format(typestr, dt))
+    #    else:
+    #        typestr = None
+    #    # End if
+    ## End if
+    # *DH 20190913
     return typestr
 
 ########################################################################