Add diag_mode to default diagnostics

MARBL_tools/MARBL_diagnostics_file_class.py

@@ -53,10 +53,11 @@ def __init__(self, default_diagnostics_file, MARBL_settings, unit_system):
                 diags_to_delete.append(diag_name)
                 continue
 
-            #    iii. frequency and operator should always be lists
+            #    iii. frequency, operator, and diag_mode should always be lists
             if not isinstance(self.diagnostics_dict[diag_name]['frequency'], list):
                 self.diagnostics_dict[diag_name]['frequency'] = [self.diagnostics_dict[diag_name]['frequency']]
                 self.diagnostics_dict[diag_name]['operator'] = [self.diagnostics_dict[diag_name]['operator']]
+                self.diagnostics_dict[diag_name]['diag_mode'] = [self.diagnostics_dict[diag_name]['diag_mode']]
 
             #    iv. update units
             fix_units = {}

MARBL_tools/MARBL_generate_diagnostics_file.py

@@ -45,13 +45,25 @@
   -a, --append          Append to existing diagnostics file (default: False)
 """
 
+if __name__ == "__main__":
+    # We need marbl_root in python path so we can import MARBL_tools from generate_settings_file()
+    import argparse
+    import os
+    import sys
+    marbl_root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+    sys.path.append(marbl_root)
+    from MARBL_tools import MARBL_settings_class
+    from MARBL_tools import MARBL_diagnostics_class
+
+import logging
+from MARBL_tools.MARBL_utils import valid_diag_modes
+
 #######################################
 
-def generate_diagnostics_file(MARBL_diagnostics, diagnostics_file_out, append=False):
+def generate_diagnostics_file(MARBL_diagnostics, diagnostics_file_out, diag_mode="full", append=False):
     """ Produce a list of MARBL diagnostic frequencies and operators from a JSON parameter file
     """
 
-    import logging
     logger = logging.getLogger(__name__)
 
     if not append:
@@ -84,12 +96,18 @@ def generate_diagnostics_file(MARBL_diagnostics, diagnostics_file_out, append=Fa
     # is also a dictionary containing frequency and operator information. Note that
     # string values of frequency and operator are converted to lists of len 1 when the
     # JSON file that generates this list is processed
+    diag_mode_opts = valid_diag_modes()
+    diag_mode_in = diag_mode_opts.index(diag_mode)
     for diag_name in sorted(MARBL_diagnostics.diagnostics_dict.keys()):
         frequencies = MARBL_diagnostics.diagnostics_dict[diag_name]['frequency']
         operators = MARBL_diagnostics.diagnostics_dict[diag_name]['operator']
+        diag_modes = MARBL_diagnostics.diagnostics_dict[diag_name]['diag_mode']
         freq_op = []
-        for freq, op in zip(frequencies, operators):
-            freq_op.append(freq + '_' + op)
+        for freq, op, dm in zip(frequencies, operators, diag_modes):
+            if diag_mode_in >= diag_mode_opts.index(dm):
+                freq_op.append(freq + '_' + op)
+            elif not freq_op: # Only append "never_{op}" if freq_op is empty list
+                freq_op.append('never_' + op)
         fout.write("%s : %s\n" % (diag_name, ", ".join(freq_op)))
     fout.close()
 
@@ -99,8 +117,6 @@ def _parse_args(marbl_root):
     """ Parse command line arguments
     """
 
-    import argparse
-
     parser = argparse.ArgumentParser(description="Generate a MARBL settings file from a JSON file",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
@@ -116,7 +132,7 @@ def _parse_args(marbl_root):
 
     # Is the GCM providing initial bury coefficients via saved state?
     parser.add_argument('-v', '--saved_state_vars_source', action='store', dest='saved_state_vars_source',
-                        default='settings_file', choices = set(('settings_file', 'GCM')),
+                        default='settings_file', choices=['settings_file', 'GCM'],
                         help="Source of initial value for saved state vars that can come from GCM or settings file")
 
     # Command line argument to specify resolution (default is None)
@@ -135,6 +151,11 @@ def _parse_args(marbl_root):
     parser.add_argument('-u', '--unit_system', action='store', dest='unit_system', default='cgs',
                         choices=['cgs', 'mks'], help='Unit system for parameter values')
 
+    # Diagnostic mode (level of output to include)
+    parser.add_argument('-m', '--diag-mode', action='store', dest='diag_mode', default='full',
+                        choices=valid_diag_modes(),
+                        help='Level of output to include')
+
     # Append to existing diagnostics file?
     parser.add_argument('-a', '--append', action='store_true', dest='append',
                         help='Append to existing diagnostics file')
@@ -144,26 +165,22 @@ def _parse_args(marbl_root):
 #######################################
 
 if __name__ == "__main__":
-    # We need marbl_root in python path so we can import MARBL_tools from generate_settings_file()
-    import sys, os
     marbl_root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
     sys.path.append(marbl_root)
 
     # Parse command-line arguments (marbl_root is used to set default for JSON file location)
     args = _parse_args(marbl_root)
 
     # Set up logging
-    import logging
     logging.basicConfig(format='%(levelname)s (%(funcName)s): %(message)s', level=logging.DEBUG)
 
-    from MARBL_tools import MARBL_settings_class
-    from MARBL_tools import MARBL_diagnostics_class
     DefaultSettings = MARBL_settings_class(args.default_settings_file,
                                            args.saved_state_vars_source,
                                            grid=args.grid,
                                            input_file=args.settings_file_in,
                                            unit_system=args.unit_system)
-    MARBL_diagnostics = MARBL_diagnostics_class(args.default_diagnostics_file, DefaultSettings, args.unit_system)
+    MARBL_diagnostics = MARBL_diagnostics_class(args.default_diagnostics_file, DefaultSettings,
+                                                args.unit_system)
 
     # Write the diagnostic file
-    generate_diagnostics_file(MARBL_diagnostics, args.diagnostics_file_out, args.append)
+    generate_diagnostics_file(MARBL_diagnostics, args.diagnostics_file_out, args.diag_mode, args.append)

MARBL_tools/MARBL_utils.py

@@ -8,6 +8,16 @@
 #                            PUBLIC MODULE METHODS                             #
 ################################################################################
 
+def valid_diag_modes():
+    """ Return list of the valid values for diag_mode.
+        Order of list => selecting specific diag mode includes all
+        diagnostics with lower index as well (e.g. "diag_mode = full"
+        will also provide diagnostics defined with minimal diag mode)
+    """
+    return ['none', 'minimal', 'full']
+
+################################################################################
+
 def settings_dictionary_is_consistent(SettingsDict):
     """ Make sure dictionary generated from JSON settings file conforms to MARBL
         parameter file standards:
@@ -82,13 +92,15 @@ def diagnostics_dictionary_is_consistent(DiagsDict):
            iii. vertical_grid (2D vars should explicitly list "none")
            iv.  frequency
            v.   operator
+           vi.  diag_mode
         3. Diagnostic variable dictionaries may contain 'dependencies' key as well,
            but it must be a dictionary itself.
-        4. Consistency between frequency and operator
-           i.   frequency and operator are both lists, or neither are
-           ii.  If they are both lists, must be same size
+        4. Consistency among frequency, operator, and diag_mode
+           i.   frequency, operator, and diag_mode are all lists, or none are
+           ii.  If they are all lists, must be same length
         5. Allowable frequencies are never, low, medium, and high
         6. Allowable operators are instantaneous, average, minimum, and maximum
+        7. Allowable diag_modes are defined in valid_diag_modes()
     """
 
     import logging
@@ -109,8 +121,8 @@ def diagnostics_dictionary_is_consistent(DiagsDict):
             continue
 
         # 2. diag_dict must have the following keys:
-        valid_keys = ["longname", "units", "vertical_grid", "frequency", "operator"]
-        for key_check in valid_keys:
+        required_keys = ["longname", "units", "vertical_grid", "frequency", "operator", "diag_mode"]
+        for key_check in required_keys:
             if key_check not in diag_dict.keys():
                 message = "Diagnostic %s is not well-defined in YAML" % diag_name
                 message = message + "\n     * Expecting %s as a key" % key_check
@@ -124,48 +136,67 @@ def diagnostics_dictionary_is_consistent(DiagsDict):
                 logger.error(message)
                 invalid_file = True
 
-        # 4. Consistency between frequency and operator
+        # 4. Consistency among frequency, operator, diag_mode
         err_prefix = "Inconsistency in DiagsDict['%s']:" % diag_name
         #    i.   frequency and operator are both lists, or neither are
-        if isinstance(diag_dict['frequency'], list) != isinstance(diag_dict['operator'], list):
-            logger.error("%s either both frequency and operator must be lists or neither can be" % err_prefix)
+        if (isinstance(diag_dict['frequency'], list) != isinstance(diag_dict['operator'], list)) or \
+           (isinstance(diag_dict['frequency'], list) != isinstance(diag_dict['diag_mode'], list)):
+            err_message = "either all of frequency, operator, and diag_mode must be lists or neither can be"
+            logger.error(f"{err_prefix} {err_message}")
             invalid_file = True
             continue
 
         #    ii.  If they are both lists, must be same size
         if isinstance(diag_dict['frequency'], list):
             freq_len = len(diag_dict['frequency'])
             op_len = len(diag_dict['operator'])
-            if freq_len != op_len:
-                logger.error("%s frequency is length %d but operator is length %d" %
-                             (err_prefix, diag_name, freq_len, op_len))
+            dm_len = len(diag_dict['diag_mode'])
+            if (freq_len != op_len) or (freq_len != op_len):
+                err_message = f"frequency, operator, diag_mode lengths are {freq_len}, {op_len}, {diag_mode}"
+                logger.error(f"{err_prefix} {err_message}")
                 invalid_file = True
                 continue
 
         # 5. Allowable frequencies are never, low, medium, and high
         # 6. Allowable operators are instantaneous, average, minimum, and maximum
+        # 7. Allowable diag_modes are defined in valid_diag_modes()
+        #    * "none" should not appear in the dictionary
         ok_freqs = ['never', 'low', 'medium', 'high']
         ok_ops = ['instantaneous', 'average', 'minimum', 'maximum']
-        invalid_freq_op = False
+        ok_dms = valid_diag_modes()[1:] # do not include 'none' in ok_dms
+        invalid_freq_op_dm = False
         if not isinstance(diag_dict['frequency'], dict):
             if isinstance(diag_dict['frequency'], list):
-                for freq, op in zip(diag_dict['frequency'], diag_dict['operator']):
+                for freq, op, dm in zip(diag_dict['frequency'], diag_dict['operator'], diag_dict['diag_mode']):
                     if freq not in ok_freqs:
-                        logger.error("%s '%s' is not a valid frequency" % (err_prefix, freq))
-                        invalid_freq_op = True
+                        err_message = f"'{freq}' is not a valid frequency"
+                        logger.error(f'{err_prefix} {err_message}')
+                        invalid_freq_op_dm = True
                     if op not in ok_ops:
-                        logger.error("%s '%s' is not a valid operator" % (err_prefix, op))
-                        invalid_freq_op = True
+                        err_message = f"'{op}' is not a valid operator"
+                        logger.error(f'{err_prefix} {err_message}')
+                        invalid_freq_op_dm = True
+                    if dm not in ok_dms:
+                        err_message = f"'{dm}' is not a valid diag_mode"
+                        logger.error(f'{err_prefix} {err_message}')
+                        invalid_freq_op_dm = True
             else:
                 freq = diag_dict['frequency']
                 op = diag_dict['operator']
+                dm = diag_dict['diag_mode']
                 if freq not in ok_freqs:
-                    logger.error("%s '%s' is not a valid frequency" % (err_prefix, freq))
-                    invalid_freq_op = True
+                    err_message = f"'{freq}' is not a valid frequency"
+                    logger.error(f'{err_prefix} {err_message}')
+                    invalid_freq_op_dm = True
                 if op not in ok_ops:
-                    logger.error("%s '%s' is not a valid operator" % (err_prefix, op))
-                    invalid_freq_op = True
-        if invalid_freq_op:
+                    err_message = f"'{op}' is not a valid operator"
+                    logger.error(f'{err_prefix} {err_message}')
+                    invalid_freq_op_dm = True
+                if dm not in ok_dms:
+                    err_message = f"'{dm}' is not a valid diag_mode"
+                    logger.error(f'{err_prefix} {err_message}')
+                    invalid_freq_op_dm = True
+        if invalid_freq_op_dm:
             invalid_file = True
 
     return (not invalid_file)

MARBL_tools/run_test_suite.sh

@@ -85,6 +85,12 @@ done
 
 # Test MARBL_generate_diagnostics_file.py
 cd ${MARBL_ROOT}/MARBL_tools
+(set -x ; ./MARBL_generate_diagnostics_file.py --diag-mode none -o marbl.diags.none)
+STATUS=$(check_return $?)
+print_status "MARBL_generate_diagnostics_file.py --diag-mode none -o marbl.diags.none" >> ${RESULTS_CACHE}
+(set -x ; ./MARBL_generate_diagnostics_file.py --diag-mode minimal -o marbl.diags.minimal)
+STATUS=$(check_return $?)
+print_status "MARBL_generate_diagnostics_file.py --diag-mode minimal marbl.diags.minimal" >> ${RESULTS_CACHE}
 (set -x ; ./MARBL_generate_diagnostics_file.py)
 STATUS=$(check_return $?)
 print_status "MARBL_generate_diagnostics_file.py" >> ${RESULTS_CACHE}