Feature #392 tcdiag_line_type (#2315)

data/config/TCPairsConfig_default

@@ -137,6 +137,47 @@ watch_warn = {
    time_offset = -14400;
 }
 
+//
+// Diagnostics to be extracted
+//
+diag_name = [];
+
+//
+// Unit conversions to be applied based on diagnostic names and units
+//
+diag_convert_map = [
+   { source = "TCDIAG";
+     key = [ "(10C)", "(10KT)", "(10M/S)" ];
+     convert(x) = x / 10; },
+
+   { source = "LSDIAG_RT";
+     key = [ "LAT",  "LON",  "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA",
+             "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS",
+             "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850",
+             "V500", "V300", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG",
+             "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08",
+             "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18",
+             "PW20", "PW21" ];
+     convert(x) = x / 10; },
+
+   { source = "LSDIAG_RT";
+     key = [ "VVAV", "VMFX", "VVAC" ];
+     convert(x) = x / 100; },
+
+   { source = "LSDIAG_RT";
+     key = [ "TADV" ];
+     convert(x) = x / 1000000; },
+
+   { source = "LSDIAG_RT";
+     key = [ "Z850", "D200", "TGRD", "DIVC" ];
+     convert(x) = x / 10000000; },
+
+   { source = "LSDIAG_RT";
+     key = [ "PENC", "PENV" ];
+     convert(x) = x / 10 + 1000; }
+
+];
+
 //
 // Modify basin names to make them consistent across ATCF input files.
 //

data/config/TCStatConfig_default

@@ -93,7 +93,7 @@ line_type = [];
 
 //
 // Stratify by checking the watch/warning status for each track point
-// common to both the ADECK and BDECK tracks.  If the watch/warning status
+// common to both the ADECK and BDECK tracks. If the watch/warning status
 // of any of the track points appears in the list, retain the entire track.
 //
 track_watch_warn = [];
@@ -134,6 +134,20 @@ init_str_val  = [];
 init_str_exc_name = [];
 init_str_exc_val  = [];
 
+//
+// Stratify track points by applying thresholds to numeric
+// storm diagnostic values in the TCDIAG lines.
+//
+diag_thresh_name = [];
+diag_thresh_val  = [];
+
+//
+// Stratify entire tracks by applying thresholds to numeric
+// storm diagnostic values in the TCDIAG lines for lead time 0.
+//
+init_diag_thresh_name = [];
+init_diag_thresh_val  = [];
+
 //
 // Stratify by the ADECK and BDECK distances to land.
 //

data/table_files/met_header_columns_V11.0.txt

@@ -37,4 +37,5 @@ V11.0 : MODE : OBJ      : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VAL
 V11.0 : MODE : CTS      : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS
 
 V11.0 : TCST : TCMPR    : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE TOTAL INDEX LEVEL WATCH_WARN INITIALS ALAT ALON BLAT BLON TK_ERR X_ERR Y_ERR ALTK_ERR CRTK_ERR ADLAND BDLAND AMSLP BMSLP AMAX_WIND BMAX_WIND AAL_WIND_34 BAL_WIND_34 ANE_WIND_34 BNE_WIND_34 ASE_WIND_34 BSE_WIND_34 ASW_WIND_34 BSW_WIND_34 ANW_WIND_34 BNW_WIND_34 AAL_WIND_50 BAL_WIND_50 ANE_WIND_50 BNE_WIND_50 ASE_WIND_50 BSE_WIND_50 ASW_WIND_50 BSW_WIND_50 ANW_WIND_50 BNW_WIND_50 AAL_WIND_64 BAL_WIND_64 ANE_WIND_64 BNE_WIND_64 ASE_WIND_64 BSE_WIND_64 ASW_WIND_64 BSW_WIND_64 ANW_WIND_64 BNW_WIND_64 ARADP BRADP ARRP BRRP AMRD BMRD AGUSTS BGUSTS AEYE BEYE ADIR BDIR ASPEED BSPEED ADEPTH BDEPTH NUM_MEMBERS TRACK_SPREAD DIST_MEAN MSLP_SPREAD MAX_WIND_SPREAD
+V11.0 : TCST : TCDIAG   : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE TOTAL INDEX SOURCE (N_DIAG) DIAG_[0-9]* VALUE_[0-9]*
 V11.0 : TCST : PROBRIRW : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE ALAT ALON BLAT BLON INITIALS TK_ERR X_ERR Y_ERR ADLAND BDLAND RIRW_BEG RIRW_END RIRW_WINDOW AWIND_END BWIND_BEG BWIND_END BDELTA BDELTA_MAX BLEVEL_BEG BLEVEL_END (N_THRESH) THRESH_[0-9]* PROB_[0-9]*

docs/Users_Guide/tc-pairs.rst

@@ -9,6 +9,16 @@ Introduction
 
 The TC-Pairs tool provides verification for tropical cyclone forecasts in ATCF file format. It matches an ATCF format tropical cyclone (TC) forecast with a second ATCF format reference TC dataset (most commonly the Best Track analysis). The TC-Pairs tool processes both track and intensity adeck data and probabilistic edeck data. The adeck matched pairs contain position errors, as well as wind, sea level pressure, and distance to land values for each TC dataset. The edeck matched pairs contain probabilistic forecast values and the verifying observation values. The pair generation can be subset based on user-defined filtering criteria. Practical aspects of the TC-Pairs tool are described in :numref:`TC-Pairs_Practical-information`. 
 
+Scientific and statistical aspects
+==================================
+
+.. _TC-Pairs_Diagnostics:
+
+Storm Diagnostics
+-----------------
+
+TODO: Add a paragraph about storm diagnostics, describing what they are, why they are important, and how they can be generated.
+
 .. _TC-Pairs_Practical-information:
 
 Practical information
@@ -24,9 +34,10 @@ The usage statement for tc_pairs is shown below:
 .. code-block:: none
 
   Usage: tc_pairs
-         -adeck source and/or -edeck source
-         -bdeck source
+         -adeck path and/or -edeck path
+         -bdeck path
          -config file
+         [-diag source path]
          [-out base]
          [-log file]
          [-v level]
@@ -36,24 +47,30 @@ tc_pairs has required arguments and can accept several optional arguments.
 Required arguments for tc_pairs
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-1. The **-adeck source** argument indicates the adeck TC-Pairs acceptable format data source containing tropical cyclone model forecast (output from tracker) data to be verified. Acceptable data formats are limited to the standard ATCF format and the one column modified ATCF file, generated by running the tracker in genesis mode. It specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. The **-adeck** or **-edeck** option must be used at least once.
+1. The **-adeck path** argument indicates the adeck TC-Pairs acceptable format data containing tropical cyclone model forecast (output from tracker) data to be verified. Acceptable data formats are limited to the standard ATCF format and the one column modified ATCF file, generated by running the tracker in genesis mode. It specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. The **-adeck** or **-edeck** option must be used at least once.
 
-2. The **-edeck source** argument indicates the edeck ATCF format data source containing probabilistic track data to be verified. It specifies the name of an ATCF format file or top-level directory containing ATCF format files ending in ".dat" to be processed. The **-adeck** or **-edeck** option must be used at least once.
+2. The **-edeck path** argument indicates the edeck ATCF format data containing probabilistic track data to be verified. It specifies the name of an ATCF format file or top-level directory containing ATCF format files ending in ".dat" to be processed. The **-adeck** or **-edeck** option must be used at least once.
 
-3. The **-bdeck source** argument indicates the TC-Pairs acceptable format data source containing the tropical cyclone reference dataset to be used for verifying the adeck source. This source is typically the NHC Best Track Analysis, but could be any TC-Pairs acceptable formatted reference. The acceptable data formats for bdecks are the same as those for adecks. This argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. 
+3. The **-bdeck path** argument indicates the TC-Pairs acceptable format data containing the tropical cyclone reference dataset to be used for verifying the adeck data. This data is typically the NHC Best Track Analysis, but could be any TC-Pairs acceptable formatted reference. The acceptable data formats for bdecks are the same as those for adecks. This argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed.
 
 4. The **-config file** argument indicates the name of the configuration file to be used. The contents of the configuration file are discussed below.
 
 Optional arguments for tc_pairs
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-5. The -**out base** argument indicates the path of the output file base. This argument overrides the default output file base (**./out_tcmpr**).
+5. The **-diag source path** argument indicates the TC-Pairs acceptable format data containing the tropical cyclone diagnostics dataset corresponding to the adeck tracks. The **source** can be set to TCDIAG, LSDIAG_RT, or LSDIAG_DEV to indicate the input diagnostics data source. The **path** argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed.
+
+6. The -**out base** argument indicates the path of the output file base. This argument overrides the default output file base (**./out_tcmpr**).
+
+7. The **-log file** option directs output and errors to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file.
 
-6. The **-log file** option directs output and errors to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file. 
+8. The **-v level** option indicates the desired level of verbosity. The contents of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging.
 
-7. The **-v level** option indicates the desired level of verbosity. The contents of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging.
+This tool currently only supports the rapid intensification (**RI**) edeck probability type but support for additional edeck probability types will be added in future releases.
 
-This tool currently only supports the rapid intensification (**RI**) edeck probability type but support for additional edeck probability types will be added in future releases. At least one **-adeck** or **-edeck** option must be specified. The **-adeck, -edeck**, and **-bdeck** options may optionally be followed with **suffix=string** to append that string to all model names found within that data source. This option may be useful when processing track data from two different sources which reuse the same model names.
+At least one **-adeck** or **-edeck** option must be specified. The **-adeck, -edeck**, and **-bdeck** options may optionally be followed with **suffix=string** to append that string to all model names found within that data source. This option may be useful when processing track data from two different sources which reuse the same model names.
+
+The **-diag** option may optionally be followed with **model=string** to override the model name of the tracks to which those diagnostics correspond. The **string** specifies a comma-separated list of one or more ATCF ID's to which these diagnostics should be paired (e.g. **model=OFCL,SHIP**).
 
 An example of the tc_pairs calling sequence is shown below:
 
@@ -67,6 +84,8 @@ The TC-Pairs tool implements the following logic:
 
 • Parse the adeck, edeck, and bdeck data files and store them as track objects.
 
+• Parse diagnostics data files and add the requested diagnostics to the existing adeck track objects.
+
 • Apply configuration file settings to filter the adeck, edeck, and bdeck track data down to a subset of interest.
 
 • Apply configuration file settings to derive additional adeck track data, such as interpolated tracks, consensus tracks, time-lagged tracks, and statistical track and intensity models.
@@ -235,6 +254,59 @@ ____________________
 
 The **watch_warn** field specifies the file name and time applied offset to the **watch_warn** flag. The **file_name** string specifies the path of the watch/warning file to be used to determine when a watch or warning is in effect during the forecast initialization and verification times. The default file is named **wwpts_us.txt**, which is found in the installed *share/met/tc_data/* directory within the MET build. The **time_offset** string is the time window (in seconds) assigned to the watch/warning. Due to the non-uniform time watches and warnings are issued, a time window is assigned for which watch/warnings are included in the verification for each valid time. The default watch/warn file is static, and therefore may not include warned storms beyond the current MET code release date; therefore users may wish to create a post in the `METplus GitHub Discussions Forum <https://github.com/dtcenter/METplus/discussions>`_ in order to obtain the most recent watch/warning file if the static file does not contain storms of interest.
 
+____________________
+
+.. code-block:: none
+
+ diag_name = [];
+
+The **diag_name** entry specifies a comma-separated list of strings for the tropical cyclone diagnostics of interest. This applies when the **-tcdiag** and/or **-lsdiag** command line options have been used to provide storm diagnostics data. If a non-zero list of diagnostic names is specified, only those diagnostics appearing in the list are written to the TCDIAG output line type. If defined as an empty list (default), all diagnostics found in the input are written to the TCDIAG output lines.
+
+A TCMPR line is written to the output for each track point. If diagnostics data is also defined for that track point, a TCDIAG line is written immediately after the corresponding TCMPR line. The contents of that TCDIAG line is determined by diagnostic names requested in the **diag_name** entry.
+
+____________________
+
+.. code-block:: none
+
+  diag_convert_map = [
+     { source = "TCDIAG";
+       key = [ "(10C)", "(10KT)", "(10M/S)" ];
+       convert(x) = x / 10; },
+
+     { source = "LSDIAG_RT";
+       key = [ "LAT",  "LON",  "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA",
+               "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS",
+               "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850",
+               "V500", "V300", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG",
+               "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08",
+               "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18",
+               "PW20", "PW21" ];
+       convert(x) = x / 10; },
+
+     { source = "LSDIAG_RT";
+       key = [ "VVAV", "VMFX", "VVAC" ];
+       convert(x) = x / 100; },
+
+     { source = "LSDIAG_RT";
+       key = [ "TADV" ];
+       convert(x) = x / 1000000; },
+
+     { source = "LSDIAG_RT";
+       key = [ "Z850", "D200", "TGRD", "DIVC" ];
+       convert(x) = x / 10000000; },
+
+     { source = "LSDIAG_RT";
+       key = [ "PENC", "PENV" ];
+       convert(x) = x / 10 + 1000; }
+
+  ];
+
+The **diag_convert_map** entries define conversion functions to be applied to diagnostics data read with the **-diag** command line option. Each array element is a dictionary consisting of a **source**, **key**, and **convert(x)** entry.
+
+The **source** is one of the supported diagnostics data sources. The **key** is an array of strings. The strings can specify diagnostic names or units, although units are only checked for **TCDIAG** sources. If both the name and units are specified, the conversion function for the name takes precedence. **convert(x)** is a function of one variable which defines how the diagnostic data should be converted. The defined function is applied to any diagnostic value whose name or units appears in the **key**.
+
+____________________
+
 .. code-block:: none
 
   basin_map = [
@@ -487,7 +559,41 @@ TC-Pairs produces output in TCST format. The default output file name can be ove
   * - 84
     - MAX_WIND_SPREAD
     - consensus variable: the standard deviation of the member's maximum wind speed values 
-
+
+.. _TCDIAG Line Type:
+
+.. list-table:: Format information for TCDIAG (Tropical Cyclone Diagnostics) output line type.
+  :widths: auto
+  :header-rows: 2
+
+  * -
+    -
+    - TCDIAG OUTPUT FORMAT
+  * - Column Number
+    - Header Column Name
+    - Description
+  * - 13
+    - TCDIAG
+    - Tropical Cyclone Diagnostics line type
+  * - 14
+    - TOTAL
+    - Total number of pairs in track
+  * - 15
+    - INDEX
+    - Index of the current track pair
+  * - 16
+    - SOURCE
+    - Diagnostics data source
+  * - 17
+    - N_DIAG
+    - Number of storm diagnostic name and value columns to follow
+  * - 18
+    - DIAG_i
+    - Name of the of the ith storm diagnostic (repeated)
+  * - 19
+    - VALUE_i
+    - Value of the ith storm diagnostic (repeated)
+
 .. _PROBRIRW Line Type:
 
 .. list-table:: Format information for PROBRIRW (Probability of Rapid Intensification/Weakening) output line type.