From b9665a4763563bace7e0b94cb9a28c9106ec2fec Mon Sep 17 00:00:00 2001 From: bikegeek Date: Tue, 21 Sep 2021 17:25:25 -0600 Subject: [PATCH 01/23] Part of Github issue #1000 METplus: added font size for legend label --- .../tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf b/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf index 22020eb49a..f86d90b6d2 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf @@ -102,6 +102,7 @@ CYCLONE_PLOTTER_INIT_HR ={init?fmt=%H} CYCLONE_PLOTTER_MODEL = GFSO CYCLONE_PLOTTER_PLOT_TITLE = Model Forecast Storm Tracks + ## # Indicate the size of symbol (point size) CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE = 2 @@ -109,7 +110,10 @@ CYCLONE_PLOTTER_CROSS_MARKER_SIZE = 3 ## # Indicate text size of annotation label -CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE=3 +CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE = 3 + +# Indicate the text size for the legend labels +CYCLONE_PLOTTER_LEGEND_FONT_SIZE = 3 ## # Resolution of saved plot in dpi (dots per inch) From 3b624f3a85a6b7c230738f77a4c542d9434cd70b Mon Sep 17 00:00:00 2001 From: bikegeek Date: Tue, 21 Sep 2021 17:39:58 -0600 Subject: [PATCH 02/23] Github Issue #1000: major refactor to accommodate "sanitizing" of longitudes that could cross the International Date Line. --- metplus/wrappers/cyclone_plotter_wrapper.py | 749 +++++++++----------- 1 file changed, 342 insertions(+), 407 deletions(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index d9062ada1f..08819e2446 100755 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -9,18 +9,21 @@ import datetime import re import sys -import collections +import pandas as pd +from collections import namedtuple # handle if module can't be loaded to run wrapper WRAPPER_CANNOT_RUN = False EXCEPTION_ERR = '' try: + import pandas as pd import matplotlib.pyplot as plt import matplotlib.ticker as mticker import cartopy.crs as ccrs import cartopy.feature as cfeature import cartopy from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER + ##If the script is run on a limited-internet access machine, the CARTOPY_DIR environment setting ##will need to be set in the user-specific system configuration file. Review the Installation section ##of the User's Guide for more details. @@ -38,7 +41,6 @@ from . import CommandBuilder - class CyclonePlotterWrapper(CommandBuilder): """! Generate plots of extra tropical storm forecast tracks. Reads input from ATCF files generated from MET TC-Pairs @@ -63,6 +65,7 @@ def __init__(self, config, instance=None, config_overrides={}): self.init_hr = self.config.getraw('config', 'CYCLONE_PLOTTER_INIT_HR') init_time_fmt = self.config.getstr('config', 'INIT_TIME_FMT', '') + if init_time_fmt: clock_time = datetime.datetime.strptime( self.config.getstr('config', @@ -78,6 +81,14 @@ def __init__(self, config, instance=None, config_overrides={}): logger=self.logger) self.init_date = do_string_sub(self.init_date, init=init_beg_dt) self.init_hr = do_string_sub(self.init_hr, init=init_beg_dt) + init_end = self.config.getraw('config', 'INIT_END') + if init_end: + init_end_dt = util.get_time_obj(init_end, + init_time_fmt, + clock_time, + logger=self.logger) + self.init_end_date = do_string_sub(self.init_date, init=init_end_dt) + self.init_end_hr = do_string_sub(self.init_hr, init=init_end_dt) self.model = self.config.getstr('config', 'CYCLONE_PLOTTER_MODEL') self.title = self.config.getstr('config', @@ -90,10 +101,10 @@ def __init__(self, config, instance=None, config_overrides={}): self.unique_storm_id = set() # Data structure to separate data based on storm id. self.storm_id_dict = {} + # Data/info which we want to retrieve from the track files. - self.columns_of_interest = ['AMODEL', 'STORM_ID', 'BASIN', 'INIT', - 'LEAD', 'VALID', 'ALAT', 'ALON', 'BLAT', - 'BLON', 'AMSLP', 'BMSLP'] + self.columns_of_interest = ['AMODEL', 'STORM_ID', 'INIT', + 'LEAD', 'VALID', 'ALAT', 'ALON'] self.circle_marker = ( self.config.getint('config', 'CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE') @@ -102,6 +113,12 @@ def __init__(self, config, instance=None, config_overrides={}): self.config.getint('config', 'CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE') ) + + self.legend_font_size = ( + self.config.getint('config', + 'CYCLONE_PLOTTER_LEGEND_FONT_SIZE') + ) + self.cross_marker = ( self.config.getint('config', 'CYCLONE_PLOTTER_CROSS_MARKER_SIZE') @@ -115,7 +132,6 @@ def __init__(self, config, instance=None, config_overrides={}): 'CYCLONE_PLOTTER_ADD_WATERMARK', True) - def run_all_times(self): """! Calls the defs needed to create the cyclone plots run_all_times() is required by CommandBuilder. @@ -125,250 +141,171 @@ def run_all_times(self): self.create_plot() def retrieve_data(self): - """! Retrieve data from track files and return the min and max lon. + """! Retrieve data from track files. Returns: None + assigns value to the useful class variables: unique_storm_ids, track_df, pts_by_track_dict """ self.logger.debug("Begin retrieving data...") all_tracks_list = [] # Store the data in the track list. if os.path.isdir(self.input_data): - self.logger.debug("Generate plot for all files in the directory" + + self.logger.debug("Get data from all files in the directory " + self.input_data) # Get the list of all files (full file path) in this directory - all_init_files = util.get_files(self.input_data, ".*.tcst", + all_input_files = util.get_files(self.input_data, ".*.tcst", self.logger) - for init_file in all_init_files: - # Ignore empty files - if os.stat(init_file).st_size == 0: - self.logger.info(f"Ignoring empty file {init_file}") - continue - - # logger.info("Consider all files under directory" + - # init_file + " with " + " init time (ymd): " + - # self.init_date + " and lead time (hh):" + self.lead_hr) - with open(init_file, 'r') as infile: - self.logger.debug("Parsing file {}".format(init_file)) - - # Extract information from the header, which is - # the first line. - header = infile.readline() - # print("header: {}".format(header)) - column_indices = self.get_columns_and_indices(header) - - # For the remaining lines of this file, - # retrieve information from each row: - # lon, lat, init time, lead hour, valid time, - # model name, mslp, and basin. - # NOTE: Some of these columns aren't used until we fully - # emulate Guang Ping's plots (ie collect all columns). - for line in infile: - track_dict = {} - col = line.split() - lat = col[column_indices['ALAT']] - lon = col[column_indices['ALON']] - init_time = col[column_indices['INIT']] - fcst_lead_hh = \ - str(col[column_indices['LEAD']]).zfill(3) - model_name = col[column_indices['AMODEL']] - valid_time = col[column_indices['VALID']] - storm_id = col[column_indices['STORM_ID']] - - # Not needed until support for regional plots - # is implemented. - # mslp = col[column_indices['AMSLP']] - # basin = col[column_indices['BASIN']] - - # Check for NA values in lon and lat, skip to - # next line in file if 'NA' is encountered. - if lon == 'NA' or lat == 'NA': - continue - else: - # convert longitudes that are in the 0 to 360 scale - # to the -180 to 180 scale - track_dict['lon'] = self.rescale_lon(float(lon)) - track_dict['lat'] = float(lat) - - # If the lead hour is 'NA', skip to next line. - # The track data was very likely generated with - # TC-Stat set to match-pairs set to True. - lead_hr = self.extract_lead_hr(fcst_lead_hh) - if lead_hr == 'NA': - continue - - # Check that the init date, init hour - # and model name are what the user requested. - init_ymd, init_hh = \ - self.extract_date_and_time_from_init(init_time) - - if init_ymd == self.init_date and \ - init_hh == self.init_hr: - if model_name == self.model: - # Check for the requested model, - # if the model matches the requested - # model name, then we have all the - # necessary information to continue. - # Store all data in dictionary - track_dict['fcst_lead_hh'] = fcst_lead_hh - track_dict['init_time'] = init_time - track_dict['model_name'] = model_name - track_dict['valid_time'] = valid_time - track_dict['storm_id'] = storm_id - - # Identify the 'first' point of the - # storm track. If the storm id is novel, then - # retrieve the date and hh from the valid time - if storm_id in self.unique_storm_id: - track_dict['first_point'] = False - track_dict['valid_dd'] = '' - track_dict['valid_hh'] = '' - else: - self.unique_storm_id.add(storm_id) - # Since this is the first storm_id with - # a valid value for lat and lon (ie not - # 'NA'), this is the first track point - # in the storm track and will be - # labelled with the corresponding - # date/hh z on the plot. - valid_match = \ - re.match(r'[0-9]{6}([0-9]{2})_' + - '([0-9]{2})[0-9]{4}', - track_dict['valid_time']) - if valid_match: - valid_dd = valid_match.group(1) - valid_hh = valid_match.group(2) - else: - # Shouldn't get here if this is - # the first point of the track. - valid_dd = '' - valid_hh = '' - track_dict['first_point'] = True - track_dict['valid_dd'] = valid_dd - track_dict['valid_hh'] = valid_hh - - # Identify points based on valid time (hh). - # Useful for plotting later on. - valid_match = \ - re.match(r'[0-9]{8}_([0-9]{2})[0-9]{4}', - track_dict['valid_time']) - if valid_match: - # Since we are only interested in 00, - # 06, 12, and 18 hr times... - valid_hh = valid_match.group(1) - - if valid_hh == '00' or valid_hh == '12': - track_dict['lead_group'] = '0' - elif valid_hh == '06' or valid_hh == '18': - track_dict['lead_group'] = '6' - else: - # To gracefully handle any hours other - # than 0, 6, 12, or 18 - track_dict['lead_group'] = '' - - all_tracks_list.append(track_dict) - - # For future work, support for MSLP when - # generating regional plots- - # implementation goes here... - # Finishing up, do any cleaning up, - # logging, etc. - self.logger.info("All criteria met, " + - "saving track data init " + - track_dict['init_time'] + - " lead " + - track_dict['fcst_lead_hh'] + - " lon " + - str(track_dict['lon']) + - " lat " + - str(track_dict['lat'])) - - else: - # Not the requested model, move to next - # row of data - continue - - else: - # Not the requested init ymd move to next - # row of data - continue - - # Now separate the data based on storm id. - for cur_unique in self.unique_storm_id: - cur_storm_list = [] - for cur_line in all_tracks_list: - if cur_line['storm_id'] == cur_unique: - cur_storm_list.append(cur_line) + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ##!!!!USING PANDAS DATAFRAME!!!! GITHUB ISSUE METPLUS #1000 + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + # read each file into pandas then concatenate them together + df_list = [pd.read_csv(file, delim_whitespace=True) for file in all_input_files] + combined = pd.concat(df_list, ignore_index=True) + + # if there are any NaN values in the ALAT, ALON, STORM_ID, LEAD, INIT, AMODEL, or VALID column, + # drop that row of data (axis=0). We need all these columns to contain valid data in order + # to create a meaningful plot. + combined_df = combined.copy(deep=True) + combined_df = combined.dropna(axis=0, how='any', + subset=['ALAT', 'ALON', 'STORM_ID', 'LEAD', 'INIT', 'AMODEL', 'VALID']) + + # Retrieve and create the columns of interest + self.logger.debug(f"Number of rows of data: {combined_df.shape[0]}") + combined_subset = combined_df[self.columns_of_interest] + df = combined_subset.copy(deep=True) + df.allows_duplicate_labels = False + df['INIT'] = df['INIT'].astype(str) + df['INIT_YMD'] = (df['INIT'].str[:8]).astype(int) + df['INIT_HOUR'] = (df['INIT'].str[9:11]).astype(int) + df['LEAD'] = df['LEAD']/10000 + df['LEAD'] = df['LEAD'].astype(int) + df['VALID_DD'] = (df['VALID'].str[6:8]).astype(int) + df['VALID_HOUR'] = (df['VALID'].str[9:11]).astype(int) + df['VALID'] = df['VALID'].astype(int) + + # clean up dataframes that are no longer needed + del combined + del combined_df + + # Subset the dataframe to include only the data relevant to the user's criteria as + # specified in the configuration file. + init_date = int(self.init_date) + init_hh = int(self.init_hr) + model_name = self.model + + mask = df[(df['AMODEL'] == model_name) & (df['INIT_YMD'] >= init_date) & + (df['INIT_HOUR'] >= init_hh)] + user_criteria_df = mask + + # Aggregate the ALON values based on unique storm id in order to sanitize the longitude values + # that cross the International Date Line. + unique_storm_ids_set = set(user_criteria_df['STORM_ID']) + self.unique_storm_id = unique_storm_ids_set + self.unique_storm_ids = list(unique_storm_ids_set) + nunique = len(self.unique_storm_ids) + self.logger.debug(f" {nunique} unique storm ids identified") + + # Use named tuples to store the relevant storm track information (their index value in the dataframe, + # track id, and ALON values and later on the SLON (sanitized ALON values). + TrackPt = namedtuple("TrackPt", "indices track alons ") + + # named tuple holding "sanitized" longitudes + SanTrackPt = namedtuple("SanTrackPt", "indices track alons slons") + + # Keep track of the unique storm tracks by their storm_id + storm_track_dict = {} + + for cur_unique in self.unique_storm_ids: + idx_list = user_criteria_df.index[user_criteria_df['STORM_ID'] == cur_unique].tolist() + alons = [] + indices = [] + + for idx in idx_list: + alons.append(user_criteria_df.iloc[idx]['ALON']) + indices.append(idx) + + # create the track_pt tuple and add it to the storm track dictionary + track_pt = TrackPt(indices, cur_unique, alons) + storm_track_dict[cur_unique] = track_pt + + # create a new dataframe to contain the sanitized lons (i.e. the original ALONs that have + # been cleaned up when crossing the International Date Line) + sanitized_df = user_criteria_df.copy(deep=True) + + # Now we have a dictionary that aggregates the data based on storm tracks (via storm id) + # and will contain the "sanitized" lons + sanitized_storm_tracks = {} + for key in storm_track_dict: + # sanitize the longitudes, create a new SanTrackPt named tuple and add that to a new dictionary + # that keeps track of the sanitized data based on the storm id + sanitized_lons = self.sanitize_lonlist(storm_track_dict[key].alons) + sanitized_track_pt = SanTrackPt(storm_track_dict[key].indices, storm_track_dict[key].track, + storm_track_dict[key].alons, sanitized_lons) + sanitized_storm_tracks[key] = sanitized_track_pt + + # fill in the sanitized dataframe sanitized_df + for key in sanitized_storm_tracks: + # now use the indices of the storm tracks to correctly assign the sanitized + # lons to the appropriate row in the dataframe to maintain the row ordering of + # the original dataframe + idx_list = sanitized_storm_tracks[key].indices + + for i, idx in enumerate(idx_list): + sanitized_df.loc[idx,'SLON'] = sanitized_storm_tracks[key].slons[i] + + # Set some useful values used for plotting. + # Set the IS_FIRST value to True if this is the first point in the storm track, False + # otherwise + if i == 0: + sanitized_df.loc[idx, 'IS_FIRST'] = True else: - # Continue to next line in all_tracks_list - continue - - # Create the storm_id_dict, which is the data - # structure used to separate the storm data based on - # storm id. - self.storm_id_dict[cur_unique] = cur_storm_list - - else: - self.log_error("{} should be a directory".format(self.input_data)) - sys.exit(1) - - def get_columns_and_indices(self, header): - """ Parse the header for the columns of interest and store the - information in a dictionary where the key is the column name - and value is the index/column number. - Returns: - column_dict: A dictionary containing the column name - and its index - """ - - all_columns = header.split() - column_dict = {} - - # Retrieve the index number of the column of interest in the header. - for col in self.columns_of_interest: - index = all_columns.index(col) - column_dict[col] = index - return column_dict - - @staticmethod - def extract_date_and_time_from_init(init_time_str): - """ Extract and return the YYYYMMDD portion and the - hh portion from the init time taken from the .tcst file. - """ - match_ymd = re.match(r'([0-9]{8}).*', init_time_str) - match_hh = re.match(r'[0-9]{8}_([0-9]{2,3})[0-9]{4}', init_time_str) - # pylint:disable=no-else-return - # Explicitly return None if no match is found - if match_ymd and match_hh: - return match_ymd.group(1), match_hh.group(1) - else: - return None - - @staticmethod - def extract_lead_hr(lead_str): - """ Extract and return the lead hours from the hhmmss lead string. - """ - match = re.match(r'([0-9]{2,3})[0-9]{4}', str(lead_str)) - if match: - return match.group(1) + sanitized_df.loc[idx, 'IS_FIRST'] = False + + # Set the lead group to the character '0' if the valid hour is 0 or 12, or to the + # charcter '6' if the valid hour is 6 or 18. Set the marker to correspond to the + # valid hour: 'o' (open circle) for 0 or 12 valid hour, or '+' (small plus/cross) + # for 6 or 18. + if sanitized_df.loc[idx, 'VALID_HOUR'] == 0 or sanitized_df.loc[idx, 'VALID_HOUR'] == 12: + sanitized_df.loc[idx, 'LEAD_GROUP'] ='0' + sanitized_df.loc[idx, 'MARKER'] = 'o' + elif sanitized_df.loc[idx, 'VALID_HOUR'] == 6 or sanitized_df.loc[idx, 'VALID_HOUR'] == 18: + sanitized_df.loc[idx, 'LEAD_GROUP'] = '6' + sanitized_df.loc[idx, 'MARKER'] = '+' + + # store the sanitized dataframe, and remove unecessary dataframes + self.sanitized_df = sanitized_df + del user_criteria_df + + # Write output ASCII file (csv) summarizing the information extracted from the input + # which will be used to generate the plot. + if self.gen_ascii: + ascii_track_parts = [self.init_date, '.csv'] + ascii_track_output_name = ''.join(ascii_track_parts) + sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) + sanitized_df.to_csv(sanitized_df_filename) + self.logger.info(f"Writing ascii track info as csv file: {sanitized_df_filename}") def create_plot(self): - """! Create the plot, using Cartopy. - """ + Create the plot, using Cartopy + """ # Use PlateCarree projection for now - #use central meridian for central longitude + # use central meridian for central longitude cm_lon = 180 ax = plt.axes(projection=ccrs.PlateCarree(central_longitude=cm_lon)) prj = ccrs.PlateCarree() + # for transforming the annotations (matplotlib to cartopy workaround from Stack Overflow) + transform = ccrs.PlateCarree()._as_mpl_transform(ax) # Add land, coastlines, and ocean ax.add_feature(cfeature.LAND) ax.coastlines() ax.add_feature(cfeature.OCEAN) + # keep map zoomed out to full world. If this # is absent, the default behavior is to zoom # into the portion of the map that contains points. @@ -391,7 +328,7 @@ def create_plot(self): plt.title(self.title + "\nFor forecast with initial time = " + self.init_date) - # Create the NCAR watermark with a timestamp + # Optional: Create the NCAR watermark with a timestamp # This will appear in the bottom right corner of the plot, below # the x-axis. NOTE: The timestamp is in the user's local time zone # and not in UTC time. @@ -405,196 +342,194 @@ def create_plot(self): # Make sure the output directory exists, and create it if it doesn't. util.mkdir_p(self.output_dir) - # Iterate over each unique storm id in self.storm_id_dict and - # set the marker, marker size, and annotation - # before drawing the line and scatter plots. - - # Use counters to set the labels for the legend. Since we don't - # want repetitions in the legend, do this for a select number - # of points. - circle_counter = 0 - plus_counter = 0 - dummy_counter = 0 - - lines_to_write = [] - for cur_storm_id in sorted(self.unique_storm_id): - # Lists used in creating each storm track. - cyclone_points = [] - lon = [] - lat = [] - marker_list = [] - size_list = [] - anno_list = [] - - # For this storm id, get a list of all data (corresponding - # to lines/rows in the tcst data file). - track_info_list = self.storm_id_dict[cur_storm_id] - if not track_info_list: - self.log_error("Empty track list, no data extracted " + - "from track files, exiting.") - return - - for track in track_info_list: - # For now, all the marker symbols will be one color. - color_list = ['red' for _ in range(0, len(track_info_list))] - - lon.append(float(track['lon'])) - lat.append(float(track['lat'])) - - # Differentiate between the forecast lead "groups", - # i.e. 0/12 vs 6/18 hr and - # assign the marker symbol and size. - if track['lead_group'] == '0': - marker = 'o' - marker_list.append(marker) - marker_size = self.circle_marker - size_list.append(marker_size) - label = "Indicates a position at 00 or 12 UTC" - - elif track['lead_group'] == '6': - marker = '+' - marker_list.append(marker) - marker_size = self.cross_marker - size_list.append(marker_size) - label = "\nIndicates a position at 06 or 18 UTC\n" - - # Determine the first point, needed later to annotate. - # pylint:disable=invalid-name - dd = track['valid_dd'] - hh = track['valid_hh'] - if dd and hh: - date_hr_str = dd + '/' + hh + 'z' - anno_list.append(date_hr_str) - else: - date_hr_str = '' - anno_list.append(date_hr_str) - - # Write to the ASCII track file, if requested - if self.gen_ascii: - line_parts = ['model_name: ', track['model_name'], ' ', - 'storm_id: ', track['storm_id'], ' ', - 'init_time: ', track['init_time'], ' ', - 'valid_time: ', track['valid_time'], ' ', - 'lat: ', str(track['lat']), ' ', - 'lon: ', str(track['lon']), ' ', - 'lead_group: ', track['lead_group'], ' ', - 'first_point:', str(track['first_point'])] - line = ''.join(line_parts) - lines_to_write.append(line) - - # Create a scatter plot to add - # the appropriate marker symbol to the forecast - # hours corresponding to 6/18 hours. - - # Annotate the first point of the storm track - for anno, adj_lon, adj_lat in zip(anno_list, lon, lat): - # Annotate the first point of the storm track by - # overlaying the annotation text over all points (all but - # one will have text). plt.annotate DOES NOT work with cartopy, - # instead, use the plt.text method. - plt.text(adj_lon+2, adj_lat+2, anno, transform=prj, - fontsize=self.annotation_font_size, color='red') - - # Generate the scatterplot, where the 6/18 Z forecast times - # are labelled with a '+' - for adj_lon, adj_lat, symbol, sz, colours in zip(lon, lat, - marker_list, - size_list, - color_list): - # red line, red +, red o, marker sizes are recognized, - # no outline color of black for 'o' - # plt.scatter(x, y, s=sz, c=colours, edgecolors=colours, - # facecolors='none', marker=symbol, zorder=2) - # Solid circle, just like the EMC NCEP plots - # Separate the first two points so we can generate the legend - if circle_counter == 0 or plus_counter == 0: - if symbol == 'o': - plt.scatter(adj_lon, adj_lat, s=sz, c=colours, - edgecolors=colours, facecolors=colours, - marker='o', zorder=2, - label="Indicates a position " + - "at 00 or 12 UTC", transform=prj) - circle_counter += 1 - elif symbol == '+': - plt.scatter(adj_lon, adj_lat, s=sz, c=colours, - edgecolors=colours, facecolors=colours, - marker='+', zorder=2, - label="\nIndicates a position at 06 or " + - "18 UTC\n", transform=prj) - plus_counter += 1 - - else: - # Set the legend for additional text using a - # dummy scatter point - if dummy_counter == 0: - plt.scatter(0, 0, zorder=2, marker=None, c='', - label="Date (dd/hhz) is the first " + - "time storm was able to be tracked " + - "in model") - dummy_counter += 1 - plt.scatter(adj_lon, adj_lat, s=sz, c=colours, - edgecolors=colours, - facecolors=colours, marker=symbol, zorder=2, - transform=prj) - - # Finally, overlay the line plot to define the storm tracks - plt.plot(lon, lat, linestyle='-', color=colours, linewidth=.3, - transform=prj) - - # If requested, create an ASCII file with the tracks that are going to - # be plotted. This is useful to debug or verify that what you - # see on the plot is what is expected. - if self.gen_ascii: - ascii_track_parts = [self.init_date, '.txt'] - ascii_track_output_name = ''.join(ascii_track_parts) - track_filename = os.path.join(self.output_dir, - ascii_track_output_name) - self.logger.info(f"Writing ascii track info: {track_filename}") - with open(track_filename, 'w') as file_handle: - for line in lines_to_write: - file_handle.write(f"{line}\n") - - # Draw the legend on the plot - # If you wish to have the legend within the plot: - # plt.legend(loc='lower left', prop={'size':5}, scatterpoints=1) - # The legend is outside the plot, below the x-axis to - # avoid obscuring any storm tracks in the Southern - # Hemisphere. - # ax.legend(loc='lower left', bbox_to_anchor=(-0.03, -0.5), - # fancybox=True, shadow=True, scatterpoints=1, - # prop={'size': 6}) - ax.legend(loc='lower left', bbox_to_anchor=(-0.01, -0.4), + + # get the points for the scatter plots (and the relevant information for annotations, etc.) + points_list = self.get_plot_points() + + # Legend labels + lead_group_0_legend = "Indicates a position at 00 or 12 UTC" + lead_group_6_legend = "Indicates a position at 06 or 18 UTC" + + # to be consistent with the NOAA website, use red for annotations, markers, and lines. + pt_color = 'red' + cross_marker_size = self.cross_marker + circle_marker_size = self.circle_marker + + # Get all the lat and lon (i.e. x and y) points for the '+' and 'o' marker types + # to be used in generating the scatter plots (one for the 0/12 hr and one for the 6/18 hr lead + # groups). Also collect ALL the lons and lats, which will be used to generate the + # line plot (the last plot that goes on top of all the scatter plots). + cross_lons = [] + cross_lats = [] + cross_annotations = [] + circle_lons = [] + circle_lats = [] + circle_annotations = [] + + for idx,pt in enumerate(points_list): + if pt.marker == '+': + cross_lons.append(pt.lon) + cross_lats.append(pt.lat) + cross_annotations.append(pt.annotation) + cross_marker = pt.marker + elif pt.marker == 'o': + circle_lons.append(pt.lon) + circle_lats.append(pt.lat) + circle_annotations.append(pt.annotation) + circle_marker = pt.marker + + # Now generate the scatter plots for the lead group 0/12 hr ('+' marker) and the + # lead group 6/18 hr ('o' marker). + plt.scatter(circle_lons, circle_lats, s=circle_marker_size, c=pt_color, + marker=circle_marker, zorder=2, label=lead_group_0_legend, transform=prj) + plt.scatter(cross_lons, cross_lats, s=cross_marker_size, c=pt_color, + marker=cross_marker, zorder=2, label=lead_group_6_legend, transform=prj) + + # annotations for the scatter plots + counter = 0 + for x,y in zip(circle_lons, circle_lats): + plt.annotate(circle_annotations[counter], (x,y+1), xycoords=transform, color=pt_color, + fontsize=self.annotation_font_size) + counter += 1 + + counter = 0 + for x, y in zip(cross_lons, cross_lats): + plt.annotate(cross_annotations[counter], (x, y + 1), xycoords=transform, color=pt_color, + fontsize=self.annotation_font_size) + counter += 1 + + # Dummy point to add the additional label explaining the labelling of the first + # point in the storm track + plt.scatter(0, 0, zorder=2, marker=None, c='', + label="Date (dd/hhz) is the first " + + "time storm was able to be tracked in model") + + # Settings for the legend box location. + self.logger.debug(f"!!!!!legend font size: {self.legend_font_size}") + ax.legend(loc='lower left', bbox_to_anchor=(0, -0.4), fancybox=True, shadow=True, scatterpoints=1, - prop={'size': 6}) + prop={'size':self.legend_font_size}) - # Write the plot to the output directory - out_filename_parts = [self.init_date, '.png'] - output_plot_name = ''.join(out_filename_parts) - plot_filename = os.path.join(self.output_dir, output_plot_name) - if self.resolution_dpi > 0: - plt.savefig(plot_filename, dpi=self.resolution_dpi) - else: - # use Matplotlib's default if no resolution is set in config file - plt.savefig(plot_filename) + # Generate the line plot + # First collect all the lats and lons for each storm track. Then for each storm track, + # generate a line plot. + pts_by_track_dict = self.get_points_by_track() - # Plot data onto axes - # Uncomment the two lines below if you wish to have a pop up - # window of the plot automatically appear, in addition to the creation - # of the .png version of the plot. - #self.logger.info("Plot is displayed in separate window. - # Close window to continue METplus execution") - # plt.show() + for key in pts_by_track_dict: + lons = [] + lats = [] + for idx, pt in enumerate(pts_by_track[key]): + lons.append(pt.lon) + lats.append(pt.lat) + + # Create the line plot for this storm track + plt.plot(lons, lats, linestyle='-', color=pt_color, linewidth=.5, transform=prj, zorder=3) + + plt.savefig("/Users/minnawin/Desktop/plot.png", dpi=800) + + + + def get_plot_points(self): + """ + Get the lon and lat points to be plotted, along with any other plotting-relevant + information like the marker, whether this is a first point (to be used in + annotating the first point using the valid day date and valid hour), etc. + + :return: A list of named tuples that represent the points to plot with corresponding + plotting information + """ + + # Create a named tuple to store the point information + PlotPt = namedtuple("PlotPt", "storm_id lon lat is_first marker valid_dd valid_hour annotation") + + points_list = [] + storm_id = self.sanitized_df['STORM_ID'] + lons = self.sanitized_df['SLON'] + lats = self.sanitized_df['ALAT'] + is_first_list = self.sanitized_df['IS_FIRST'] + marker_list = self.sanitized_df['MARKER'] + valid_dd_list = self.sanitized_df['VALID_DD'] + valid_hour_list = self.sanitized_df['VALID_HOUR'] + annotation_list = [] + + for idx, cur_lon in enumerate(lons): + if is_first_list[idx] is True: + annotation = str(valid_dd_list[idx]).zfill(2) + '/' + \ + str(valid_hour_list[idx]).zfill(2) + 'z' + else: + annotation = None + + annotation_list.append(annotation) + cur_pt = PlotPt(storm_id, lons[idx], lats[idx], is_first_list[idx], marker_list[idx], + valid_dd_list[idx], valid_hour_list[idx], annotation) + points_list.append(cur_pt) + + return points_list + + def get_points_by_track(self): + """ + Get all the lats and lons for each storm track. Used to generate the line + plot of the storm tracks. + + Args: + + Returns: + points_by_track: Points aggregated by storm track. + Returns a dictionary: where the key is the storm_id + and values are the points (lon,lat) stored in a named tuple + """ + track_dict = {} + LonLat = namedtuple("LonLat", "lon lat") + + for cur_unique in self.unique_storm_ids: + # retrieve the ALAT and ALON values that correspond to the rows for a unique storm id. + # i.e. Get the index value(s) corresponding to this unique storm id + idx_list = self.sanitized_df.index[self.sanitized_df['STORM_ID'] == cur_unique].tolist() + + sanitized_lons_and_lats = [] + indices = [] + for idx in idx_list: + cur_lonlat = LonLat(self.sanitized_df.iloc[idx]['SLON'], self.sanitized_df.iloc[idx]['ALAT']) + sanitized_lons_and_lats.append(cur_lonlat) + indices.append(idx) + + # update the track dictionary + track_dict[cur_unique] = sanitized_lons_and_lats + + # assign to the class variable + # self.pts_by_track = track_dict + + return pts_by_track @staticmethod - def rescale_lon(lon): - """! Rescales longitude, using the same logic employed by MET + def sanitize_lonlist(lon): + """ + Solution from Stack Overflow for "sanitizing" longitudes that cross the International Date Line + https://stackoverflow.com/questions/67730660/plotting-line-across-international-dateline-with-cartopy + + Args: + @param lon: A list of longitudes (float) that correspond to a storm track + + Returns: + new_list: a list of "sanitized" lons that are "corrected" for crossing the + International Date Line """ - if float(lon) > 180.: - adj_lon = lon - 360. - else: - adj_lon = lon + new_list = [] + oldval = 0 + # used to compare adjacent longitudes in a storm track + treshold = 10 + for ix, ea in enumerate(lon): + diff = oldval - ea + if (ix > 0): + if (diff > treshold): + ea = ea + 360 + oldval = ea + new_list.append(ea) + return new_list + + + - return adj_lon From e8034f2822f30ac019acd8df0b82e40829995712 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Tue, 21 Sep 2021 18:43:58 -0600 Subject: [PATCH 03/23] Correctly returning the track dictionary in the get_points_by_track() method. --- metplus/wrappers/cyclone_plotter_wrapper.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 08819e2446..9275523e3b 100755 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -419,7 +419,7 @@ def create_plot(self): for key in pts_by_track_dict: lons = [] lats = [] - for idx, pt in enumerate(pts_by_track[key]): + for idx, pt in enumerate(pts_by_track_dict[key]): lons.append(pt.lon) lats.append(pt.lat) @@ -497,10 +497,8 @@ def get_points_by_track(self): # update the track dictionary track_dict[cur_unique] = sanitized_lons_and_lats - # assign to the class variable - # self.pts_by_track = track_dict - return pts_by_track + return track_dict @staticmethod From 275847623c58b6f0d0f90aee1fff18c402b83ac9 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 22 Sep 2021 08:52:27 -0600 Subject: [PATCH 04/23] removed extra import of pandas --- metplus/wrappers/cyclone_plotter_wrapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 9275523e3b..61ad03aa97 100755 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -9,7 +9,6 @@ import datetime import re import sys -import pandas as pd from collections import namedtuple # handle if module can't be loaded to run wrapper From 73e4ba7ed62ce28f944f5d2e8bca8420e4dc727f Mon Sep 17 00:00:00 2001 From: bikegeek <3753118+bikegeek@users.noreply.github.com> Date: Wed, 22 Sep 2021 10:24:15 -0600 Subject: [PATCH 05/23] Update use_case_groups.json --- .github/parm/use_case_groups.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 689f7186fb..fe39f9f391 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -1,4 +1,9 @@ [ + { + "category": "met_tool_wrapper", + "index_list": "0", + "run": true + }, { "category": "met_tool_wrapper", "index_list": "0-57", From 931c41e9c4349100c11af196d7d760c704b8d32b Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 23 Sep 2021 10:40:24 -0600 Subject: [PATCH 06/23] Cleaned up unneccessary comments and other things in retrieve_data() method. --- metplus/wrappers/cyclone_plotter_wrapper.py | 533 +------------------- 1 file changed, 1 insertion(+), 532 deletions(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 61ad03aa97..879272d74c 100755 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -1,532 +1 @@ -"""!@namespace ExtraTropicalCyclonePlotter -A Python class that generates plots of extra tropical cyclone forecast data, - replicating the NCEP tropical and extra tropical cyclone tracks and - verification plots http://www.emc.ncep.noaa.gov/mmb/gplou/emchurr/glblgen/ -""" - -import os -import time -import datetime -import re -import sys -from collections import namedtuple - -# handle if module can't be loaded to run wrapper -WRAPPER_CANNOT_RUN = False -EXCEPTION_ERR = '' -try: - import pandas as pd - import matplotlib.pyplot as plt - import matplotlib.ticker as mticker - import cartopy.crs as ccrs - import cartopy.feature as cfeature - import cartopy - from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER - - ##If the script is run on a limited-internet access machine, the CARTOPY_DIR environment setting - ##will need to be set in the user-specific system configuration file. Review the Installation section - ##of the User's Guide for more details. - if os.getenv('CARTOPY_DIR'): - cartopy.config['data_dir'] = os.getenv('CARTOPY_DIR', cartopy.config.get('data_dir')) - -except Exception as err_msg: - WRAPPER_CANNOT_RUN = True - EXCEPTION_ERR = err_msg - -import produtil.setup - -from ..util import met_util as util -from ..util import do_string_sub -from . import CommandBuilder - - -class CyclonePlotterWrapper(CommandBuilder): - """! Generate plots of extra tropical storm forecast tracks. - Reads input from ATCF files generated from MET TC-Pairs - """ - - def __init__(self, config, instance=None, config_overrides={}): - self.app_name = 'cyclone_plotter' - - super().__init__(config, - instance=instance, - config_overrides=config_overrides) - - if WRAPPER_CANNOT_RUN: - self.log_error("There was a problem importing modules: " - f"{EXCEPTION_ERR}\n") - return - - self.input_data = self.config.getdir('CYCLONE_PLOTTER_INPUT_DIR') - self.output_dir = self.config.getdir('CYCLONE_PLOTTER_OUTPUT_DIR') - self.init_date = self.config.getraw('config', - 'CYCLONE_PLOTTER_INIT_DATE') - self.init_hr = self.config.getraw('config', 'CYCLONE_PLOTTER_INIT_HR') - - init_time_fmt = self.config.getstr('config', 'INIT_TIME_FMT', '') - - if init_time_fmt: - clock_time = datetime.datetime.strptime( - self.config.getstr('config', - 'CLOCK_TIME'), - '%Y%m%d%H%M%S' - ) - - init_beg = self.config.getraw('config', 'INIT_BEG') - if init_beg: - init_beg_dt = util.get_time_obj(init_beg, - init_time_fmt, - clock_time, - logger=self.logger) - self.init_date = do_string_sub(self.init_date, init=init_beg_dt) - self.init_hr = do_string_sub(self.init_hr, init=init_beg_dt) - init_end = self.config.getraw('config', 'INIT_END') - if init_end: - init_end_dt = util.get_time_obj(init_end, - init_time_fmt, - clock_time, - logger=self.logger) - self.init_end_date = do_string_sub(self.init_date, init=init_end_dt) - self.init_end_hr = do_string_sub(self.init_hr, init=init_end_dt) - - self.model = self.config.getstr('config', 'CYCLONE_PLOTTER_MODEL') - self.title = self.config.getstr('config', - 'CYCLONE_PLOTTER_PLOT_TITLE') - self.gen_ascii = ( - self.config.getbool('config', - 'CYCLONE_PLOTTER_GENERATE_TRACK_ASCII') - ) - # Create a set to keep track of unique storm_ids for each track file. - self.unique_storm_id = set() - # Data structure to separate data based on storm id. - self.storm_id_dict = {} - - # Data/info which we want to retrieve from the track files. - self.columns_of_interest = ['AMODEL', 'STORM_ID', 'INIT', - 'LEAD', 'VALID', 'ALAT', 'ALON'] - self.circle_marker = ( - self.config.getint('config', - 'CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE') - ) - self.annotation_font_size = ( - self.config.getint('config', - 'CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE') - ) - - self.legend_font_size = ( - self.config.getint('config', - 'CYCLONE_PLOTTER_LEGEND_FONT_SIZE') - ) - - self.cross_marker = ( - self.config.getint('config', - 'CYCLONE_PLOTTER_CROSS_MARKER_SIZE') - ) - self.resolution_dpi = ( - self.config.getint('config', - 'CYCLONE_PLOTTER_RESOLUTION_DPI') - ) - - self.add_watermark = self.config.getbool('config', - 'CYCLONE_PLOTTER_ADD_WATERMARK', - True) - - def run_all_times(self): - """! Calls the defs needed to create the cyclone plots - run_all_times() is required by CommandBuilder. - - """ - self.retrieve_data() - self.create_plot() - - def retrieve_data(self): - """! Retrieve data from track files. - Returns: - None - assigns value to the useful class variables: unique_storm_ids, track_df, pts_by_track_dict - """ - self.logger.debug("Begin retrieving data...") - all_tracks_list = [] - - # Store the data in the track list. - if os.path.isdir(self.input_data): - self.logger.debug("Get data from all files in the directory " + - self.input_data) - # Get the list of all files (full file path) in this directory - all_input_files = util.get_files(self.input_data, ".*.tcst", - self.logger) - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ##!!!!USING PANDAS DATAFRAME!!!! GITHUB ISSUE METPLUS #1000 - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # read each file into pandas then concatenate them together - df_list = [pd.read_csv(file, delim_whitespace=True) for file in all_input_files] - combined = pd.concat(df_list, ignore_index=True) - - # if there are any NaN values in the ALAT, ALON, STORM_ID, LEAD, INIT, AMODEL, or VALID column, - # drop that row of data (axis=0). We need all these columns to contain valid data in order - # to create a meaningful plot. - combined_df = combined.copy(deep=True) - combined_df = combined.dropna(axis=0, how='any', - subset=['ALAT', 'ALON', 'STORM_ID', 'LEAD', 'INIT', 'AMODEL', 'VALID']) - - # Retrieve and create the columns of interest - self.logger.debug(f"Number of rows of data: {combined_df.shape[0]}") - combined_subset = combined_df[self.columns_of_interest] - df = combined_subset.copy(deep=True) - df.allows_duplicate_labels = False - df['INIT'] = df['INIT'].astype(str) - df['INIT_YMD'] = (df['INIT'].str[:8]).astype(int) - df['INIT_HOUR'] = (df['INIT'].str[9:11]).astype(int) - df['LEAD'] = df['LEAD']/10000 - df['LEAD'] = df['LEAD'].astype(int) - df['VALID_DD'] = (df['VALID'].str[6:8]).astype(int) - df['VALID_HOUR'] = (df['VALID'].str[9:11]).astype(int) - df['VALID'] = df['VALID'].astype(int) - - # clean up dataframes that are no longer needed - del combined - del combined_df - - # Subset the dataframe to include only the data relevant to the user's criteria as - # specified in the configuration file. - init_date = int(self.init_date) - init_hh = int(self.init_hr) - model_name = self.model - - mask = df[(df['AMODEL'] == model_name) & (df['INIT_YMD'] >= init_date) & - (df['INIT_HOUR'] >= init_hh)] - user_criteria_df = mask - - # Aggregate the ALON values based on unique storm id in order to sanitize the longitude values - # that cross the International Date Line. - unique_storm_ids_set = set(user_criteria_df['STORM_ID']) - self.unique_storm_id = unique_storm_ids_set - self.unique_storm_ids = list(unique_storm_ids_set) - nunique = len(self.unique_storm_ids) - self.logger.debug(f" {nunique} unique storm ids identified") - - # Use named tuples to store the relevant storm track information (their index value in the dataframe, - # track id, and ALON values and later on the SLON (sanitized ALON values). - TrackPt = namedtuple("TrackPt", "indices track alons ") - - # named tuple holding "sanitized" longitudes - SanTrackPt = namedtuple("SanTrackPt", "indices track alons slons") - - # Keep track of the unique storm tracks by their storm_id - storm_track_dict = {} - - for cur_unique in self.unique_storm_ids: - idx_list = user_criteria_df.index[user_criteria_df['STORM_ID'] == cur_unique].tolist() - alons = [] - indices = [] - - for idx in idx_list: - alons.append(user_criteria_df.iloc[idx]['ALON']) - indices.append(idx) - - # create the track_pt tuple and add it to the storm track dictionary - track_pt = TrackPt(indices, cur_unique, alons) - storm_track_dict[cur_unique] = track_pt - - # create a new dataframe to contain the sanitized lons (i.e. the original ALONs that have - # been cleaned up when crossing the International Date Line) - sanitized_df = user_criteria_df.copy(deep=True) - - # Now we have a dictionary that aggregates the data based on storm tracks (via storm id) - # and will contain the "sanitized" lons - sanitized_storm_tracks = {} - for key in storm_track_dict: - # sanitize the longitudes, create a new SanTrackPt named tuple and add that to a new dictionary - # that keeps track of the sanitized data based on the storm id - sanitized_lons = self.sanitize_lonlist(storm_track_dict[key].alons) - sanitized_track_pt = SanTrackPt(storm_track_dict[key].indices, storm_track_dict[key].track, - storm_track_dict[key].alons, sanitized_lons) - sanitized_storm_tracks[key] = sanitized_track_pt - - # fill in the sanitized dataframe sanitized_df - for key in sanitized_storm_tracks: - # now use the indices of the storm tracks to correctly assign the sanitized - # lons to the appropriate row in the dataframe to maintain the row ordering of - # the original dataframe - idx_list = sanitized_storm_tracks[key].indices - - for i, idx in enumerate(idx_list): - sanitized_df.loc[idx,'SLON'] = sanitized_storm_tracks[key].slons[i] - - # Set some useful values used for plotting. - # Set the IS_FIRST value to True if this is the first point in the storm track, False - # otherwise - if i == 0: - sanitized_df.loc[idx, 'IS_FIRST'] = True - else: - sanitized_df.loc[idx, 'IS_FIRST'] = False - - # Set the lead group to the character '0' if the valid hour is 0 or 12, or to the - # charcter '6' if the valid hour is 6 or 18. Set the marker to correspond to the - # valid hour: 'o' (open circle) for 0 or 12 valid hour, or '+' (small plus/cross) - # for 6 or 18. - if sanitized_df.loc[idx, 'VALID_HOUR'] == 0 or sanitized_df.loc[idx, 'VALID_HOUR'] == 12: - sanitized_df.loc[idx, 'LEAD_GROUP'] ='0' - sanitized_df.loc[idx, 'MARKER'] = 'o' - elif sanitized_df.loc[idx, 'VALID_HOUR'] == 6 or sanitized_df.loc[idx, 'VALID_HOUR'] == 18: - sanitized_df.loc[idx, 'LEAD_GROUP'] = '6' - sanitized_df.loc[idx, 'MARKER'] = '+' - - # store the sanitized dataframe, and remove unecessary dataframes - self.sanitized_df = sanitized_df - del user_criteria_df - - # Write output ASCII file (csv) summarizing the information extracted from the input - # which will be used to generate the plot. - if self.gen_ascii: - ascii_track_parts = [self.init_date, '.csv'] - ascii_track_output_name = ''.join(ascii_track_parts) - sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) - sanitized_df.to_csv(sanitized_df_filename) - self.logger.info(f"Writing ascii track info as csv file: {sanitized_df_filename}") - - def create_plot(self): - """ - Create the plot, using Cartopy - - """ - - # Use PlateCarree projection for now - # use central meridian for central longitude - cm_lon = 180 - ax = plt.axes(projection=ccrs.PlateCarree(central_longitude=cm_lon)) - prj = ccrs.PlateCarree() - # for transforming the annotations (matplotlib to cartopy workaround from Stack Overflow) - transform = ccrs.PlateCarree()._as_mpl_transform(ax) - - # Add land, coastlines, and ocean - ax.add_feature(cfeature.LAND) - ax.coastlines() - ax.add_feature(cfeature.OCEAN) - - # keep map zoomed out to full world. If this - # is absent, the default behavior is to zoom - # into the portion of the map that contains points. - ax.set_global() - - # Add grid lines for longitude and latitude - ax.gridlines(draw_labels=False, xlocs=[180, -180]) - gl = ax.gridlines(crs=prj, - draw_labels=True, linewidth=1, color='gray', - alpha=0.5, linestyle='--') - gl.xlabels_top = False - gl.ylabels_left = False - gl.xlines = True - gl.xformatter = LONGITUDE_FORMATTER - gl.yformatter = LATITUDE_FORMATTER - gl.xlabel_style = {'size': 9, 'color': 'blue'} - gl.xlabel_style = {'color': 'black', 'weight': 'normal'} - - # Plot title - plt.title(self.title + "\nFor forecast with initial time = " + - self.init_date) - - # Optional: Create the NCAR watermark with a timestamp - # This will appear in the bottom right corner of the plot, below - # the x-axis. NOTE: The timestamp is in the user's local time zone - # and not in UTC time. - if self.add_watermark: - ts = time.time() - st = datetime.datetime.fromtimestamp(ts).strftime( - '%Y-%m-%d %H:%M:%S') - watermark = 'DTC METplus\nplot created at: ' + st - plt.text(60, -130, watermark, fontsize=5, alpha=0.25) - - # Make sure the output directory exists, and create it if it doesn't. - util.mkdir_p(self.output_dir) - - - # get the points for the scatter plots (and the relevant information for annotations, etc.) - points_list = self.get_plot_points() - - # Legend labels - lead_group_0_legend = "Indicates a position at 00 or 12 UTC" - lead_group_6_legend = "Indicates a position at 06 or 18 UTC" - - # to be consistent with the NOAA website, use red for annotations, markers, and lines. - pt_color = 'red' - cross_marker_size = self.cross_marker - circle_marker_size = self.circle_marker - - # Get all the lat and lon (i.e. x and y) points for the '+' and 'o' marker types - # to be used in generating the scatter plots (one for the 0/12 hr and one for the 6/18 hr lead - # groups). Also collect ALL the lons and lats, which will be used to generate the - # line plot (the last plot that goes on top of all the scatter plots). - cross_lons = [] - cross_lats = [] - cross_annotations = [] - circle_lons = [] - circle_lats = [] - circle_annotations = [] - - for idx,pt in enumerate(points_list): - if pt.marker == '+': - cross_lons.append(pt.lon) - cross_lats.append(pt.lat) - cross_annotations.append(pt.annotation) - cross_marker = pt.marker - elif pt.marker == 'o': - circle_lons.append(pt.lon) - circle_lats.append(pt.lat) - circle_annotations.append(pt.annotation) - circle_marker = pt.marker - - # Now generate the scatter plots for the lead group 0/12 hr ('+' marker) and the - # lead group 6/18 hr ('o' marker). - plt.scatter(circle_lons, circle_lats, s=circle_marker_size, c=pt_color, - marker=circle_marker, zorder=2, label=lead_group_0_legend, transform=prj) - plt.scatter(cross_lons, cross_lats, s=cross_marker_size, c=pt_color, - marker=cross_marker, zorder=2, label=lead_group_6_legend, transform=prj) - - # annotations for the scatter plots - counter = 0 - for x,y in zip(circle_lons, circle_lats): - plt.annotate(circle_annotations[counter], (x,y+1), xycoords=transform, color=pt_color, - fontsize=self.annotation_font_size) - counter += 1 - - counter = 0 - for x, y in zip(cross_lons, cross_lats): - plt.annotate(cross_annotations[counter], (x, y + 1), xycoords=transform, color=pt_color, - fontsize=self.annotation_font_size) - counter += 1 - - # Dummy point to add the additional label explaining the labelling of the first - # point in the storm track - plt.scatter(0, 0, zorder=2, marker=None, c='', - label="Date (dd/hhz) is the first " + - "time storm was able to be tracked in model") - - # Settings for the legend box location. - self.logger.debug(f"!!!!!legend font size: {self.legend_font_size}") - ax.legend(loc='lower left', bbox_to_anchor=(0, -0.4), - fancybox=True, shadow=True, scatterpoints=1, - prop={'size':self.legend_font_size}) - - - # Generate the line plot - # First collect all the lats and lons for each storm track. Then for each storm track, - # generate a line plot. - pts_by_track_dict = self.get_points_by_track() - - for key in pts_by_track_dict: - lons = [] - lats = [] - for idx, pt in enumerate(pts_by_track_dict[key]): - lons.append(pt.lon) - lats.append(pt.lat) - - # Create the line plot for this storm track - plt.plot(lons, lats, linestyle='-', color=pt_color, linewidth=.5, transform=prj, zorder=3) - - plt.savefig("/Users/minnawin/Desktop/plot.png", dpi=800) - - - - def get_plot_points(self): - """ - Get the lon and lat points to be plotted, along with any other plotting-relevant - information like the marker, whether this is a first point (to be used in - annotating the first point using the valid day date and valid hour), etc. - - :return: A list of named tuples that represent the points to plot with corresponding - plotting information - """ - - # Create a named tuple to store the point information - PlotPt = namedtuple("PlotPt", "storm_id lon lat is_first marker valid_dd valid_hour annotation") - - points_list = [] - storm_id = self.sanitized_df['STORM_ID'] - lons = self.sanitized_df['SLON'] - lats = self.sanitized_df['ALAT'] - is_first_list = self.sanitized_df['IS_FIRST'] - marker_list = self.sanitized_df['MARKER'] - valid_dd_list = self.sanitized_df['VALID_DD'] - valid_hour_list = self.sanitized_df['VALID_HOUR'] - annotation_list = [] - - for idx, cur_lon in enumerate(lons): - if is_first_list[idx] is True: - annotation = str(valid_dd_list[idx]).zfill(2) + '/' + \ - str(valid_hour_list[idx]).zfill(2) + 'z' - else: - annotation = None - - annotation_list.append(annotation) - cur_pt = PlotPt(storm_id, lons[idx], lats[idx], is_first_list[idx], marker_list[idx], - valid_dd_list[idx], valid_hour_list[idx], annotation) - points_list.append(cur_pt) - - return points_list - - def get_points_by_track(self): - """ - Get all the lats and lons for each storm track. Used to generate the line - plot of the storm tracks. - - Args: - - Returns: - points_by_track: Points aggregated by storm track. - Returns a dictionary: where the key is the storm_id - and values are the points (lon,lat) stored in a named tuple - """ - track_dict = {} - LonLat = namedtuple("LonLat", "lon lat") - - for cur_unique in self.unique_storm_ids: - # retrieve the ALAT and ALON values that correspond to the rows for a unique storm id. - # i.e. Get the index value(s) corresponding to this unique storm id - idx_list = self.sanitized_df.index[self.sanitized_df['STORM_ID'] == cur_unique].tolist() - - sanitized_lons_and_lats = [] - indices = [] - for idx in idx_list: - cur_lonlat = LonLat(self.sanitized_df.iloc[idx]['SLON'], self.sanitized_df.iloc[idx]['ALAT']) - sanitized_lons_and_lats.append(cur_lonlat) - indices.append(idx) - - # update the track dictionary - track_dict[cur_unique] = sanitized_lons_and_lats - - - return track_dict - - - @staticmethod - def sanitize_lonlist(lon): - """ - Solution from Stack Overflow for "sanitizing" longitudes that cross the International Date Line - https://stackoverflow.com/questions/67730660/plotting-line-across-international-dateline-with-cartopy - - Args: - @param lon: A list of longitudes (float) that correspond to a storm track - - Returns: - new_list: a list of "sanitized" lons that are "corrected" for crossing the - International Date Line - """ - - new_list = [] - oldval = 0 - # used to compare adjacent longitudes in a storm track - treshold = 10 - for ix, ea in enumerate(lon): - diff = oldval - ea - if (ix > 0): - if (diff > treshold): - ea = ea + 360 - oldval = ea - new_list.append(ea) - return new_list - - - - +"""!@namespace ExtraTropicalCyclonePlotter A Python class that generates plots of extra tropical cyclone forecast data, replicating the NCEP tropical and extra tropical cyclone tracks and verification plots http://www.emc.ncep.noaa.gov/mmb/gplou/emchurr/glblgen/ """ import os import time import datetime import re import sys from collections import namedtuple # handle if module can't be loaded to run wrapper WRAPPER_CANNOT_RUN = False EXCEPTION_ERR = '' try: import pandas as pd import matplotlib.pyplot as plt import matplotlib.ticker as mticker import cartopy.crs as ccrs import cartopy.feature as cfeature import cartopy from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER ##If the script is run on a limited-internet access machine, the CARTOPY_DIR environment setting ##will need to be set in the user-specific system configuration file. Review the Installation section ##of the User's Guide for more details. if os.getenv('CARTOPY_DIR'): cartopy.config['data_dir'] = os.getenv('CARTOPY_DIR', cartopy.config.get('data_dir')) except Exception as err_msg: WRAPPER_CANNOT_RUN = True EXCEPTION_ERR = err_msg import produtil.setup from ..util import met_util as util from ..util import do_string_sub from . import CommandBuilder class CyclonePlotterWrapper(CommandBuilder): """! Generate plots of extra tropical storm forecast tracks. Reads input from ATCF files generated from MET TC-Pairs """ def __init__(self, config, instance=None, config_overrides={}): self.app_name = 'cyclone_plotter' super().__init__(config, instance=instance, config_overrides=config_overrides) if WRAPPER_CANNOT_RUN: self.log_error("There was a problem importing modules: " f"{EXCEPTION_ERR}\n") return self.input_data = self.config.getdir('CYCLONE_PLOTTER_INPUT_DIR') self.output_dir = self.config.getdir('CYCLONE_PLOTTER_OUTPUT_DIR') self.init_date = self.config.getraw('config', 'CYCLONE_PLOTTER_INIT_DATE') self.init_hr = self.config.getraw('config', 'CYCLONE_PLOTTER_INIT_HR') init_time_fmt = self.config.getstr('config', 'INIT_TIME_FMT', '') if init_time_fmt: clock_time = datetime.datetime.strptime( self.config.getstr('config', 'CLOCK_TIME'), '%Y%m%d%H%M%S' ) init_beg = self.config.getraw('config', 'INIT_BEG') if init_beg: init_beg_dt = util.get_time_obj(init_beg, init_time_fmt, clock_time, logger=self.logger) self.init_date = do_string_sub(self.init_date, init=init_beg_dt) self.init_hr = do_string_sub(self.init_hr, init=init_beg_dt) init_end = self.config.getraw('config', 'INIT_END') if init_end: init_end_dt = util.get_time_obj(init_end, init_time_fmt, clock_time, logger=self.logger) self.init_end_date = do_string_sub(self.init_date, init=init_end_dt) self.init_end_hr = do_string_sub(self.init_hr, init=init_end_dt) self.model = self.config.getstr('config', 'CYCLONE_PLOTTER_MODEL') self.title = self.config.getstr('config', 'CYCLONE_PLOTTER_PLOT_TITLE') self.gen_ascii = ( self.config.getbool('config', 'CYCLONE_PLOTTER_GENERATE_TRACK_ASCII') ) # Create a set to keep track of unique storm_ids for each track file. self.unique_storm_id = set() # Data structure to separate data based on storm id. self.storm_id_dict = {} # Data/info which we want to retrieve from the track files. self.columns_of_interest = ['AMODEL', 'STORM_ID', 'INIT', 'LEAD', 'VALID', 'ALAT', 'ALON'] self.circle_marker = ( self.config.getint('config', 'CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE') ) self.annotation_font_size = ( self.config.getint('config', 'CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE') ) self.legend_font_size = ( self.config.getint('config', 'CYCLONE_PLOTTER_LEGEND_FONT_SIZE') ) self.cross_marker = ( self.config.getint('config', 'CYCLONE_PLOTTER_CROSS_MARKER_SIZE') ) self.resolution_dpi = ( self.config.getint('config', 'CYCLONE_PLOTTER_RESOLUTION_DPI') ) self.add_watermark = self.config.getbool('config', 'CYCLONE_PLOTTER_ADD_WATERMARK', True) def run_all_times(self): """! Calls the defs needed to create the cyclone plots run_all_times() is required by CommandBuilder. """ self.sanitized_df = self.retrieve_data() self.create_plot() def retrieve_data(self): """! Retrieve data from track files. Returns: None """ self.logger.debug("Begin retrieving data...") all_tracks_list = [] # Store the data in the track list. if os.path.isdir(self.input_data): self.logger.debug("Get data from all files in the directory " + self.input_data) # Get the list of all files (full file path) in this directory all_input_files = util.get_files(self.input_data, ".*.tcst", self.logger) # read each file into pandas then concatenate them together df_list = [pd.read_csv(file, delim_whitespace=True) for file in all_input_files] combined = pd.concat(df_list, ignore_index=True) # if there are any NaN values in the ALAT, ALON, STORM_ID, LEAD, INIT, AMODEL, or VALID column, # drop that row of data (axis=0). We need all these columns to contain valid data in order # to create a meaningful plot. combined_df = combined.copy(deep=True) combined_df = combined.dropna(axis=0, how='any', subset=['ALAT', 'ALON', 'STORM_ID', 'LEAD', 'INIT', 'AMODEL', 'VALID']) # Retrieve and create the columns of interest self.logger.debug(f"Number of rows of data: {combined_df.shape[0]}") combined_subset = combined_df[self.columns_of_interest] df = combined_subset.copy(deep=True) df.allows_duplicate_labels = False df['INIT'] = df['INIT'].astype(str) df['INIT_YMD'] = (df['INIT'].str[:8]).astype(int) df['INIT_HOUR'] = (df['INIT'].str[9:11]).astype(int) df['LEAD'] = df['LEAD']/10000 df['LEAD'] = df['LEAD'].astype(int) df['VALID_DD'] = (df['VALID'].str[6:8]).astype(int) df['VALID_HOUR'] = (df['VALID'].str[9:11]).astype(int) df['VALID'] = df['VALID'].astype(int) # clean up dataframes that are no longer needed del combined del combined_df # Subset the dataframe to include only the data relevant to the user's criteria as # specified in the configuration file. init_date = int(self.init_date) init_hh = int(self.init_hr) model_name = self.model mask = df[(df['AMODEL'] == model_name) & (df['INIT_YMD'] >= init_date) & (df['INIT_HOUR'] >= init_hh)] user_criteria_df = mask # Aggregate the ALON values based on unique storm id in order to sanitize the longitude values # that cross the International Date Line. unique_storm_ids_set = set(user_criteria_df['STORM_ID']) self.unique_storm_id = unique_storm_ids_set self.unique_storm_ids = list(unique_storm_ids_set) nunique = len(self.unique_storm_ids) self.logger.debug(f" {nunique} unique storm ids identified") # Use named tuples to store the relevant storm track information (their index value in the dataframe, # track id, and ALON values and later on the SLON (sanitized ALON values). TrackPt = namedtuple("TrackPt", "indices track alons ") # named tuple holding "sanitized" longitudes SanTrackPt = namedtuple("SanTrackPt", "indices track alons slons") # Keep track of the unique storm tracks by their storm_id storm_track_dict = {} for cur_unique in self.unique_storm_ids: idx_list = user_criteria_df.index[user_criteria_df['STORM_ID'] == cur_unique].tolist() alons = [] indices = [] for idx in idx_list: alons.append(user_criteria_df.iloc[idx]['ALON']) indices.append(idx) # create the track_pt tuple and add it to the storm track dictionary track_pt = TrackPt(indices, cur_unique, alons) storm_track_dict[cur_unique] = track_pt # create a new dataframe to contain the sanitized lons (i.e. the original ALONs that have # been cleaned up when crossing the International Date Line) sanitized_df = user_criteria_df.copy(deep=True) # Now we have a dictionary that aggregates the data based on storm tracks (via storm id) # and will contain the "sanitized" lons sanitized_storm_tracks = {} for key in storm_track_dict: # sanitize the longitudes, create a new SanTrackPt named tuple and add that to a new dictionary # that keeps track of the sanitized data based on the storm id sanitized_lons = self.sanitize_lonlist(storm_track_dict[key].alons) sanitized_track_pt = SanTrackPt(storm_track_dict[key].indices, storm_track_dict[key].track, storm_track_dict[key].alons, sanitized_lons) sanitized_storm_tracks[key] = sanitized_track_pt # fill in the sanitized dataframe sanitized_df for key in sanitized_storm_tracks: # now use the indices of the storm tracks to correctly assign the sanitized # lons to the appropriate row in the dataframe to maintain the row ordering of # the original dataframe idx_list = sanitized_storm_tracks[key].indices for i, idx in enumerate(idx_list): sanitized_df.loc[idx,'SLON'] = sanitized_storm_tracks[key].slons[i] # Set some useful values used for plotting. # Set the IS_FIRST value to True if this is the first point in the storm track, False # otherwise if i == 0: sanitized_df.loc[idx, 'IS_FIRST'] = True else: sanitized_df.loc[idx, 'IS_FIRST'] = False # Set the lead group to the character '0' if the valid hour is 0 or 12, or to the # charcter '6' if the valid hour is 6 or 18. Set the marker to correspond to the # valid hour: 'o' (open circle) for 0 or 12 valid hour, or '+' (small plus/cross) # for 6 or 18. if sanitized_df.loc[idx, 'VALID_HOUR'] == 0 or sanitized_df.loc[idx, 'VALID_HOUR'] == 12: sanitized_df.loc[idx, 'LEAD_GROUP'] ='0' sanitized_df.loc[idx, 'MARKER'] = 'o' elif sanitized_df.loc[idx, 'VALID_HOUR'] == 6 or sanitized_df.loc[idx, 'VALID_HOUR'] == 18: sanitized_df.loc[idx, 'LEAD_GROUP'] = '6' sanitized_df.loc[idx, 'MARKER'] = '+' # Write output ASCII file (csv) summarizing the information extracted from the input # which will be used to generate the plot. if self.gen_ascii: ascii_track_parts = [self.init_date, '.csv'] ascii_track_output_name = ''.join(ascii_track_parts) sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) sanitized_df.to_csv(sanitized_df_filename) self.logger.info(f"Writing ascii track info as csv file: {sanitized_df_filename}") return sanitized_df def create_plot(self): """ Create the plot, using Cartopy """ # Use PlateCarree projection for now # use central meridian for central longitude cm_lon = 180 ax = plt.axes(projection=ccrs.PlateCarree(central_longitude=cm_lon)) prj = ccrs.PlateCarree() # for transforming the annotations (matplotlib to cartopy workaround from Stack Overflow) transform = ccrs.PlateCarree()._as_mpl_transform(ax) # Add land, coastlines, and ocean ax.add_feature(cfeature.LAND) ax.coastlines() ax.add_feature(cfeature.OCEAN) # keep map zoomed out to full world. If this # is absent, the default behavior is to zoom # into the portion of the map that contains points. ax.set_global() # Add grid lines for longitude and latitude ax.gridlines(draw_labels=False, xlocs=[180, -180]) gl = ax.gridlines(crs=prj, draw_labels=True, linewidth=1, color='gray', alpha=0.5, linestyle='--') gl.xlabels_top = False gl.ylabels_left = False gl.xlines = True gl.xformatter = LONGITUDE_FORMATTER gl.yformatter = LATITUDE_FORMATTER gl.xlabel_style = {'size': 9, 'color': 'blue'} gl.xlabel_style = {'color': 'black', 'weight': 'normal'} # Plot title plt.title(self.title + "\nFor forecast with initial time = " + self.init_date) # Optional: Create the NCAR watermark with a timestamp # This will appear in the bottom right corner of the plot, below # the x-axis. NOTE: The timestamp is in the user's local time zone # and not in UTC time. if self.add_watermark: ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d %H:%M:%S') watermark = 'DTC METplus\nplot created at: ' + st plt.text(60, -130, watermark, fontsize=5, alpha=0.25) # Make sure the output directory exists, and create it if it doesn't. util.mkdir_p(self.output_dir) # get the points for the scatter plots (and the relevant information for annotations, etc.) points_list = self.get_plot_points() # Legend labels lead_group_0_legend = "Indicates a position at 00 or 12 UTC" lead_group_6_legend = "Indicates a position at 06 or 18 UTC" # to be consistent with the NOAA website, use red for annotations, markers, and lines. pt_color = 'red' cross_marker_size = self.cross_marker circle_marker_size = self.circle_marker # Get all the lat and lon (i.e. x and y) points for the '+' and 'o' marker types # to be used in generating the scatter plots (one for the 0/12 hr and one for the 6/18 hr lead # groups). Also collect ALL the lons and lats, which will be used to generate the # line plot (the last plot that goes on top of all the scatter plots). cross_lons = [] cross_lats = [] cross_annotations = [] circle_lons = [] circle_lats = [] circle_annotations = [] for idx,pt in enumerate(points_list): if pt.marker == '+': cross_lons.append(pt.lon) cross_lats.append(pt.lat) cross_annotations.append(pt.annotation) cross_marker = pt.marker elif pt.marker == 'o': circle_lons.append(pt.lon) circle_lats.append(pt.lat) circle_annotations.append(pt.annotation) circle_marker = pt.marker # Now generate the scatter plots for the lead group 0/12 hr ('+' marker) and the # lead group 6/18 hr ('o' marker). plt.scatter(circle_lons, circle_lats, s=circle_marker_size, c=pt_color, marker=circle_marker, zorder=2, label=lead_group_0_legend, transform=prj) plt.scatter(cross_lons, cross_lats, s=cross_marker_size, c=pt_color, marker=cross_marker, zorder=2, label=lead_group_6_legend, transform=prj) # annotations for the scatter plots counter = 0 for x,y in zip(circle_lons, circle_lats): plt.annotate(circle_annotations[counter], (x,y+1), xycoords=transform, color=pt_color, fontsize=self.annotation_font_size) counter += 1 counter = 0 for x, y in zip(cross_lons, cross_lats): plt.annotate(cross_annotations[counter], (x, y + 1), xycoords=transform, color=pt_color, fontsize=self.annotation_font_size) counter += 1 # Dummy point to add the additional label explaining the labelling of the first # point in the storm track plt.scatter(0, 0, zorder=2, marker=None, c='', label="Date (dd/hhz) is the first " + "time storm was able to be tracked in model") # Settings for the legend box location. self.logger.debug(f"!!!!!legend font size: {self.legend_font_size}") ax.legend(loc='lower left', bbox_to_anchor=(0, -0.4), fancybox=True, shadow=True, scatterpoints=1, prop={'size':self.legend_font_size}) # Generate the line plot # First collect all the lats and lons for each storm track. Then for each storm track, # generate a line plot. pts_by_track_dict = self.get_points_by_track() for key in pts_by_track_dict: lons = [] lats = [] for idx, pt in enumerate(pts_by_track_dict[key]): lons.append(pt.lon) lats.append(pt.lat) # Create the line plot for this storm track plt.plot(lons, lats, linestyle='-', color=pt_color, linewidth=.5, transform=prj, zorder=3) plt.savefig("/Users/minnawin/Desktop/plot.png", dpi=800) def get_plot_points(self): """ Get the lon and lat points to be plotted, along with any other plotting-relevant information like the marker, whether this is a first point (to be used in annotating the first point using the valid day date and valid hour), etc. :return: A list of named tuples that represent the points to plot with corresponding plotting information """ # Create a named tuple to store the point information PlotPt = namedtuple("PlotPt", "storm_id lon lat is_first marker valid_dd valid_hour annotation") points_list = [] storm_id = self.sanitized_df['STORM_ID'] lons = self.sanitized_df['SLON'] lats = self.sanitized_df['ALAT'] is_first_list = self.sanitized_df['IS_FIRST'] marker_list = self.sanitized_df['MARKER'] valid_dd_list = self.sanitized_df['VALID_DD'] valid_hour_list = self.sanitized_df['VALID_HOUR'] annotation_list = [] for idx, cur_lon in enumerate(lons): if is_first_list[idx] is True: annotation = str(valid_dd_list[idx]).zfill(2) + '/' + \ str(valid_hour_list[idx]).zfill(2) + 'z' else: annotation = None annotation_list.append(annotation) cur_pt = PlotPt(storm_id, lons[idx], lats[idx], is_first_list[idx], marker_list[idx], valid_dd_list[idx], valid_hour_list[idx], annotation) points_list.append(cur_pt) return points_list def get_points_by_track(self): """ Get all the lats and lons for each storm track. Used to generate the line plot of the storm tracks. Args: Returns: points_by_track: Points aggregated by storm track. Returns a dictionary: where the key is the storm_id and values are the points (lon,lat) stored in a named tuple """ track_dict = {} LonLat = namedtuple("LonLat", "lon lat") for cur_unique in self.unique_storm_ids: # retrieve the ALAT and ALON values that correspond to the rows for a unique storm id. # i.e. Get the index value(s) corresponding to this unique storm id idx_list = self.sanitized_df.index[self.sanitized_df['STORM_ID'] == cur_unique].tolist() sanitized_lons_and_lats = [] indices = [] for idx in idx_list: cur_lonlat = LonLat(self.sanitized_df.iloc[idx]['SLON'], self.sanitized_df.iloc[idx]['ALAT']) sanitized_lons_and_lats.append(cur_lonlat) indices.append(idx) # update the track dictionary track_dict[cur_unique] = sanitized_lons_and_lats return track_dict @staticmethod def sanitize_lonlist(lon): """ Solution from Stack Overflow for "sanitizing" longitudes that cross the International Date Line https://stackoverflow.com/questions/67730660/plotting-line-across-international-dateline-with-cartopy Args: @param lon: A list of longitudes (float) that correspond to a storm track Returns: new_list: a list of "sanitized" lons that are "corrected" for crossing the International Date Line """ new_list = [] oldval = 0 # used to compare adjacent longitudes in a storm track treshold = 10 for ix, ea in enumerate(lon): diff = oldval - ea if (ix > 0): if (diff > treshold): ea = ea + 360 oldval = ea new_list.append(ea) return new_list \ No newline at end of file From a9f94640ae36ebe963dcca572371e43dbfafb50e Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 23 Sep 2021 10:57:13 -0600 Subject: [PATCH 07/23] Previous checkin was missing text due to IDE settings. --- metplus/wrappers/cyclone_plotter_wrapper.py | 525 +++++++++++++++++++- 1 file changed, 524 insertions(+), 1 deletion(-) mode change 100755 => 100644 metplus/wrappers/cyclone_plotter_wrapper.py diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py old mode 100755 new mode 100644 index 879272d74c..74690f11b1 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -1 +1,524 @@ -"""!@namespace ExtraTropicalCyclonePlotter A Python class that generates plots of extra tropical cyclone forecast data, replicating the NCEP tropical and extra tropical cyclone tracks and verification plots http://www.emc.ncep.noaa.gov/mmb/gplou/emchurr/glblgen/ """ import os import time import datetime import re import sys from collections import namedtuple # handle if module can't be loaded to run wrapper WRAPPER_CANNOT_RUN = False EXCEPTION_ERR = '' try: import pandas as pd import matplotlib.pyplot as plt import matplotlib.ticker as mticker import cartopy.crs as ccrs import cartopy.feature as cfeature import cartopy from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER ##If the script is run on a limited-internet access machine, the CARTOPY_DIR environment setting ##will need to be set in the user-specific system configuration file. Review the Installation section ##of the User's Guide for more details. if os.getenv('CARTOPY_DIR'): cartopy.config['data_dir'] = os.getenv('CARTOPY_DIR', cartopy.config.get('data_dir')) except Exception as err_msg: WRAPPER_CANNOT_RUN = True EXCEPTION_ERR = err_msg import produtil.setup from ..util import met_util as util from ..util import do_string_sub from . import CommandBuilder class CyclonePlotterWrapper(CommandBuilder): """! Generate plots of extra tropical storm forecast tracks. Reads input from ATCF files generated from MET TC-Pairs """ def __init__(self, config, instance=None, config_overrides={}): self.app_name = 'cyclone_plotter' super().__init__(config, instance=instance, config_overrides=config_overrides) if WRAPPER_CANNOT_RUN: self.log_error("There was a problem importing modules: " f"{EXCEPTION_ERR}\n") return self.input_data = self.config.getdir('CYCLONE_PLOTTER_INPUT_DIR') self.output_dir = self.config.getdir('CYCLONE_PLOTTER_OUTPUT_DIR') self.init_date = self.config.getraw('config', 'CYCLONE_PLOTTER_INIT_DATE') self.init_hr = self.config.getraw('config', 'CYCLONE_PLOTTER_INIT_HR') init_time_fmt = self.config.getstr('config', 'INIT_TIME_FMT', '') if init_time_fmt: clock_time = datetime.datetime.strptime( self.config.getstr('config', 'CLOCK_TIME'), '%Y%m%d%H%M%S' ) init_beg = self.config.getraw('config', 'INIT_BEG') if init_beg: init_beg_dt = util.get_time_obj(init_beg, init_time_fmt, clock_time, logger=self.logger) self.init_date = do_string_sub(self.init_date, init=init_beg_dt) self.init_hr = do_string_sub(self.init_hr, init=init_beg_dt) init_end = self.config.getraw('config', 'INIT_END') if init_end: init_end_dt = util.get_time_obj(init_end, init_time_fmt, clock_time, logger=self.logger) self.init_end_date = do_string_sub(self.init_date, init=init_end_dt) self.init_end_hr = do_string_sub(self.init_hr, init=init_end_dt) self.model = self.config.getstr('config', 'CYCLONE_PLOTTER_MODEL') self.title = self.config.getstr('config', 'CYCLONE_PLOTTER_PLOT_TITLE') self.gen_ascii = ( self.config.getbool('config', 'CYCLONE_PLOTTER_GENERATE_TRACK_ASCII') ) # Create a set to keep track of unique storm_ids for each track file. self.unique_storm_id = set() # Data structure to separate data based on storm id. self.storm_id_dict = {} # Data/info which we want to retrieve from the track files. self.columns_of_interest = ['AMODEL', 'STORM_ID', 'INIT', 'LEAD', 'VALID', 'ALAT', 'ALON'] self.circle_marker = ( self.config.getint('config', 'CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE') ) self.annotation_font_size = ( self.config.getint('config', 'CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE') ) self.legend_font_size = ( self.config.getint('config', 'CYCLONE_PLOTTER_LEGEND_FONT_SIZE') ) self.cross_marker = ( self.config.getint('config', 'CYCLONE_PLOTTER_CROSS_MARKER_SIZE') ) self.resolution_dpi = ( self.config.getint('config', 'CYCLONE_PLOTTER_RESOLUTION_DPI') ) self.add_watermark = self.config.getbool('config', 'CYCLONE_PLOTTER_ADD_WATERMARK', True) def run_all_times(self): """! Calls the defs needed to create the cyclone plots run_all_times() is required by CommandBuilder. """ self.sanitized_df = self.retrieve_data() self.create_plot() def retrieve_data(self): """! Retrieve data from track files. Returns: None """ self.logger.debug("Begin retrieving data...") all_tracks_list = [] # Store the data in the track list. if os.path.isdir(self.input_data): self.logger.debug("Get data from all files in the directory " + self.input_data) # Get the list of all files (full file path) in this directory all_input_files = util.get_files(self.input_data, ".*.tcst", self.logger) # read each file into pandas then concatenate them together df_list = [pd.read_csv(file, delim_whitespace=True) for file in all_input_files] combined = pd.concat(df_list, ignore_index=True) # if there are any NaN values in the ALAT, ALON, STORM_ID, LEAD, INIT, AMODEL, or VALID column, # drop that row of data (axis=0). We need all these columns to contain valid data in order # to create a meaningful plot. combined_df = combined.copy(deep=True) combined_df = combined.dropna(axis=0, how='any', subset=['ALAT', 'ALON', 'STORM_ID', 'LEAD', 'INIT', 'AMODEL', 'VALID']) # Retrieve and create the columns of interest self.logger.debug(f"Number of rows of data: {combined_df.shape[0]}") combined_subset = combined_df[self.columns_of_interest] df = combined_subset.copy(deep=True) df.allows_duplicate_labels = False df['INIT'] = df['INIT'].astype(str) df['INIT_YMD'] = (df['INIT'].str[:8]).astype(int) df['INIT_HOUR'] = (df['INIT'].str[9:11]).astype(int) df['LEAD'] = df['LEAD']/10000 df['LEAD'] = df['LEAD'].astype(int) df['VALID_DD'] = (df['VALID'].str[6:8]).astype(int) df['VALID_HOUR'] = (df['VALID'].str[9:11]).astype(int) df['VALID'] = df['VALID'].astype(int) # clean up dataframes that are no longer needed del combined del combined_df # Subset the dataframe to include only the data relevant to the user's criteria as # specified in the configuration file. init_date = int(self.init_date) init_hh = int(self.init_hr) model_name = self.model mask = df[(df['AMODEL'] == model_name) & (df['INIT_YMD'] >= init_date) & (df['INIT_HOUR'] >= init_hh)] user_criteria_df = mask # Aggregate the ALON values based on unique storm id in order to sanitize the longitude values # that cross the International Date Line. unique_storm_ids_set = set(user_criteria_df['STORM_ID']) self.unique_storm_id = unique_storm_ids_set self.unique_storm_ids = list(unique_storm_ids_set) nunique = len(self.unique_storm_ids) self.logger.debug(f" {nunique} unique storm ids identified") # Use named tuples to store the relevant storm track information (their index value in the dataframe, # track id, and ALON values and later on the SLON (sanitized ALON values). TrackPt = namedtuple("TrackPt", "indices track alons ") # named tuple holding "sanitized" longitudes SanTrackPt = namedtuple("SanTrackPt", "indices track alons slons") # Keep track of the unique storm tracks by their storm_id storm_track_dict = {} for cur_unique in self.unique_storm_ids: idx_list = user_criteria_df.index[user_criteria_df['STORM_ID'] == cur_unique].tolist() alons = [] indices = [] for idx in idx_list: alons.append(user_criteria_df.iloc[idx]['ALON']) indices.append(idx) # create the track_pt tuple and add it to the storm track dictionary track_pt = TrackPt(indices, cur_unique, alons) storm_track_dict[cur_unique] = track_pt # create a new dataframe to contain the sanitized lons (i.e. the original ALONs that have # been cleaned up when crossing the International Date Line) sanitized_df = user_criteria_df.copy(deep=True) # Now we have a dictionary that aggregates the data based on storm tracks (via storm id) # and will contain the "sanitized" lons sanitized_storm_tracks = {} for key in storm_track_dict: # sanitize the longitudes, create a new SanTrackPt named tuple and add that to a new dictionary # that keeps track of the sanitized data based on the storm id sanitized_lons = self.sanitize_lonlist(storm_track_dict[key].alons) sanitized_track_pt = SanTrackPt(storm_track_dict[key].indices, storm_track_dict[key].track, storm_track_dict[key].alons, sanitized_lons) sanitized_storm_tracks[key] = sanitized_track_pt # fill in the sanitized dataframe sanitized_df for key in sanitized_storm_tracks: # now use the indices of the storm tracks to correctly assign the sanitized # lons to the appropriate row in the dataframe to maintain the row ordering of # the original dataframe idx_list = sanitized_storm_tracks[key].indices for i, idx in enumerate(idx_list): sanitized_df.loc[idx,'SLON'] = sanitized_storm_tracks[key].slons[i] # Set some useful values used for plotting. # Set the IS_FIRST value to True if this is the first point in the storm track, False # otherwise if i == 0: sanitized_df.loc[idx, 'IS_FIRST'] = True else: sanitized_df.loc[idx, 'IS_FIRST'] = False # Set the lead group to the character '0' if the valid hour is 0 or 12, or to the # charcter '6' if the valid hour is 6 or 18. Set the marker to correspond to the # valid hour: 'o' (open circle) for 0 or 12 valid hour, or '+' (small plus/cross) # for 6 or 18. if sanitized_df.loc[idx, 'VALID_HOUR'] == 0 or sanitized_df.loc[idx, 'VALID_HOUR'] == 12: sanitized_df.loc[idx, 'LEAD_GROUP'] ='0' sanitized_df.loc[idx, 'MARKER'] = 'o' elif sanitized_df.loc[idx, 'VALID_HOUR'] == 6 or sanitized_df.loc[idx, 'VALID_HOUR'] == 18: sanitized_df.loc[idx, 'LEAD_GROUP'] = '6' sanitized_df.loc[idx, 'MARKER'] = '+' # Write output ASCII file (csv) summarizing the information extracted from the input # which will be used to generate the plot. if self.gen_ascii: ascii_track_parts = [self.init_date, '.csv'] ascii_track_output_name = ''.join(ascii_track_parts) sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) sanitized_df.to_csv(sanitized_df_filename) self.logger.info(f"Writing ascii track info as csv file: {sanitized_df_filename}") return sanitized_df def create_plot(self): """ Create the plot, using Cartopy """ # Use PlateCarree projection for now # use central meridian for central longitude cm_lon = 180 ax = plt.axes(projection=ccrs.PlateCarree(central_longitude=cm_lon)) prj = ccrs.PlateCarree() # for transforming the annotations (matplotlib to cartopy workaround from Stack Overflow) transform = ccrs.PlateCarree()._as_mpl_transform(ax) # Add land, coastlines, and ocean ax.add_feature(cfeature.LAND) ax.coastlines() ax.add_feature(cfeature.OCEAN) # keep map zoomed out to full world. If this # is absent, the default behavior is to zoom # into the portion of the map that contains points. ax.set_global() # Add grid lines for longitude and latitude ax.gridlines(draw_labels=False, xlocs=[180, -180]) gl = ax.gridlines(crs=prj, draw_labels=True, linewidth=1, color='gray', alpha=0.5, linestyle='--') gl.xlabels_top = False gl.ylabels_left = False gl.xlines = True gl.xformatter = LONGITUDE_FORMATTER gl.yformatter = LATITUDE_FORMATTER gl.xlabel_style = {'size': 9, 'color': 'blue'} gl.xlabel_style = {'color': 'black', 'weight': 'normal'} # Plot title plt.title(self.title + "\nFor forecast with initial time = " + self.init_date) # Optional: Create the NCAR watermark with a timestamp # This will appear in the bottom right corner of the plot, below # the x-axis. NOTE: The timestamp is in the user's local time zone # and not in UTC time. if self.add_watermark: ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d %H:%M:%S') watermark = 'DTC METplus\nplot created at: ' + st plt.text(60, -130, watermark, fontsize=5, alpha=0.25) # Make sure the output directory exists, and create it if it doesn't. util.mkdir_p(self.output_dir) # get the points for the scatter plots (and the relevant information for annotations, etc.) points_list = self.get_plot_points() # Legend labels lead_group_0_legend = "Indicates a position at 00 or 12 UTC" lead_group_6_legend = "Indicates a position at 06 or 18 UTC" # to be consistent with the NOAA website, use red for annotations, markers, and lines. pt_color = 'red' cross_marker_size = self.cross_marker circle_marker_size = self.circle_marker # Get all the lat and lon (i.e. x and y) points for the '+' and 'o' marker types # to be used in generating the scatter plots (one for the 0/12 hr and one for the 6/18 hr lead # groups). Also collect ALL the lons and lats, which will be used to generate the # line plot (the last plot that goes on top of all the scatter plots). cross_lons = [] cross_lats = [] cross_annotations = [] circle_lons = [] circle_lats = [] circle_annotations = [] for idx,pt in enumerate(points_list): if pt.marker == '+': cross_lons.append(pt.lon) cross_lats.append(pt.lat) cross_annotations.append(pt.annotation) cross_marker = pt.marker elif pt.marker == 'o': circle_lons.append(pt.lon) circle_lats.append(pt.lat) circle_annotations.append(pt.annotation) circle_marker = pt.marker # Now generate the scatter plots for the lead group 0/12 hr ('+' marker) and the # lead group 6/18 hr ('o' marker). plt.scatter(circle_lons, circle_lats, s=circle_marker_size, c=pt_color, marker=circle_marker, zorder=2, label=lead_group_0_legend, transform=prj) plt.scatter(cross_lons, cross_lats, s=cross_marker_size, c=pt_color, marker=cross_marker, zorder=2, label=lead_group_6_legend, transform=prj) # annotations for the scatter plots counter = 0 for x,y in zip(circle_lons, circle_lats): plt.annotate(circle_annotations[counter], (x,y+1), xycoords=transform, color=pt_color, fontsize=self.annotation_font_size) counter += 1 counter = 0 for x, y in zip(cross_lons, cross_lats): plt.annotate(cross_annotations[counter], (x, y + 1), xycoords=transform, color=pt_color, fontsize=self.annotation_font_size) counter += 1 # Dummy point to add the additional label explaining the labelling of the first # point in the storm track plt.scatter(0, 0, zorder=2, marker=None, c='', label="Date (dd/hhz) is the first " + "time storm was able to be tracked in model") # Settings for the legend box location. self.logger.debug(f"!!!!!legend font size: {self.legend_font_size}") ax.legend(loc='lower left', bbox_to_anchor=(0, -0.4), fancybox=True, shadow=True, scatterpoints=1, prop={'size':self.legend_font_size}) # Generate the line plot # First collect all the lats and lons for each storm track. Then for each storm track, # generate a line plot. pts_by_track_dict = self.get_points_by_track() for key in pts_by_track_dict: lons = [] lats = [] for idx, pt in enumerate(pts_by_track_dict[key]): lons.append(pt.lon) lats.append(pt.lat) # Create the line plot for this storm track plt.plot(lons, lats, linestyle='-', color=pt_color, linewidth=.5, transform=prj, zorder=3) plt.savefig("/Users/minnawin/Desktop/plot.png", dpi=800) def get_plot_points(self): """ Get the lon and lat points to be plotted, along with any other plotting-relevant information like the marker, whether this is a first point (to be used in annotating the first point using the valid day date and valid hour), etc. :return: A list of named tuples that represent the points to plot with corresponding plotting information """ # Create a named tuple to store the point information PlotPt = namedtuple("PlotPt", "storm_id lon lat is_first marker valid_dd valid_hour annotation") points_list = [] storm_id = self.sanitized_df['STORM_ID'] lons = self.sanitized_df['SLON'] lats = self.sanitized_df['ALAT'] is_first_list = self.sanitized_df['IS_FIRST'] marker_list = self.sanitized_df['MARKER'] valid_dd_list = self.sanitized_df['VALID_DD'] valid_hour_list = self.sanitized_df['VALID_HOUR'] annotation_list = [] for idx, cur_lon in enumerate(lons): if is_first_list[idx] is True: annotation = str(valid_dd_list[idx]).zfill(2) + '/' + \ str(valid_hour_list[idx]).zfill(2) + 'z' else: annotation = None annotation_list.append(annotation) cur_pt = PlotPt(storm_id, lons[idx], lats[idx], is_first_list[idx], marker_list[idx], valid_dd_list[idx], valid_hour_list[idx], annotation) points_list.append(cur_pt) return points_list def get_points_by_track(self): """ Get all the lats and lons for each storm track. Used to generate the line plot of the storm tracks. Args: Returns: points_by_track: Points aggregated by storm track. Returns a dictionary: where the key is the storm_id and values are the points (lon,lat) stored in a named tuple """ track_dict = {} LonLat = namedtuple("LonLat", "lon lat") for cur_unique in self.unique_storm_ids: # retrieve the ALAT and ALON values that correspond to the rows for a unique storm id. # i.e. Get the index value(s) corresponding to this unique storm id idx_list = self.sanitized_df.index[self.sanitized_df['STORM_ID'] == cur_unique].tolist() sanitized_lons_and_lats = [] indices = [] for idx in idx_list: cur_lonlat = LonLat(self.sanitized_df.iloc[idx]['SLON'], self.sanitized_df.iloc[idx]['ALAT']) sanitized_lons_and_lats.append(cur_lonlat) indices.append(idx) # update the track dictionary track_dict[cur_unique] = sanitized_lons_and_lats return track_dict @staticmethod def sanitize_lonlist(lon): """ Solution from Stack Overflow for "sanitizing" longitudes that cross the International Date Line https://stackoverflow.com/questions/67730660/plotting-line-across-international-dateline-with-cartopy Args: @param lon: A list of longitudes (float) that correspond to a storm track Returns: new_list: a list of "sanitized" lons that are "corrected" for crossing the International Date Line """ new_list = [] oldval = 0 # used to compare adjacent longitudes in a storm track treshold = 10 for ix, ea in enumerate(lon): diff = oldval - ea if (ix > 0): if (diff > treshold): ea = ea + 360 oldval = ea new_list.append(ea) return new_list \ No newline at end of file +"""!@namespace ExtraTropicalCyclonePlotter +A Python class that generates plots of extra tropical cyclone forecast data, + replicating the NCEP tropical and extra tropical cyclone tracks and + verification plots http://www.emc.ncep.noaa.gov/mmb/gplou/emchurr/glblgen/ +""" + +import os +import time +import datetime +import re +import sys +from collections import namedtuple + +# handle if module can't be loaded to run wrapper +WRAPPER_CANNOT_RUN = False +EXCEPTION_ERR = '' +try: + import pandas as pd + import matplotlib.pyplot as plt + import matplotlib.ticker as mticker + import cartopy.crs as ccrs + import cartopy.feature as cfeature + import cartopy + from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER + + ##If the script is run on a limited-internet access machine, the CARTOPY_DIR environment setting + ##will need to be set in the user-specific system configuration file. Review the Installation section + ##of the User's Guide for more details. + if os.getenv('CARTOPY_DIR'): + cartopy.config['data_dir'] = os.getenv('CARTOPY_DIR', cartopy.config.get('data_dir')) + +except Exception as err_msg: + WRAPPER_CANNOT_RUN = True + EXCEPTION_ERR = err_msg + +import produtil.setup + +from ..util import met_util as util +from ..util import do_string_sub +from . import CommandBuilder + + +class CyclonePlotterWrapper(CommandBuilder): + """! Generate plots of extra tropical storm forecast tracks. + Reads input from ATCF files generated from MET TC-Pairs + """ + + def __init__(self, config, instance=None, config_overrides={}): + self.app_name = 'cyclone_plotter' + + super().__init__(config, + instance=instance, + config_overrides=config_overrides) + + if WRAPPER_CANNOT_RUN: + self.log_error("There was a problem importing modules: " + f"{EXCEPTION_ERR}\n") + return + + self.input_data = self.config.getdir('CYCLONE_PLOTTER_INPUT_DIR') + self.output_dir = self.config.getdir('CYCLONE_PLOTTER_OUTPUT_DIR') + self.init_date = self.config.getraw('config', + 'CYCLONE_PLOTTER_INIT_DATE') + self.init_hr = self.config.getraw('config', 'CYCLONE_PLOTTER_INIT_HR') + + init_time_fmt = self.config.getstr('config', 'INIT_TIME_FMT', '') + + if init_time_fmt: + clock_time = datetime.datetime.strptime( + self.config.getstr('config', + 'CLOCK_TIME'), + '%Y%m%d%H%M%S' + ) + + init_beg = self.config.getraw('config', 'INIT_BEG') + if init_beg: + init_beg_dt = util.get_time_obj(init_beg, + init_time_fmt, + clock_time, + logger=self.logger) + self.init_date = do_string_sub(self.init_date, init=init_beg_dt) + self.init_hr = do_string_sub(self.init_hr, init=init_beg_dt) + init_end = self.config.getraw('config', 'INIT_END') + if init_end: + init_end_dt = util.get_time_obj(init_end, + init_time_fmt, + clock_time, + logger=self.logger) + self.init_end_date = do_string_sub(self.init_date, init=init_end_dt) + self.init_end_hr = do_string_sub(self.init_hr, init=init_end_dt) + + self.model = self.config.getstr('config', 'CYCLONE_PLOTTER_MODEL') + self.title = self.config.getstr('config', + 'CYCLONE_PLOTTER_PLOT_TITLE') + self.gen_ascii = ( + self.config.getbool('config', + 'CYCLONE_PLOTTER_GENERATE_TRACK_ASCII') + ) + # Create a set to keep track of unique storm_ids for each track file. + self.unique_storm_id = set() + # Data structure to separate data based on storm id. + self.storm_id_dict = {} + + # Data/info which we want to retrieve from the track files. + self.columns_of_interest = ['AMODEL', 'STORM_ID', 'INIT', + 'LEAD', 'VALID', 'ALAT', 'ALON'] + self.circle_marker = ( + self.config.getint('config', + 'CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE') + ) + self.annotation_font_size = ( + self.config.getint('config', + 'CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE') + ) + + self.legend_font_size = ( + self.config.getint('config', + 'CYCLONE_PLOTTER_LEGEND_FONT_SIZE') + ) + + self.cross_marker = ( + self.config.getint('config', + 'CYCLONE_PLOTTER_CROSS_MARKER_SIZE') + ) + self.resolution_dpi = ( + self.config.getint('config', + 'CYCLONE_PLOTTER_RESOLUTION_DPI') + ) + + self.add_watermark = self.config.getbool('config', + 'CYCLONE_PLOTTER_ADD_WATERMARK', + True) + + def run_all_times(self): + """! Calls the defs needed to create the cyclone plots + run_all_times() is required by CommandBuilder. + + """ + self.sanitized_df = self.retrieve_data() + self.create_plot() + + def retrieve_data(self): + """! Retrieve data from track files. + Returns: + None + + """ + self.logger.debug("Begin retrieving data...") + all_tracks_list = [] + + # Store the data in the track list. + if os.path.isdir(self.input_data): + self.logger.debug("Get data from all files in the directory " + + self.input_data) + # Get the list of all files (full file path) in this directory + all_input_files = util.get_files(self.input_data, ".*.tcst", + self.logger) + + # read each file into pandas then concatenate them together + df_list = [pd.read_csv(file, delim_whitespace=True) for file in all_input_files] + combined = pd.concat(df_list, ignore_index=True) + + # if there are any NaN values in the ALAT, ALON, STORM_ID, LEAD, INIT, AMODEL, or VALID column, + # drop that row of data (axis=0). We need all these columns to contain valid data in order + # to create a meaningful plot. + combined_df = combined.copy(deep=True) + combined_df = combined.dropna(axis=0, how='any', + subset=['ALAT', 'ALON', 'STORM_ID', + 'LEAD', 'INIT', 'AMODEL', 'VALID']) + + # Retrieve and create the columns of interest + self.logger.debug(f"Number of rows of data: {combined_df.shape[0]}") + combined_subset = combined_df[self.columns_of_interest] + df = combined_subset.copy(deep=True) + df.allows_duplicate_labels = False + df['INIT'] = df['INIT'].astype(str) + df['INIT_YMD'] = (df['INIT'].str[:8]).astype(int) + df['INIT_HOUR'] = (df['INIT'].str[9:11]).astype(int) + df['LEAD'] = df['LEAD']/10000 + df['LEAD'] = df['LEAD'].astype(int) + df['VALID_DD'] = (df['VALID'].str[6:8]).astype(int) + df['VALID_HOUR'] = (df['VALID'].str[9:11]).astype(int) + df['VALID'] = df['VALID'].astype(int) + + # clean up dataframes that are no longer needed + del combined + del combined_df + + # Subset the dataframe to include only the data relevant to the user's criteria as + # specified in the configuration file. + init_date = int(self.init_date) + init_hh = int(self.init_hr) + model_name = self.model + + mask = df[(df['AMODEL'] == model_name) & (df['INIT_YMD'] >= init_date) & + (df['INIT_HOUR'] >= init_hh)] + user_criteria_df = mask + + # Aggregate the ALON values based on unique storm id in order to sanitize the longitude values + # that cross the International Date Line. + unique_storm_ids_set = set(user_criteria_df['STORM_ID']) + self.unique_storm_id = unique_storm_ids_set + self.unique_storm_ids = list(unique_storm_ids_set) + nunique = len(self.unique_storm_ids) + self.logger.debug(f" {nunique} unique storm ids identified") + + # Use named tuples to store the relevant storm track information (their index value in the dataframe, + # track id, and ALON values and later on the SLON (sanitized ALON values). + TrackPt = namedtuple("TrackPt", "indices track alons ") + + # named tuple holding "sanitized" longitudes + SanTrackPt = namedtuple("SanTrackPt", "indices track alons slons") + + # Keep track of the unique storm tracks by their storm_id + storm_track_dict = {} + + for cur_unique in self.unique_storm_ids: + idx_list = user_criteria_df.index[user_criteria_df['STORM_ID'] == cur_unique].tolist() + alons = [] + indices = [] + + for idx in idx_list: + alons.append(user_criteria_df.iloc[idx]['ALON']) + indices.append(idx) + + # create the track_pt tuple and add it to the storm track dictionary + track_pt = TrackPt(indices, cur_unique, alons) + storm_track_dict[cur_unique] = track_pt + + # create a new dataframe to contain the sanitized lons (i.e. the original ALONs that have + # been cleaned up when crossing the International Date Line) + sanitized_df = user_criteria_df.copy(deep=True) + + # Now we have a dictionary that aggregates the data based on storm tracks (via storm id) + # and will contain the "sanitized" lons + sanitized_storm_tracks = {} + for key in storm_track_dict: + # sanitize the longitudes, create a new SanTrackPt named tuple and add that to a new dictionary + # that keeps track of the sanitized data based on the storm id + sanitized_lons = self.sanitize_lonlist(storm_track_dict[key].alons) + sanitized_track_pt = SanTrackPt(storm_track_dict[key].indices, storm_track_dict[key].track, + storm_track_dict[key].alons, sanitized_lons) + sanitized_storm_tracks[key] = sanitized_track_pt + + # fill in the sanitized dataframe sanitized_df + for key in sanitized_storm_tracks: + # now use the indices of the storm tracks to correctly assign the sanitized + # lons to the appropriate row in the dataframe to maintain the row ordering of + # the original dataframe + idx_list = sanitized_storm_tracks[key].indices + + for i, idx in enumerate(idx_list): + sanitized_df.loc[idx,'SLON'] = sanitized_storm_tracks[key].slons[i] + + # Set some useful values used for plotting. + # Set the IS_FIRST value to True if this is the first point in the storm track, False + # otherwise + if i == 0: + sanitized_df.loc[idx, 'IS_FIRST'] = True + else: + sanitized_df.loc[idx, 'IS_FIRST'] = False + + # Set the lead group to the character '0' if the valid hour is 0 or 12, or to the + # charcter '6' if the valid hour is 6 or 18. Set the marker to correspond to the + # valid hour: 'o' (open circle) for 0 or 12 valid hour, or '+' (small plus/cross) + # for 6 or 18. + if sanitized_df.loc[idx, 'VALID_HOUR'] == 0 or sanitized_df.loc[idx, 'VALID_HOUR'] == 12: + sanitized_df.loc[idx, 'LEAD_GROUP'] ='0' + sanitized_df.loc[idx, 'MARKER'] = 'o' + elif sanitized_df.loc[idx, 'VALID_HOUR'] == 6 or sanitized_df.loc[idx, 'VALID_HOUR'] == 18: + sanitized_df.loc[idx, 'LEAD_GROUP'] = '6' + sanitized_df.loc[idx, 'MARKER'] = '+' + + # Write output ASCII file (csv) summarizing the information extracted from the input + # which will be used to generate the plot. + if self.gen_ascii: + ascii_track_parts = [self.init_date, '.csv'] + ascii_track_output_name = ''.join(ascii_track_parts) + sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) + sanitized_df.to_csv(sanitized_df_filename) + self.logger.info(f"Writing ascii track info as csv file: {sanitized_df_filename}") + + return sanitized_df + + def create_plot(self): + """ + Create the plot, using Cartopy + + """ + + # Use PlateCarree projection for now + # use central meridian for central longitude + cm_lon = 180 + ax = plt.axes(projection=ccrs.PlateCarree(central_longitude=cm_lon)) + prj = ccrs.PlateCarree() + # for transforming the annotations (matplotlib to cartopy workaround from Stack Overflow) + transform = ccrs.PlateCarree()._as_mpl_transform(ax) + + # Add land, coastlines, and ocean + ax.add_feature(cfeature.LAND) + ax.coastlines() + ax.add_feature(cfeature.OCEAN) + + # keep map zoomed out to full world. If this + # is absent, the default behavior is to zoom + # into the portion of the map that contains points. + ax.set_global() + + # Add grid lines for longitude and latitude + ax.gridlines(draw_labels=False, xlocs=[180, -180]) + gl = ax.gridlines(crs=prj, + draw_labels=True, linewidth=1, color='gray', + alpha=0.5, linestyle='--') + gl.xlabels_top = False + gl.ylabels_left = False + gl.xlines = True + gl.xformatter = LONGITUDE_FORMATTER + gl.yformatter = LATITUDE_FORMATTER + gl.xlabel_style = {'size': 9, 'color': 'blue'} + gl.xlabel_style = {'color': 'black', 'weight': 'normal'} + + # Plot title + plt.title(self.title + "\nFor forecast with initial time = " + + self.init_date) + + # Optional: Create the NCAR watermark with a timestamp + # This will appear in the bottom right corner of the plot, below + # the x-axis. NOTE: The timestamp is in the user's local time zone + # and not in UTC time. + if self.add_watermark: + ts = time.time() + st = datetime.datetime.fromtimestamp(ts).strftime( + '%Y-%m-%d %H:%M:%S') + watermark = 'DTC METplus\nplot created at: ' + st + plt.text(60, -130, watermark, fontsize=5, alpha=0.25) + + # Make sure the output directory exists, and create it if it doesn't. + util.mkdir_p(self.output_dir) + + + # get the points for the scatter plots (and the relevant information for annotations, etc.) + points_list = self.get_plot_points() + + # Legend labels + lead_group_0_legend = "Indicates a position at 00 or 12 UTC" + lead_group_6_legend = "Indicates a position at 06 or 18 UTC" + + # to be consistent with the NOAA website, use red for annotations, markers, and lines. + pt_color = 'red' + cross_marker_size = self.cross_marker + circle_marker_size = self.circle_marker + + # Get all the lat and lon (i.e. x and y) points for the '+' and 'o' marker types + # to be used in generating the scatter plots (one for the 0/12 hr and one for the 6/18 hr lead + # groups). Also collect ALL the lons and lats, which will be used to generate the + # line plot (the last plot that goes on top of all the scatter plots). + cross_lons = [] + cross_lats = [] + cross_annotations = [] + circle_lons = [] + circle_lats = [] + circle_annotations = [] + + for idx,pt in enumerate(points_list): + if pt.marker == '+': + cross_lons.append(pt.lon) + cross_lats.append(pt.lat) + cross_annotations.append(pt.annotation) + cross_marker = pt.marker + elif pt.marker == 'o': + circle_lons.append(pt.lon) + circle_lats.append(pt.lat) + circle_annotations.append(pt.annotation) + circle_marker = pt.marker + + # Now generate the scatter plots for the lead group 0/12 hr ('+' marker) and the + # lead group 6/18 hr ('o' marker). + plt.scatter(circle_lons, circle_lats, s=circle_marker_size, c=pt_color, + marker=circle_marker, zorder=2, label=lead_group_0_legend, transform=prj) + plt.scatter(cross_lons, cross_lats, s=cross_marker_size, c=pt_color, + marker=cross_marker, zorder=2, label=lead_group_6_legend, transform=prj) + + # annotations for the scatter plots + counter = 0 + for x,y in zip(circle_lons, circle_lats): + plt.annotate(circle_annotations[counter], (x,y+1), xycoords=transform, color=pt_color, + fontsize=self.annotation_font_size) + counter += 1 + + counter = 0 + for x, y in zip(cross_lons, cross_lats): + plt.annotate(cross_annotations[counter], (x, y + 1), xycoords=transform, color=pt_color, + fontsize=self.annotation_font_size) + counter += 1 + + # Dummy point to add the additional label explaining the labelling of the first + # point in the storm track + plt.scatter(0, 0, zorder=2, marker=None, c='', + label="Date (dd/hhz) is the first " + + "time storm was able to be tracked in model") + + # Settings for the legend box location. + self.logger.debug(f"!!!!!legend font size: {self.legend_font_size}") + ax.legend(loc='lower left', bbox_to_anchor=(0, -0.4), + fancybox=True, shadow=True, scatterpoints=1, + prop={'size':self.legend_font_size}) + + + # Generate the line plot + # First collect all the lats and lons for each storm track. Then for each storm track, + # generate a line plot. + pts_by_track_dict = self.get_points_by_track() + + for key in pts_by_track_dict: + lons = [] + lats = [] + for idx, pt in enumerate(pts_by_track_dict[key]): + lons.append(pt.lon) + lats.append(pt.lat) + + # Create the line plot for this storm track + plt.plot(lons, lats, linestyle='-', color=pt_color, linewidth=.5, transform=prj, zorder=3) + + plt.savefig("/Users/minnawin/Desktop/plot.png", dpi=800) + + + + def get_plot_points(self): + """ + Get the lon and lat points to be plotted, along with any other plotting-relevant + information like the marker, whether this is a first point (to be used in + annotating the first point using the valid day date and valid hour), etc. + + :return: A list of named tuples that represent the points to plot with corresponding + plotting information + """ + + # Create a named tuple to store the point information + PlotPt = namedtuple("PlotPt", "storm_id lon lat is_first marker valid_dd valid_hour annotation") + + points_list = [] + storm_id = self.sanitized_df['STORM_ID'] + lons = self.sanitized_df['SLON'] + lats = self.sanitized_df['ALAT'] + is_first_list = self.sanitized_df['IS_FIRST'] + marker_list = self.sanitized_df['MARKER'] + valid_dd_list = self.sanitized_df['VALID_DD'] + valid_hour_list = self.sanitized_df['VALID_HOUR'] + annotation_list = [] + + for idx, cur_lon in enumerate(lons): + if is_first_list[idx] is True: + annotation = str(valid_dd_list[idx]).zfill(2) + '/' + \ + str(valid_hour_list[idx]).zfill(2) + 'z' + else: + annotation = None + + annotation_list.append(annotation) + cur_pt = PlotPt(storm_id, lons[idx], lats[idx], is_first_list[idx], marker_list[idx], + valid_dd_list[idx], valid_hour_list[idx], annotation) + points_list.append(cur_pt) + + return points_list + + def get_points_by_track(self): + """ + Get all the lats and lons for each storm track. Used to generate the line + plot of the storm tracks. + + Args: + + Returns: + points_by_track: Points aggregated by storm track. + Returns a dictionary: where the key is the storm_id + and values are the points (lon,lat) stored in a named tuple + """ + track_dict = {} + LonLat = namedtuple("LonLat", "lon lat") + + for cur_unique in self.unique_storm_ids: + # retrieve the ALAT and ALON values that correspond to the rows for a unique storm id. + # i.e. Get the index value(s) corresponding to this unique storm id + idx_list = self.sanitized_df.index[self.sanitized_df['STORM_ID'] == cur_unique].tolist() + + sanitized_lons_and_lats = [] + indices = [] + for idx in idx_list: + cur_lonlat = LonLat(self.sanitized_df.iloc[idx]['SLON'], self.sanitized_df.iloc[idx]['ALAT']) + sanitized_lons_and_lats.append(cur_lonlat) + indices.append(idx) + + # update the track dictionary + track_dict[cur_unique] = sanitized_lons_and_lats + + + return track_dict + + + @staticmethod + def sanitize_lonlist(lon): + """ + Solution from Stack Overflow for "sanitizing" longitudes that cross the International Date Line + https://stackoverflow.com/questions/67730660/plotting-line-across-international-dateline-with-cartopy + + Args: + @param lon: A list of longitudes (float) that correspond to a storm track + + Returns: + new_list: a list of "sanitized" lons that are "corrected" for crossing the + International Date Line + """ + + new_list = [] + oldval = 0 + # used to compare adjacent longitudes in a storm track + treshold = 10 + for ix, ea in enumerate(lon): + diff = oldval - ea + if (ix > 0): + if (diff > treshold): + ea = ea + 360 + oldval = ea + new_list.append(ea) + return new_list From 807f8880d417f4e248016ba23a4211fda7dc7997 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 23 Sep 2021 18:23:13 -0600 Subject: [PATCH 08/23] Add correction to retrieve_data() to create directory where ASCII csv file will be written --- metplus/wrappers/cyclone_plotter_wrapper.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 74690f11b1..6844f7f9a9 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -142,7 +142,10 @@ def run_all_times(self): def retrieve_data(self): """! Retrieve data from track files. Returns: - None + sanitized_df: a pandas dataframe containing the + "sanitized" longitudes and some markers and + lead group information needed for generating + scatter plots. """ self.logger.debug("Begin retrieving data...") @@ -274,9 +277,12 @@ def retrieve_data(self): # Write output ASCII file (csv) summarizing the information extracted from the input # which will be used to generate the plot. if self.gen_ascii: + self.logger.debug(f" output dir: {self.output_dir}") + util.mkdir_p(self.output_dir) ascii_track_parts = [self.init_date, '.csv'] ascii_track_output_name = ''.join(ascii_track_parts) sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) + sanitized_df.to_csv(sanitized_df_filename) self.logger.info(f"Writing ascii track info as csv file: {sanitized_df_filename}") From 9722076483770eb8d276447a249194931b1dfd50 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 23 Sep 2021 19:07:54 -0600 Subject: [PATCH 09/23] Clean up logic that writes the csv output file. --- metplus/wrappers/cyclone_plotter_wrapper.py | 26 +++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 6844f7f9a9..61a82b6294 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -274,18 +274,20 @@ def retrieve_data(self): sanitized_df.loc[idx, 'LEAD_GROUP'] = '6' sanitized_df.loc[idx, 'MARKER'] = '+' - # Write output ASCII file (csv) summarizing the information extracted from the input - # which will be used to generate the plot. - if self.gen_ascii: - self.logger.debug(f" output dir: {self.output_dir}") - util.mkdir_p(self.output_dir) - ascii_track_parts = [self.init_date, '.csv'] - ascii_track_output_name = ''.join(ascii_track_parts) - sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) - - sanitized_df.to_csv(sanitized_df_filename) - self.logger.info(f"Writing ascii track info as csv file: {sanitized_df_filename}") - + # Write output ASCII file (csv) summarizing the information extracted from the input + # which will be used to generate the plot. + if self.gen_ascii: + self.logger.debug(f" output dir: {self.output_dir}") + util.mkdir_p(self.output_dir) + ascii_track_parts = [self.init_date, '.csv'] + ascii_track_output_name = ''.join(ascii_track_parts) + sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) + + sanitized_df.to_csv(sanitized_df_filename) + self.logger.info(f"Writing ascii track info as csv file: {sanitized_df_filename}") + else: + self.logger.error("CYCLONE_PLOTTER_INPUT_DIR isn't a valid directory, check config file.") + sys.exit("CYCLONE_PLOTTER_INPUT_DIR isn't a valid directory.") return sanitized_df def create_plot(self): From d8ab16876b7636502ba339b0f16e9a8976ef7851 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 24 Sep 2021 09:19:24 -0600 Subject: [PATCH 10/23] Reinstate saving the png file (final plot), initial checkin had hard-coded filename and location. --- metplus/wrappers/cyclone_plotter_wrapper.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 61a82b6294..6cf340a6f2 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -429,8 +429,15 @@ def create_plot(self): # Create the line plot for this storm track plt.plot(lons, lats, linestyle='-', color=pt_color, linewidth=.5, transform=prj, zorder=3) - plt.savefig("/Users/minnawin/Desktop/plot.png", dpi=800) - + # Write the plot to the output directory + out_filename_parts = [self.init_date, '.png'] + output_plot_name = ''.join(out_filename_parts) + plot_filename = os.path.join(self.output_dir, output_plot_name) + if self.resolution_dpi > 0: + plt.savefig(plot_filename, dpi=self.resolution_dpi) + else: + # use Matplotlib's default if no resolution is set in config file + plt.savefig(plot_filename) def get_plot_points(self): From 3e7df9900797db3d55cfb41dd95fd168080c9184 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 24 Sep 2021 14:32:30 -0600 Subject: [PATCH 11/23] Add support for when no CYCLONE_PLOTTER_MODEL value is set and reindex the subsetted dataframe to avoid array indexing issues. --- metplus/wrappers/cyclone_plotter_wrapper.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 6cf340a6f2..126bafbebf 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -195,9 +195,19 @@ def retrieve_data(self): init_hh = int(self.init_hr) model_name = self.model - mask = df[(df['AMODEL'] == model_name) & (df['INIT_YMD'] >= init_date) & - (df['INIT_HOUR'] >= init_hh)] + if model_name: + self.logger.debug("Subsetting based on ", init_date, ", ", init_date, + ", and model:", model_name ) + mask = df[(df['AMODEL'] == model_name) & (df['INIT_YMD'] >= init_date) & + (df['INIT_HOUR'] >= init_hh)] + else: + mask = df[(df['INIT_YMD'] >= init_date) & + (df['INIT_HOUR'] >= init_hh)] + self.logger.debug("Subsetting based on ", init_date, ", and ", init_date) + user_criteria_df = mask + # reset the index so things are ordered properly in the new dataframe + user_criteria_df.reset_index(inplace=True) # Aggregate the ALON values based on unique storm id in order to sanitize the longitude values # that cross the International Date Line. @@ -230,6 +240,7 @@ def retrieve_data(self): track_pt = TrackPt(indices, cur_unique, alons) storm_track_dict[cur_unique] = track_pt + # create a new dataframe to contain the sanitized lons (i.e. the original ALONs that have # been cleaned up when crossing the International Date Line) sanitized_df = user_criteria_df.copy(deep=True) From 7b86c5fadb472e75a115e7223238e75a0ce7886f Mon Sep 17 00:00:00 2001 From: Minna Win Date: Fri, 24 Sep 2021 14:36:06 -0600 Subject: [PATCH 12/23] Add configuration setting for LOOP_BY --- .../tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf b/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf index 99cc165b09..73218c47ed 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf @@ -28,6 +28,7 @@ TC_PAIRS_OUTPUT_TEMPLATE = {date?fmt=%Y%m}/{basin?fmt=%s}q{date?fmt=%Y%m%d%H}.gf PROCESS_LIST = TCPairs, CyclonePlotter LOOP_ORDER = processes +LOOP_BY = init ## Config options below used by tc_pairs_wrapper module. # ------------------------------------------------------- From 26b79085a8c892dc2c1a87c628c3be3ce5cb412e Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 24 Sep 2021 15:19:21 -0600 Subject: [PATCH 13/23] Added CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE and modified size of the marker sizes to smaller values. --- .../Plotter_fcstGFS_obsGFS_ExtraTC.conf | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf b/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf index 99cc165b09..12d6874624 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf @@ -28,6 +28,7 @@ TC_PAIRS_OUTPUT_TEMPLATE = {date?fmt=%Y%m}/{basin?fmt=%s}q{date?fmt=%Y%m%d%H}.gf PROCESS_LIST = TCPairs, CyclonePlotter LOOP_ORDER = processes +LOOP_BY = init ## Config options below used by tc_pairs_wrapper module. # ------------------------------------------------------- @@ -62,7 +63,6 @@ INIT_EXCLUDE = # Specify model valid time window in format YYYYMM[DD[_hh]]. # Only tracks that fall within the valid time window will # be used. -# VALID_BEG = VALID_END = @@ -135,8 +135,14 @@ CYCLONE_PLOTTER_PLOT_TITLE = Model Forecast Storm Tracks ## # Indicate the size of symbol (point size) -CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE = 41 -CYCLONE_PLOTTER_CROSS_MARKER_SIZE = 51 +# CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE = 41 +# CYCLONE_PLOTTER_CROSS_MARKER_SIZE = 51 +CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE = 2 +CYCLONE_PLOTTER_CROSS_MARKER_SIZE = 3 + +## +# Indicate text size of annotation label +CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE = 3 ## # Turn on/off the generation of an ASCII output file listing all the From 9650b5b4fac313a8e9099c5eeaeb4b5c01e00285 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 24 Sep 2021 18:10:56 -0600 Subject: [PATCH 14/23] Updated config file with resolution setting, annotation size setting, and watermark on/off. --- .../tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf b/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf index 12d6874624..d16593ceef 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf @@ -150,3 +150,12 @@ CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE = 3 # that what is plotted is consistent with the data. # CYCLONE_PLOTTER_GENERATE_TRACK_ASCII = yes + +## +# Resolution of saved plot in dpi (dots per inch) +# Set to 0 to allow Matplotlib to determine, based on your computer +CYCLONE_PLOTTER_RESOLUTION_DPI = 400 + +CYCLONE_PLOTTER_GENERATE_TRACK_ASCII = yes + +CYCLONE_PLOTTER_ADD_WATERMARK = False From 98495830ab56eb2edcad0f8eca32c6d5c0d03cda Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 24 Sep 2021 18:13:02 -0600 Subject: [PATCH 15/23] Remove unneccessary debug logging message --- metplus/wrappers/cyclone_plotter_wrapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 126bafbebf..c5725bb181 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -419,7 +419,6 @@ def create_plot(self): "time storm was able to be tracked in model") # Settings for the legend box location. - self.logger.debug(f"!!!!!legend font size: {self.legend_font_size}") ax.legend(loc='lower left', bbox_to_anchor=(0, -0.4), fancybox=True, shadow=True, scatterpoints=1, prop={'size':self.legend_font_size}) From a31b7a88853e4f3c14130d302651ee7e2dda9a25 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Tue, 5 Oct 2021 17:52:17 -0600 Subject: [PATCH 16/23] Github Issue #1000 added legend label fontsize and central longitude to configuration --- .../tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf b/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf index f86d90b6d2..4d9af0e629 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf @@ -15,6 +15,7 @@ PROCESS_LIST = UserScript, TCPairs, CyclonePlotter LOOP_BY = INIT + # The init time begin and end times, increment, and last init hour. INIT_TIME_FMT = %Y%m%d%H INIT_BEG = 2020100700 @@ -115,6 +116,8 @@ CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE = 3 # Indicate the text size for the legend labels CYCLONE_PLOTTER_LEGEND_FONT_SIZE = 3 +# Indicate the central latitude (i.e. the center of the map) +CYCLONE_PLOTTER_CENTRAL_LATITUDE = 180 ## # Resolution of saved plot in dpi (dots per inch) # Set to 0 to allow Matplotlib to determine, based on your computer From 567ddd2902f3e44de1cd5fbdc227f27f5d2899bb Mon Sep 17 00:00:00 2001 From: bikegeek Date: Tue, 5 Oct 2021 18:09:36 -0600 Subject: [PATCH 17/23] Github Issue #1000 fix line plot (now use the Geodesic coordinate reference system), remove unneccessary code and comments, add support to read the central longitude from the config file. --- metplus/wrappers/cyclone_plotter_wrapper.py | 56 ++++++++++----------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index c5725bb181..f555608fc5 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -80,14 +80,6 @@ def __init__(self, config, instance=None, config_overrides={}): logger=self.logger) self.init_date = do_string_sub(self.init_date, init=init_beg_dt) self.init_hr = do_string_sub(self.init_hr, init=init_beg_dt) - init_end = self.config.getraw('config', 'INIT_END') - if init_end: - init_end_dt = util.get_time_obj(init_end, - init_time_fmt, - clock_time, - logger=self.logger) - self.init_end_date = do_string_sub(self.init_date, init=init_end_dt) - self.init_end_hr = do_string_sub(self.init_hr, init=init_end_dt) self.model = self.config.getstr('config', 'CYCLONE_PLOTTER_MODEL') self.title = self.config.getstr('config', @@ -118,6 +110,11 @@ def __init__(self, config, instance=None, config_overrides={}): 'CYCLONE_PLOTTER_LEGEND_FONT_SIZE') ) + self.central_latitude = ( + self.config.getint('config', + 'CYCLONE_PLOTTER_CENTRAL_LATITUDE') + ) + self.cross_marker = ( self.config.getint('config', 'CYCLONE_PLOTTER_CROSS_MARKER_SIZE') @@ -168,8 +165,7 @@ def retrieve_data(self): # to create a meaningful plot. combined_df = combined.copy(deep=True) combined_df = combined.dropna(axis=0, how='any', - subset=['ALAT', 'ALON', 'STORM_ID', - 'LEAD', 'INIT', 'AMODEL', 'VALID']) + subset=self.columns_of_interest) # Retrieve and create the columns of interest self.logger.debug(f"Number of rows of data: {combined_df.shape[0]}") @@ -185,10 +181,6 @@ def retrieve_data(self): df['VALID_HOUR'] = (df['VALID'].str[9:11]).astype(int) df['VALID'] = df['VALID'].astype(int) - # clean up dataframes that are no longer needed - del combined - del combined_df - # Subset the dataframe to include only the data relevant to the user's criteria as # specified in the configuration file. init_date = int(self.init_date) @@ -196,30 +188,30 @@ def retrieve_data(self): model_name = self.model if model_name: - self.logger.debug("Subsetting based on ", init_date, ", ", init_date, - ", and model:", model_name ) + self.logger.debug("Subsetting based on " + str(init_date) + " " + str(init_hh) + + ", and model:" + model_name ) mask = df[(df['AMODEL'] == model_name) & (df['INIT_YMD'] >= init_date) & (df['INIT_HOUR'] >= init_hh)] else: mask = df[(df['INIT_YMD'] >= init_date) & (df['INIT_HOUR'] >= init_hh)] - self.logger.debug("Subsetting based on ", init_date, ", and ", init_date) + self.logger.debug("Subsetting based on " + str(init_date) + ", and "+ str(init_hh)) user_criteria_df = mask # reset the index so things are ordered properly in the new dataframe user_criteria_df.reset_index(inplace=True) + # Aggregate the ALON values based on unique storm id in order to sanitize the longitude values # that cross the International Date Line. unique_storm_ids_set = set(user_criteria_df['STORM_ID']) - self.unique_storm_id = unique_storm_ids_set self.unique_storm_ids = list(unique_storm_ids_set) nunique = len(self.unique_storm_ids) self.logger.debug(f" {nunique} unique storm ids identified") # Use named tuples to store the relevant storm track information (their index value in the dataframe, # track id, and ALON values and later on the SLON (sanitized ALON values). - TrackPt = namedtuple("TrackPt", "indices track alons ") + TrackPt = namedtuple("TrackPt", "indices track alons alats") # named tuple holding "sanitized" longitudes SanTrackPt = namedtuple("SanTrackPt", "indices track alons slons") @@ -230,14 +222,16 @@ def retrieve_data(self): for cur_unique in self.unique_storm_ids: idx_list = user_criteria_df.index[user_criteria_df['STORM_ID'] == cur_unique].tolist() alons = [] + alats = [] indices = [] for idx in idx_list: - alons.append(user_criteria_df.iloc[idx]['ALON']) + alons.append(user_criteria_df.loc[idx, 'ALON']) + alats.append(user_criteria_df.loc[idx, 'ALAT']) indices.append(idx) # create the track_pt tuple and add it to the storm track dictionary - track_pt = TrackPt(indices, cur_unique, alons) + track_pt = TrackPt(indices, cur_unique, alons, alats) storm_track_dict[cur_unique] = track_pt @@ -249,8 +243,10 @@ def retrieve_data(self): # and will contain the "sanitized" lons sanitized_storm_tracks = {} for key in storm_track_dict: - # sanitize the longitudes, create a new SanTrackPt named tuple and add that to a new dictionary + # "Sanitize" the longitudes to shift the lons that cross the International Date Line. + # Create a new SanTrackPt named tuple and add that to a new dictionary # that keeps track of the sanitized data based on the storm id + # sanitized_lons = self.sanitize_lonlist(storm_track_dict[key].alons) sanitized_lons = self.sanitize_lonlist(storm_track_dict[key].alons) sanitized_track_pt = SanTrackPt(storm_track_dict[key].indices, storm_track_dict[key].track, storm_track_dict[key].alons, sanitized_lons) @@ -294,8 +290,8 @@ def retrieve_data(self): ascii_track_output_name = ''.join(ascii_track_parts) sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) - sanitized_df.to_csv(sanitized_df_filename) - self.logger.info(f"Writing ascii track info as csv file: {sanitized_df_filename}") + # Make sure that the dataframe is sorted by STORM_ID, INIT_YMD, INIT_HOUR, and LEAD + sanitized_df.sort_values(by=['STORM_ID', 'INIT_YMD', 'INIT_HOUR', 'LEAD'], inplace=True) else: self.logger.error("CYCLONE_PLOTTER_INPUT_DIR isn't a valid directory, check config file.") sys.exit("CYCLONE_PLOTTER_INPUT_DIR isn't a valid directory.") @@ -309,7 +305,8 @@ def create_plot(self): # Use PlateCarree projection for now # use central meridian for central longitude - cm_lon = 180 + # cm_lon = 180 is the "default" set in the config file + cm_lon = self.central_latitude ax = plt.axes(projection=ccrs.PlateCarree(central_longitude=cm_lon)) prj = ccrs.PlateCarree() # for transforming the annotations (matplotlib to cartopy workaround from Stack Overflow) @@ -436,8 +433,10 @@ def create_plot(self): lons.append(pt.lon) lats.append(pt.lat) - # Create the line plot for this storm track - plt.plot(lons, lats, linestyle='-', color=pt_color, linewidth=.5, transform=prj, zorder=3) + # Create the line plot for this storm track, use the Geodetic coordinate reference system + # to correctly connect adjacent points that have been sanitized and cross the + # International Date line or the Prime Meridian. + plt.plot(lons, lats, linestyle='-', color=pt_color, linewidth=.5, transform=ccrs.Geodetic(), zorder=3) # Write the plot to the output directory out_filename_parts = [self.init_date, '.png'] @@ -510,7 +509,7 @@ def get_points_by_track(self): sanitized_lons_and_lats = [] indices = [] for idx in idx_list: - cur_lonlat = LonLat(self.sanitized_df.iloc[idx]['SLON'], self.sanitized_df.iloc[idx]['ALAT']) + cur_lonlat = LonLat(self.sanitized_df.loc[idx, 'SLON'], self.sanitized_df.loc[idx, 'ALAT']) sanitized_lons_and_lats.append(cur_lonlat) indices.append(idx) @@ -520,7 +519,6 @@ def get_points_by_track(self): return track_dict - @staticmethod def sanitize_lonlist(lon): """ From a98718cfe51d91ed69d5125b75bcfc09fe83d494 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 7 Oct 2021 19:15:44 -0600 Subject: [PATCH 18/23] Remove the setting for the central longitude, this should ALWAYS be 180 to center the map over the Pacific Ocean --- .../CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf b/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf index b210887f4e..94a0773846 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf @@ -116,8 +116,6 @@ CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE = 3 # Indicate the text size for the legend labels CYCLONE_PLOTTER_LEGEND_FONT_SIZE = 3 -# Indicate the central latitude (i.e. the center of the map) -CYCLONE_PLOTTER_CENTRAL_LATITUDE = 180 ## # Resolution of saved plot in dpi (dots per inch) # Set to 0 to allow Matplotlib to determine, based on your computer From 54ce942f8f19d77228b54577f9f5894753ce37fe Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 7 Oct 2021 19:17:46 -0600 Subject: [PATCH 19/23] Change size of markers and remove setting for central longitude, this will always be set to 180 in the plotting script to center the map on the Pacific Ocean. --- .../Plotter_fcstGFS_obsGFS_ExtraTC.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf b/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf index d16593ceef..7734c929d9 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf @@ -135,8 +135,6 @@ CYCLONE_PLOTTER_PLOT_TITLE = Model Forecast Storm Tracks ## # Indicate the size of symbol (point size) -# CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE = 41 -# CYCLONE_PLOTTER_CROSS_MARKER_SIZE = 51 CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE = 2 CYCLONE_PLOTTER_CROSS_MARKER_SIZE = 3 @@ -144,6 +142,9 @@ CYCLONE_PLOTTER_CROSS_MARKER_SIZE = 3 # Indicate text size of annotation label CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE = 3 +# Indicate the text size for the legend labels +CYCLONE_PLOTTER_LEGEND_FONT_SIZE = 3 + ## # Turn on/off the generation of an ASCII output file listing all the # tracks that are in the plot. This can be helpful in debugging or verifying @@ -151,11 +152,10 @@ CYCLONE_PLOTTER_ANNOTATION_FONT_SIZE = 3 # CYCLONE_PLOTTER_GENERATE_TRACK_ASCII = yes +CYCLONE_PLOTTER_ADD_WATERMARK = False + ## # Resolution of saved plot in dpi (dots per inch) # Set to 0 to allow Matplotlib to determine, based on your computer CYCLONE_PLOTTER_RESOLUTION_DPI = 400 -CYCLONE_PLOTTER_GENERATE_TRACK_ASCII = yes - -CYCLONE_PLOTTER_ADD_WATERMARK = False From 5e55bc5d0fcdd50735bd007bc9d9d61c86ba267d Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 7 Oct 2021 19:20:19 -0600 Subject: [PATCH 20/23] Change class variable names to be more informative, add better commenting, cleaning up code, moving hard-coded values from the plotting method into initialization section. --- metplus/wrappers/cyclone_plotter_wrapper.py | 107 +++++++++++--------- 1 file changed, 60 insertions(+), 47 deletions(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index f555608fc5..9796123f1d 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -11,6 +11,7 @@ import sys from collections import namedtuple + # handle if module can't be loaded to run wrapper WRAPPER_CANNOT_RUN = False EXCEPTION_ERR = '' @@ -96,7 +97,7 @@ def __init__(self, config, instance=None, config_overrides={}): # Data/info which we want to retrieve from the track files. self.columns_of_interest = ['AMODEL', 'STORM_ID', 'INIT', 'LEAD', 'VALID', 'ALAT', 'ALON'] - self.circle_marker = ( + self.circle_marker_size = ( self.config.getint('config', 'CYCLONE_PLOTTER_CIRCLE_MARKER_SIZE') ) @@ -110,13 +111,10 @@ def __init__(self, config, instance=None, config_overrides={}): 'CYCLONE_PLOTTER_LEGEND_FONT_SIZE') ) - self.central_latitude = ( - self.config.getint('config', - 'CYCLONE_PLOTTER_CENTRAL_LATITUDE') - ) + # Map centered on Pacific Ocean + self.central_latitude = 180.0 - self.cross_marker = ( - self.config.getint('config', + self.cross_marker_size = (self.config.getint('config', 'CYCLONE_PLOTTER_CROSS_MARKER_SIZE') ) self.resolution_dpi = ( @@ -127,6 +125,12 @@ def __init__(self, config, instance=None, config_overrides={}): self.add_watermark = self.config.getbool('config', 'CYCLONE_PLOTTER_ADD_WATERMARK', True) + # Set the marker symbols, '+' for a small cross, '.' for a small circle. + # NOTE: originally, 'o' was used. 'o' is also a circle, but it creates a + # very large filled circle on the plots that appears too large. + self.cross_marker = '+' + self.circle_marker = '.' + def run_all_times(self): """! Calls the defs needed to create the cyclone plots @@ -136,11 +140,12 @@ def run_all_times(self): self.sanitized_df = self.retrieve_data() self.create_plot() + def retrieve_data(self): """! Retrieve data from track files. Returns: sanitized_df: a pandas dataframe containing the - "sanitized" longitudes and some markers and + "sanitized" longitudes, as well as some markers and lead group information needed for generating scatter plots. @@ -160,6 +165,11 @@ def retrieve_data(self): df_list = [pd.read_csv(file, delim_whitespace=True) for file in all_input_files] combined = pd.concat(df_list, ignore_index=True) + # check for empty dataframe, set error message and exit + if combined.empty: + self.logger.error("No data found in specified files. Please check your config file settings.") + sys.exit("No data found.") + # if there are any NaN values in the ALAT, ALON, STORM_ID, LEAD, INIT, AMODEL, or VALID column, # drop that row of data (axis=0). We need all these columns to contain valid data in order # to create a meaningful plot. @@ -172,6 +182,9 @@ def retrieve_data(self): combined_subset = combined_df[self.columns_of_interest] df = combined_subset.copy(deep=True) df.allows_duplicate_labels = False + # INIT, LEAD, VALID correspond to the column headers from the MET + # TC tool output. INIT_YMD, INIT_HOUR, VALID_DD, and VALID_HOUR are + # new columns (for a new dataframe) created from these MET columns. df['INIT'] = df['INIT'].astype(str) df['INIT_YMD'] = (df['INIT'].str[:8]).astype(int) df['INIT_HOUR'] = (df['INIT'].str[9:11]).astype(int) @@ -193,6 +206,7 @@ def retrieve_data(self): mask = df[(df['AMODEL'] == model_name) & (df['INIT_YMD'] >= init_date) & (df['INIT_HOUR'] >= init_hh)] else: + # no model specified, just subset on init date and init hour mask = df[(df['INIT_YMD'] >= init_date) & (df['INIT_HOUR'] >= init_hh)] self.logger.debug("Subsetting based on " + str(init_date) + ", and "+ str(init_hh)) @@ -201,9 +215,8 @@ def retrieve_data(self): # reset the index so things are ordered properly in the new dataframe user_criteria_df.reset_index(inplace=True) - - # Aggregate the ALON values based on unique storm id in order to sanitize the longitude values - # that cross the International Date Line. + # Aggregate the ALON values based on unique storm id to facilitate "sanitizing" the + # longitude values (to handle lons that cross the International Date Line). unique_storm_ids_set = set(user_criteria_df['STORM_ID']) self.unique_storm_ids = list(unique_storm_ids_set) nunique = len(self.unique_storm_ids) @@ -234,13 +247,12 @@ def retrieve_data(self): track_pt = TrackPt(indices, cur_unique, alons, alats) storm_track_dict[cur_unique] = track_pt - # create a new dataframe to contain the sanitized lons (i.e. the original ALONs that have # been cleaned up when crossing the International Date Line) sanitized_df = user_criteria_df.copy(deep=True) - # Now we have a dictionary that aggregates the data based on storm tracks (via storm id) - # and will contain the "sanitized" lons + # Now we have a dictionary that helps in aggregating the data based on + # storm tracks (via storm id) and will contain the "sanitized" lons sanitized_storm_tracks = {} for key in storm_track_dict: # "Sanitize" the longitudes to shift the lons that cross the International Date Line. @@ -252,7 +264,7 @@ def retrieve_data(self): storm_track_dict[key].alons, sanitized_lons) sanitized_storm_tracks[key] = sanitized_track_pt - # fill in the sanitized dataframe sanitized_df + # fill in the sanitized dataframe, sanitized_df for key in sanitized_storm_tracks: # now use the indices of the storm tracks to correctly assign the sanitized # lons to the appropriate row in the dataframe to maintain the row ordering of @@ -263,26 +275,27 @@ def retrieve_data(self): sanitized_df.loc[idx,'SLON'] = sanitized_storm_tracks[key].slons[i] # Set some useful values used for plotting. - # Set the IS_FIRST value to True if this is the first point in the storm track, False + # Set the IS_FIRST value to True if this is the first + # point in the storm track, False # otherwise if i == 0: sanitized_df.loc[idx, 'IS_FIRST'] = True else: sanitized_df.loc[idx, 'IS_FIRST'] = False - # Set the lead group to the character '0' if the valid hour is 0 or 12, or to the - # charcter '6' if the valid hour is 6 or 18. Set the marker to correspond to the - # valid hour: 'o' (open circle) for 0 or 12 valid hour, or '+' (small plus/cross) - # for 6 or 18. + # Set the lead group to the character '0' if the valid hour is 0 or 12, + # or to the charcter '6' if the valid hour is 6 or 18. Set the marker + # to correspond to the valid hour: 'o' (open circle) for 0 or 12 valid hour, + # or '+' (small plus/cross) for 6 or 18. if sanitized_df.loc[idx, 'VALID_HOUR'] == 0 or sanitized_df.loc[idx, 'VALID_HOUR'] == 12: sanitized_df.loc[idx, 'LEAD_GROUP'] ='0' - sanitized_df.loc[idx, 'MARKER'] = 'o' + sanitized_df.loc[idx, 'MARKER'] = self.circle_marker elif sanitized_df.loc[idx, 'VALID_HOUR'] == 6 or sanitized_df.loc[idx, 'VALID_HOUR'] == 18: sanitized_df.loc[idx, 'LEAD_GROUP'] = '6' - sanitized_df.loc[idx, 'MARKER'] = '+' + sanitized_df.loc[idx, 'MARKER'] = self.cross_marker # Write output ASCII file (csv) summarizing the information extracted from the input - # which will be used to generate the plot. + # which is used to generate the plot. if self.gen_ascii: self.logger.debug(f" output dir: {self.output_dir}") util.mkdir_p(self.output_dir) @@ -291,21 +304,24 @@ def retrieve_data(self): sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) # Make sure that the dataframe is sorted by STORM_ID, INIT_YMD, INIT_HOUR, and LEAD + # to ensure that the line plot is connecting the points in the correct order. sanitized_df.sort_values(by=['STORM_ID', 'INIT_YMD', 'INIT_HOUR', 'LEAD'], inplace=True) + sanitized_df.to_csv(sanitized_df_filename) else: + # The user's specified directory isn't valid, log the error and exit. self.logger.error("CYCLONE_PLOTTER_INPUT_DIR isn't a valid directory, check config file.") sys.exit("CYCLONE_PLOTTER_INPUT_DIR isn't a valid directory.") return sanitized_df + def create_plot(self): """ Create the plot, using Cartopy """ - # Use PlateCarree projection for now - # use central meridian for central longitude - # cm_lon = 180 is the "default" set in the config file + # Use PlateCarree projection for scatter plots + # and Geodetic projection for line plots. cm_lon = self.central_latitude ax = plt.axes(projection=ccrs.PlateCarree(central_longitude=cm_lon)) prj = ccrs.PlateCarree() @@ -353,7 +369,6 @@ def create_plot(self): # Make sure the output directory exists, and create it if it doesn't. util.mkdir_p(self.output_dir) - # get the points for the scatter plots (and the relevant information for annotations, etc.) points_list = self.get_plot_points() @@ -363,8 +378,8 @@ def create_plot(self): # to be consistent with the NOAA website, use red for annotations, markers, and lines. pt_color = 'red' - cross_marker_size = self.cross_marker - circle_marker_size = self.circle_marker + cross_marker_size = self.cross_marker_size + circle_marker_size = self.circle_marker_size # Get all the lat and lon (i.e. x and y) points for the '+' and 'o' marker types # to be used in generating the scatter plots (one for the 0/12 hr and one for the 6/18 hr lead @@ -378,23 +393,23 @@ def create_plot(self): circle_annotations = [] for idx,pt in enumerate(points_list): - if pt.marker == '+': + if pt.marker == self.cross_marker: cross_lons.append(pt.lon) cross_lats.append(pt.lat) cross_annotations.append(pt.annotation) - cross_marker = pt.marker - elif pt.marker == 'o': + # cross_marker = pt.marker + elif pt.marker == self.circle_marker: circle_lons.append(pt.lon) circle_lats.append(pt.lat) circle_annotations.append(pt.annotation) - circle_marker = pt.marker + # circle_marker = pt.marker # Now generate the scatter plots for the lead group 0/12 hr ('+' marker) and the - # lead group 6/18 hr ('o' marker). - plt.scatter(circle_lons, circle_lats, s=circle_marker_size, c=pt_color, - marker=circle_marker, zorder=2, label=lead_group_0_legend, transform=prj) - plt.scatter(cross_lons, cross_lats, s=cross_marker_size, c=pt_color, - marker=cross_marker, zorder=2, label=lead_group_6_legend, transform=prj) + # lead group 6/18 hr ('.' marker). + plt.scatter(circle_lons, circle_lats, s=self.circle_marker_size, c=pt_color, + marker=self.circle_marker, zorder=2, label=lead_group_0_legend, transform=prj ) + plt.scatter(cross_lons, cross_lats, s=self.cross_marker_size, c=pt_color, + marker=self.cross_marker, zorder=2, label=lead_group_6_legend, transform=prj) # annotations for the scatter plots counter = 0 @@ -420,7 +435,6 @@ def create_plot(self): fancybox=True, shadow=True, scatterpoints=1, prop={'size':self.legend_font_size}) - # Generate the line plot # First collect all the lats and lons for each storm track. Then for each storm track, # generate a line plot. @@ -433,10 +447,10 @@ def create_plot(self): lons.append(pt.lon) lats.append(pt.lat) - # Create the line plot for this storm track, use the Geodetic coordinate reference system + # Create the line plot for the current storm track, use the Geodetic coordinate reference system # to correctly connect adjacent points that have been sanitized and cross the # International Date line or the Prime Meridian. - plt.plot(lons, lats, linestyle='-', color=pt_color, linewidth=.5, transform=ccrs.Geodetic(), zorder=3) + plt.plot(lons, lats, linestyle='-', color=pt_color, linewidth=.4, transform=ccrs.Geodetic(), zorder=3) # Write the plot to the output directory out_filename_parts = [self.init_date, '.png'] @@ -486,6 +500,7 @@ def get_plot_points(self): return points_list + def get_points_by_track(self): """ Get all the lats and lons for each storm track. Used to generate the line @@ -493,19 +508,17 @@ def get_points_by_track(self): Args: - Returns: + :return: points_by_track: Points aggregated by storm track. - Returns a dictionary: where the key is the storm_id + Returns a dictionary where the key is the storm_id and values are the points (lon,lat) stored in a named tuple """ track_dict = {} LonLat = namedtuple("LonLat", "lon lat") - for cur_unique in self.unique_storm_ids: # retrieve the ALAT and ALON values that correspond to the rows for a unique storm id. # i.e. Get the index value(s) corresponding to this unique storm id idx_list = self.sanitized_df.index[self.sanitized_df['STORM_ID'] == cur_unique].tolist() - sanitized_lons_and_lats = [] indices = [] for idx in idx_list: @@ -516,9 +529,9 @@ def get_points_by_track(self): # update the track dictionary track_dict[cur_unique] = sanitized_lons_and_lats - return track_dict + @staticmethod def sanitize_lonlist(lon): """ @@ -544,4 +557,4 @@ def sanitize_lonlist(lon): ea = ea + 360 oldval = ea new_list.append(ea) - return new_list + return new_list \ No newline at end of file From 57429710a310cf5901739e4ae13bad2454df499f Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 14 Oct 2021 17:15:49 -0600 Subject: [PATCH 21/23] Github issue #1000, using the suggested gridlines command from https://github.com/SciTools/cartopy/issues/1401 and removing the original command of ax.gridlines(draw_labels=False, xlocs=[180, -180]) to prevent overlapping of 180E and 180W for central_longitude=180. Also fixed the sorting so the final ASCII (.csv) representation of the final data frame has been reindexed. --- metplus/wrappers/cyclone_plotter_wrapper.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 9796123f1d..82a16fa9fc 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -301,17 +301,20 @@ def retrieve_data(self): util.mkdir_p(self.output_dir) ascii_track_parts = [self.init_date, '.csv'] ascii_track_output_name = ''.join(ascii_track_parts) - sanitized_df_filename = os.path.join(self.output_dir, ascii_track_output_name) + final_df_filename = os.path.join(self.output_dir, ascii_track_output_name) # Make sure that the dataframe is sorted by STORM_ID, INIT_YMD, INIT_HOUR, and LEAD # to ensure that the line plot is connecting the points in the correct order. - sanitized_df.sort_values(by=['STORM_ID', 'INIT_YMD', 'INIT_HOUR', 'LEAD'], inplace=True) - sanitized_df.to_csv(sanitized_df_filename) + # sanitized_df.sort_values(by=['STORM_ID', 'INIT_YMD', 'INIT_HOUR', 'LEAD'], + # inplace=True).reset_index(drop=True,inplace=True) + # sanitized_df.reset_index(inplace=True,drop=True) + final_df = sanitized_df.sort_values(by=['STORM_ID', 'INIT_YMD', 'INIT_HOUR', 'LEAD'], ignore_index=True) + final_df.to_csv(final_df_filename) else: # The user's specified directory isn't valid, log the error and exit. self.logger.error("CYCLONE_PLOTTER_INPUT_DIR isn't a valid directory, check config file.") sys.exit("CYCLONE_PLOTTER_INPUT_DIR isn't a valid directory.") - return sanitized_df + return final_df def create_plot(self): @@ -339,7 +342,6 @@ def create_plot(self): ax.set_global() # Add grid lines for longitude and latitude - ax.gridlines(draw_labels=False, xlocs=[180, -180]) gl = ax.gridlines(crs=prj, draw_labels=True, linewidth=1, color='gray', alpha=0.5, linestyle='--') From eaf73c5c769277760951ea266278f62bf510758d Mon Sep 17 00:00:00 2001 From: bikegeek <3753118+bikegeek@users.noreply.github.com> Date: Fri, 15 Oct 2021 18:21:52 -0600 Subject: [PATCH 22/23] Update use_case_groups.json removed extra entry for met_tools to test the cycloneplotter wrapper earlier in the process. --- .github/parm/use_case_groups.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index a65a7bb075..ef1af905df 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -1,9 +1,4 @@ [ - { - "category": "met_tool_wrapper", - "index_list": "0", - "run": true - }, { "category": "met_tool_wrapper", "index_list": "0-57", From 15402905d5ed4c3fa17a58daefb385901fb8da43 Mon Sep 17 00:00:00 2001 From: bikegeek <3753118+bikegeek@users.noreply.github.com> Date: Fri, 15 Oct 2021 18:43:35 -0600 Subject: [PATCH 23/23] Update installation.rst update cartopy version from 0.17 to 0.18 (the 0.18 version "fixes" the issue with overlapping 180/-180 longitude labels when the central longitude is set to 180). --- docs/Users_Guide/installation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Users_Guide/installation.rst b/docs/Users_Guide/installation.rst index d9dbfdf4df..be724cc5a3 100644 --- a/docs/Users_Guide/installation.rst +++ b/docs/Users_Guide/installation.rst @@ -115,12 +115,12 @@ to run. - MakePlots wrapper - - cartopy (0.17.0) + - cartopy (0.18.0) - pandas (1.0.5) - CyclonePlotter wrapper - - cartopy (0.17.0) + - cartopy (0.18.0) - matplotlib (3.3.4) Cartopy, one of the dependencies of CyclonePlotter, attempts to download shapefiles from the internet to complete successfully. So if CyclonePlotter is run on a closed system (i.e. no internet), additional steps need to be taken. First, go to the Natural Earth Data webpage and download the small scale (1:110m) cultural and physical files that will have multiple extensions (e.g. .dbf, .shp, .shx). Untar these files in a noted location. Finally, create an environment variable in the user-specific system configuration file for CARTOPY_DIR, setting it to the location where the shapefiles are located.