From eaf3591b74b2d40bec2c6ebcbdf163a5885eda95 Mon Sep 17 00:00:00 2001 From: "malcolm.roberts" Date: Mon, 9 Sep 2024 08:35:57 +0100 Subject: [PATCH] Fix bug in atmos_river archiving --- inline_model_storms/tempest_atmos_river.py | 20 +++++++++++--------- inline_model_storms/um_postprocess.py | 15 ++++++++++++--- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/inline_model_storms/tempest_atmos_river.py b/inline_model_storms/tempest_atmos_river.py index 76125bf..c2625f6 100644 --- a/inline_model_storms/tempest_atmos_river.py +++ b/inline_model_storms/tempest_atmos_river.py @@ -166,15 +166,15 @@ def run(self, *args, **kwargs): self.logger.error(f"no dot files to process ") # Test if new year, if so then concatenate all the previous year data into 1 file - is_new_year = (timestamp_day[0:4] != timestamp_previous[0:4]) and \ - _is_date_after(timestamp_previous, self.startdate) + is_end_year = (timestamp_day[:4] != self.next_cycle[:4]) and \ + _is_date_after(timestamp_day, self.startdate) - if is_new_year: - self._produce_annual_ar_file(self.outdir, timestamp_previous[0:4]) + if is_end_year: + self._produce_annual_ar_file(self.outdir, timestamp_day[0:4]) - if self.is_last_cycle == "true" or is_new_year: + if self.is_last_cycle == "true" or is_end_year: # archive any remaining AR data - self._archive_ar_data(self.outdir, is_new_year, timestamp_previous[0:4]) + self._archive_ar_data(self.outdir, is_end_year, timestamp_day[0:4]) def _run_detect_blobs(self, timestamp): """ @@ -328,8 +328,9 @@ def _produce_annual_ar_file( ) raise RuntimeError(msg) else: - for f in files_to_join: - os.remove(f) + if os.path.exists(file_year): + for f in files_to_join: + os.remove(f) def _archive_ar_data(self, outdir, is_new_year, year): """ @@ -354,7 +355,8 @@ def _archive_ar_data(self, outdir, is_new_year, year): shutil.copy(ar_file, ar_archive_file) else: os.replace(ar_file, ar_archive_file) - os.remove(ar_file) + if os.path.exists(ar_file): + os.remove(ar_file) with open(ar_archive_file + ".arch", "a"): os.utime(ar_archive_file + ".arch", None) diff --git a/inline_model_storms/um_postprocess.py b/inline_model_storms/um_postprocess.py index e3eba05..5b086f5 100644 --- a/inline_model_storms/um_postprocess.py +++ b/inline_model_storms/um_postprocess.py @@ -154,13 +154,21 @@ def _archive_tracking_to_mass( continue archive_error = False - # the .arch is in the archive_dir, but the file is in + # the .arch is in the archive_dir, but the file may be in # the directory above fname = os.path.join(os.path.dirname( fname_arch), '../', os.path.basename(fname_arch)[:-5]) + if not os.path.exists(fname): - self.logger.debug(f"File does not exist for archiving {fname}") + fname1 = os.path.join(os.path.dirname( + fname_arch), + os.path.basename(fname_arch)[:-5]) + + if not os.path.exists(fname1): + self.logger.debug(f"File does not exist for archiving {fname} {fname1}") + else: + fname = fname1 if fname[-2:] == "nc": mass_stream = "any.nc.file" @@ -186,7 +194,8 @@ def _archive_tracking_to_mass( archive_error = True else: #os.remove(fname) - os.rename(fname_arch, fname_archived) + if 'archive' not in os.path.dirname(fname): + os.rename(fname_arch, fname_archived) self.logger.debug(sts.stdout) if archive_error: