Fix bug in atmos_river archiving

robertsmalcolm · Sep 9, 2024 · eaf3591 · eaf3591
1 parent 1c1f7e4
commit eaf3591
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 12 deletions.
diff --git a/inline_model_storms/tempest_atmos_river.py b/inline_model_storms/tempest_atmos_river.py
@@ -166,15 +166,15 @@ def run(self, *args, **kwargs):
             self.logger.error(f"no dot files to process ")
 
         # Test if new year, if so then concatenate all the previous year data into 1 file
-        is_new_year = (timestamp_day[0:4] != timestamp_previous[0:4]) and \
-                _is_date_after(timestamp_previous, self.startdate)
+        is_end_year = (timestamp_day[:4] != self.next_cycle[:4]) and \
+                _is_date_after(timestamp_day, self.startdate)
 
-        if is_new_year:
-            self._produce_annual_ar_file(self.outdir, timestamp_previous[0:4])
+        if is_end_year:
+            self._produce_annual_ar_file(self.outdir, timestamp_day[0:4])
 
-        if self.is_last_cycle == "true" or is_new_year:
+        if self.is_last_cycle == "true" or is_end_year:
             # archive any remaining AR data
-            self._archive_ar_data(self.outdir, is_new_year, timestamp_previous[0:4])
+            self._archive_ar_data(self.outdir, is_end_year, timestamp_day[0:4])
 
     def _run_detect_blobs(self, timestamp):
         """
@@ -328,8 +328,9 @@ def _produce_annual_ar_file(
                 )
                 raise RuntimeError(msg)
             else:
-                for f in files_to_join:
-                    os.remove(f)
+                if os.path.exists(file_year):
+                    for f in files_to_join:
+                        os.remove(f)
 
     def _archive_ar_data(self, outdir, is_new_year, year):
         """
@@ -354,7 +355,8 @@ def _archive_ar_data(self, outdir, is_new_year, year):
                     shutil.copy(ar_file, ar_archive_file)
                 else:
                     os.replace(ar_file, ar_archive_file)
-                    os.remove(ar_file)
+                    if os.path.exists(ar_file):
+                        os.remove(ar_file)
 
                 with open(ar_archive_file + ".arch", "a"):
                     os.utime(ar_archive_file + ".arch", None)

diff --git a/inline_model_storms/um_postprocess.py b/inline_model_storms/um_postprocess.py
@@ -154,13 +154,21 @@ def _archive_tracking_to_mass(
                     continue
 
                 archive_error = False
-                # the .arch is in the archive_dir, but the file is in 
+                # the .arch is in the archive_dir, but the file may be in 
                 # the directory above
                 fname = os.path.join(os.path.dirname(
                     fname_arch), '../', 
                     os.path.basename(fname_arch)[:-5])
+
                 if not os.path.exists(fname):
-                    self.logger.debug(f"File does not exist for archiving {fname}")
+                    fname1 = os.path.join(os.path.dirname(
+                        fname_arch), 
+                        os.path.basename(fname_arch)[:-5])
+
+                    if not os.path.exists(fname1):
+                        self.logger.debug(f"File does not exist for archiving {fname} {fname1}")
+                    else:
+                        fname = fname1
 
                 if fname[-2:] == "nc":
                     mass_stream = "any.nc.file"
@@ -186,7 +194,8 @@ def _archive_tracking_to_mass(
                     archive_error = True
                 else:
                     #os.remove(fname)
-                    os.rename(fname_arch, fname_archived)
+                    if 'archive' not in os.path.dirname(fname):
+                        os.rename(fname_arch, fname_archived)
                 self.logger.debug(sts.stdout)
 
                 if archive_error: