Merge pull request #1060 from metno/read-colocated-files-857

Read colocated files made outside of pyaerocom

pyaerocom/aeroval/experiment_processor.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import glob
 import logging
 
 from pyaerocom.aeroval._processing_base import HasColocator, ProcessingEngine
@@ -52,6 +53,18 @@ def _run_single_entry(self, model_name, obs_name, var_list):
                 f"marked to be used only as part of a superobs "
                 f"network"
             )
+        elif ocfg["only_json"]:
+            if not ocfg["coldata_dir"]:
+                raise Exception(
+                    f"No coldata_dir provided for an obs network for which only_json=True. The assumption of setting only_json=True is that colocated files already exist, and so a directory for these files must be provided."
+                )
+            else:
+                preprocessed_coldata_dir = ocfg["coldata_dir"]
+                mask = f"{preprocessed_coldata_dir}/*.nc"
+                files_to_convert = glob.glob(mask)
+                engine = ColdataToJsonEngine(self.cfg)
+                engine.run(files_to_convert)
+
         else:
             col = self.get_colocator(model_name, obs_name)
             if self.cfg.processing_opts.only_json:

pyaerocom/aeroval/obsentry.py

@@ -49,6 +49,15 @@ class ObsEntry(BrowseDict):
     read_opts_ungridded : :obj:`dict`, optional
         dictionary that specifies reading constraints for ungridded reading
         (c.g. :class:`pyaerocom.io.ReadUngridded`).
+    only_json : bool
+        Only to be set if the obs entry already has colocated data files which were
+        preprocessed outside of pyaerocom. Setting to True will skip the colcoation
+        and just create the JSON output.
+    coldata_dir : str
+        Only to be set if the obs entry already has colocated data files which were
+        preprocessed outside of pyaerocom. This is the directory in which the
+        colocated data files are located.
+
     """
 
     SUPPORTED_VERT_CODES = ["Column", "Profile", "Surface"]  # , "2D"]
@@ -74,6 +83,9 @@ def __init__(self, **kwargs):
         self.profile_layer_limits = None
 
         self.read_opts_ungridded = {}
+        # attributes for reading colocated data files made outside of pyaerocom
+        self.only_json = False
+        self.coldata_dir = None
 
         self.update(**kwargs)
         self.check_cfg()