Added substitutions for init file

padocc/core/project.py

@@ -177,7 +177,7 @@ def run(
             self._dryrun = dryrun
 
         try:
-            status = self._run(mode)
+            status = self._run(mode=mode)
             self.save_files()
             return status
         except Exception as err:

padocc/operations/mixins.py

@@ -28,7 +28,7 @@ class InitialisationMixin:
     def init_from_stac(self):
         pass
 
-    def init_from_file(self, input_file: str):
+    def init_from_file(self, input_file: str, substitutions: dict = None):
         """
         Run initialisation by loading configurations from input sources, determine
         input file type and use appropriate functions to instantiate group and project
@@ -38,6 +38,9 @@ def init_from_file(self, input_file: str):
 
         :returns:   None
         """
+
+        substitutions = substitutions or {}
+
         self.logger.info('Starting initialisation')
 
         if not input_file:
@@ -73,7 +76,7 @@ def init_from_file(self, input_file: str):
                 self.logger.debug('Ingesting csv file')
 
                 group_config = extract_file(input_file)
-            self._init_group(group_config)
+            self._init_group(group_config, substitutions=substitutions)
 
         else:
             # Only base-cfg style files are accepted here.
@@ -105,14 +108,17 @@ def _init_project(self, config: dict):
 
         proj_op.save_files()
 
-    def _init_group(self, datasets : list):
+    def _init_group(self, datasets : list, substitutions: dict = None):
         """
         Create a new group within the working directory, and all 
         associated projects.
         """
 
         self.logger.info('Creating project directories')
         # Group config is the contents of datasets.csv
+        if substitutions:
+            datasets = _apply_substitutions('init_file',subs=substitutions, content=datasets)
+
         self.datasets.set(datasets)
 
         if 'proj_code' in datasets[0]:
@@ -127,8 +133,13 @@ def _open_json(file):
             cfg_values = {}
             ds_values  = datasets[index].split(',')
 
-            proj_code               = ds_values[0]
-            cfg_values['pattern']   = ds_values[1]
+            proj_code = ds_values[0]
+            pattern   = ds_values[1]
+
+            if ds_values[1].endswith('.txt') and substitutions:
+                pattern = _apply_substitutions('dataset_file', subs=substitutions, content=[pattern])[0]
+
+            cfg_values['pattern'] = pattern
             proj_codes.append(proj_code)
 
             if len(ds_values) > 2:
@@ -719,5 +730,16 @@ def _get_input(
 
     return config
 
+def _apply_substitutions(subkey: str, subs: dict = None, content: list = None):
+    if not subs:
+        return content, ""
+
+    if subkey not in subs:
+        return content, f"Subkey {subkey} is not valid for substitutions"
+
+    content = '\n'.join(content)
+    for f, r in subs[subkey].items():
+        content = content.replace(f,r)
+    return content.split('\n')
 
 

padocc/phases/compute.py

@@ -30,7 +30,8 @@
     ConcatFatalError,
     SourceNotFoundError,
     ValidationError,
-    IdenticalVariablesError
+    IdenticalVariablesError,
+    ComputeError
 )
 
 from padocc.phases.validate import validate_selection
@@ -291,6 +292,16 @@ def __init__(
 
         self.logger.debug('Finished all setup steps')
 
+    def _run(self, mode: str = 'kerchunk'):
+        """
+        Default _run hook for compute operations. A user should aim to use the
+        configuration options to use the Kerchunk or Zarr DS classes rather than
+        the bare parent class. Override this class with a _run parameter for new 
+        DS classes (COG etc.)"""
+
+        self.logger.error('Nothing to do with this class - use KerchunkDS/ZarrDS instead!')
+        raise ComputeError
+
     def _run_with_timings(self, func):
         """
         Configure all required steps for Kerchunk processing.
@@ -911,7 +922,7 @@ def _identify_mode(self):
         self.mode = 'zarr'
         self.fmt = '.zarr'
 
-    def _run(self) -> None:
+    def _run(self, **kwargs) -> None:
         """
         Recommended way of running an operation - includes timers etc.
         """

tests/test_init.py

@@ -12,12 +12,21 @@ def test_init_basic(self, wd=WORKDIR):
 
         kwargs = {}
 
+        substitutions = {
+            'init_file': {
+                '/home/users/dwest77/cedadev/padocc/':''
+            },
+            'dataset_file': {
+                '/home/users/dwest77/cedadev/padocc/':''
+            },
+        }
+
         process = GroupOperation(
             groupID,
             workdir=workdir,
             label='test_init')
 
-        process.init_from_file(infile)
+        process.init_from_file(infile, substitutions=substitutions)
 
 if __name__ == '__main__':
     TestInit().test_init_basic()