Data Request

.github/workflows/CI-test.yaml

@@ -15,6 +15,8 @@ jobs:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v3
         with:

setup.cfg

@@ -4,3 +4,8 @@ style = pep440
 versionfile_source = src/pymorize/_version.py
 versionfile_build = pymorize/_version.py
 tag_prefix = 'v'
+[black]
+max-line-length = 120
+[flake8]
+max-line-length = 120
+exclude = cmip6-cmor-tables/CMIP6_CVs/src

src/pymorize/cmorizer.py

@@ -4,7 +4,7 @@
 from dask.distributed import Client
 from rich.progress import track
 
-# from . import logging_helper
+from .data_request import DataRequest, DataRequestTable, IgnoreTableFiles
 from .logging import logger
 from .pipeline import Pipeline
 from .rule import Rule
@@ -27,6 +27,49 @@ def __init__(
 
         self._post_init_create_pipelines()
         self._post_init_create_rules()
+        self._post_init_read_bare_tables()
+        self._post_init_create_data_request()
+        self._post_init_populate_rules_with_tables()
+
+    def _post_init_read_bare_tables(self):
+        """
+        Loads all the tables from table directory as a mapping object.
+        A shortened version of the filename (i.e., ``CMIP6_Omon.json`` -> ``Omon``) is used as the mapping key.
+        The same key format is used in CMIP6_table_id.json
+        """
+        table_dir = Path(self._general_cfg["CMIP_Tables_Dir"])
+        table_files = {
+            path.stem.replace("CMIP6_", ""): path for path in table_dir.glob("*.json")
+        }
+        tables = {}
+        ignore_files = set(ignore_file.value for ignore_file in IgnoreTableFiles)
+        for tbl_name, tbl_file in table_files.items():
+            logger.debug(f"{tbl_name}, {tbl_file}")
+            if tbl_file.name not in ignore_files:
+                logger.debug(f"Adding Table {tbl_name}")
+                tables[tbl_name] = DataRequestTable(tbl_file)
+        self._general_cfg["tables"] = self.tables = tables
+
+    def _post_init_create_data_request(self):
+        """
+        Creates a DataRequest object from the tables directory.
+        """
+        table_dir = self._general_cfg["CMIP_Tables_Dir"]
+        self.data_request = DataRequest.from_tables_dir(table_dir)
+
+    def _post_init_populate_rules_with_tables(self):
+        """
+        Populates the rules with the tables in which the variable described by that rule is found.
+        """
+        tables = self._general_cfg["tables"]
+        for rule in self.rules:
+            for tbl in tables.values():
+                if rule.cmor_variable in tbl.variable_ids:
+                    rule.add_table(tbl)
+
+    def _post_init_data_request_variables(self):
+        for rule in self.rules:
+            rule.cmor_variable
 
     def _post_init_create_pipelines(self):
         pipelines = []
@@ -50,12 +93,14 @@ def _post_init_checks(self):
     @classmethod
     def from_dict(cls, data):
         instance = cls(
-            pymorize_cfg=data.get("pymorize_cfg", {}),
-            general_cfg=data.get("general_cfg", {}),
+            pymorize_cfg=data.get("pymorize", {}),
+            general_cfg=data.get("general", {}),
         )
         for rule in data.get("rules", []):
             rule_obj = Rule.from_dict(rule)
             instance.add_rule(rule_obj)
+        instance._post_init_populate_rules_with_tables()
+        instance._post_init_create_data_request()
         for pipeline in data.get("pipelines", []):
             pipeline_obj = Pipeline.from_dict(pipeline)
             instance.add_pipeline(pipeline_obj)

src/pymorize/controlled_vocabularies.py

@@ -0,0 +1,68 @@
+"""
+Controlled vocabularies for CMIP6
+"""
+
+import glob
+import json
+import os
+
+
+class ControlledVocabularies(dict):
+    """Controlled vocabularies for CMIP6"""
+
+    def __init__(self, json_files):
+        """Create a new ControlledVocabularies object from a list of json files
+
+        Parameters
+        ----------
+        json_files : list
+            List of json files to load
+
+        Returns
+        -------
+        ControlledVocabularies
+            A new ControlledVocabularies object, behaves like a dictionary.
+        """
+        super().__init__()
+        for f in json_files:
+            d = self.dict_from_json_file(f)
+            self.update(d)
+
+    @classmethod
+    def new_from_dir(cls, cmip6_cvs_dir):
+        """Create a new ControlledVocabularies object from a directory of json files
+
+        Parameters
+        ----------
+        cmip6_cvs_dir : str
+            Path to the directory containing the json files
+        """
+        json_files = glob.glob(os.path.join(cmip6_cvs_dir, "*.json"))
+        return cls(json_files)
+
+    def print_experiment_ids(self):
+        """Print experiment ids with start and end years and parent experiment ids"""
+        for k, v in self["experiment_id"].items():
+            print(
+                f"{k} {v['start_year']}-{v['end_year']} parent:{', '.join(v['parent_experiment_id'])}"
+            )
+
+    @staticmethod
+    def dict_from_json_file(path):
+        """Load a json file into a dictionary object
+
+        Parameters
+        ----------
+        path : str
+            Path to the json file to load
+
+        Raises
+        ------
+        ValueError
+            If the file cannot be loaded
+        """
+        try:
+            with open(path, "r") as file:
+                return json.load(file)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"file {path}: {e.msg}")