Add support for configure using TOML files (#444)

docs/changelog.rst

@@ -14,6 +14,7 @@ Added
 - docs now include a dropdown for selecting older versions of the docs. (#457)
 - Support for loading the same data source but from different places (e.g. local & aws)
 - Add support for reading and writing tabular data in ``parquet`` format. (PR #445)
+- Add support for reading model configs in ``TOML`` format. (PR #444)
 
 Changed
 -------

hydromt/cli/api.py

@@ -94,7 +94,7 @@ def get_datasets(data_libs: Union[List, str]) -> Dict:
     Parameters
     ----------
     data_libs: (list of) str, Path, optional
-        One or more paths to data catalog yaml files or names of predefined
+        One or more paths to data catalog configuration files or names of predefined
         data catalogs. By default the data catalog is initiated without data entries.
         See :py:func:`~hydromt.data_adapter.DataCatalog.from_yml`
         for accepted yaml format.
@@ -139,7 +139,7 @@ def get_region(
     region : dict
         dictionary containing region definition
     data_libs : (list of) str, Path, optional
-        One or more paths to data catalog yaml files or names of predefined
+        One or more paths to data catalog configuration files or names of predefined
         data catalogs. By default the data catalog is initiated without data entries.
         See :py:func:`~hydromt.data_adapter.DataCatalog.from_yml`
         for accepted yaml format.

hydromt/config.py

@@ -9,13 +9,43 @@
 from typing import Dict, List, Union
 
 import yaml
+from tomli import load as load_toml
+from tomli_w import dump as dump_toml
 
 __all__ = [
     "configread",
     "configwrite",
 ]
 
 
+def _process_config_out(d):
+    ret = {}
+    if isinstance(d, dict):
+        for k, v in d.items():
+            if v is None:
+                ret[k] = "NONE"
+            else:
+                ret[k] = _process_config_out(v)
+    else:
+        ret = d
+
+    return ret
+
+
+def _process_config_in(d):
+    ret = {}
+    if isinstance(d, dict):
+        for k, v in d.items():
+            if v == "NONE":
+                ret[k] = None
+            else:
+                ret[k] = _process_config_in(v)
+    else:
+        ret = d
+
+    return ret
+
+
 def configread(
     config_fn: Union[Path, str],
     defaults: Dict = dict(),
@@ -48,15 +78,22 @@ def configread(
         Configuration dictionary.
     """
     # read
-    if splitext(config_fn)[-1] in [".yaml", ".yml"]:
+    ext = splitext(config_fn)[-1].strip()
+    if ext in [".yaml", ".yml"]:
         with open(config_fn, "rb") as f:
             cfdict = yaml.safe_load(f)
+        cfdict = _process_config_in(cfdict)
+    elif ext == ".toml":  # user defined
+        with open(config_fn, "rb") as f:
+            cfdict = load_toml(f)
+        cfdict = _process_config_in(cfdict)
     else:
         cfdict = read_ini_config(config_fn, **kwargs)
     # parse absolute paths
     if abs_path:
         root = Path(dirname(config_fn))
         cfdict = parse_abspath(cfdict, root, skip_abspath_sections)
+
     # update defaults
     if defaults:
         _cfdict = defaults.copy()
@@ -89,9 +126,15 @@ def configwrite(config_fn: Union[str, Path], cfdict: dict, **kwargs) -> None:
     """
     root = Path(dirname(config_fn))
     _cfdict = parse_relpath(cfdict.copy(), root)
-    if splitext(config_fn)[-1] in [".yaml", ".yml"]:
+    ext = splitext(config_fn)[-1].strip()
+    if ext in [".yaml", ".yml"]:
+        _cfdict = _process_config_out(_cfdict)  # should not be done for ini
         with open(config_fn, "w") as f:
             yaml.dump(_cfdict, f, sort_keys=False)
+    elif ext == ".toml":  # user defined
+        _cfdict = _process_config_out(_cfdict)
+        with open(config_fn, "wb") as f:
+            dump_toml(_cfdict, f)
     else:
         write_ini_config(config_fn, _cfdict, **kwargs)
 

hydromt/data_catalog.py

@@ -71,9 +71,9 @@ def __init__(
         Arguments
         ---------
         data_libs: (list of) str, Path, optional
-            One or more paths to data catalog yaml files or names of predefined data
-            catalogs. By default the data catalog is initiated without data entries.
-            See :py:func:`~hydromt.data_adapter.DataCatalog.from_yml` for
+            One or more paths to data catalog configuration files or names of predefined
+            data catalogs. By default the data catalog is initiated without data
+            entries. See :py:func:`~hydromt.data_adapter.DataCatalog.from_yml` for
             accepted yaml format.
         fallback_lib:
             Name of pre-defined data catalog to read if no data_libs are provided,
@@ -499,7 +499,7 @@ def from_yml(
         {'RasterDataset', 'GeoDataset', 'GeoDataFrame'}. See the specific data adapters
         for more information about the required and optional arguments.
 
-        .. code-block:: console
+        .. code-block:: yaml
 
             meta:
               root: <path>
@@ -703,7 +703,7 @@ def to_dict(
             List of source names to export, by default None in which case all sources
             are exported.
         root : str, Path, optional
-            Global root for all relative paths in yml file.
+            Global root for all relative paths in the file.
         meta: dict, optional
             key-value pairs to add to the data catalog meta section, such as 'version',
             by default empty.

hydromt/models/model_api.py

@@ -77,7 +77,7 @@ def __init__(
             Model simulation configuration file, by default None.
             Note that this is not the HydroMT model setup configuration file!
         data_libs : List[str], optional
-            List of data catalog yaml files, by default None
+            List of data catalog configuration files, by default None
         **artifact_keys:
             Additional keyword arguments to be passed down.
         logger:
@@ -548,7 +548,7 @@ def write_data_catalog(
         Parameters
         ----------
         root: str, Path, optional
-            Global root for all relative paths in yaml file.
+            Global root for all relative paths in configuration file.
             If "auto" the data source paths are relative to the yaml output ``path``.
         data_lib_fn: str, Path, optional
             Path of output yml file, absolute or relative to the model root,
@@ -560,7 +560,7 @@ def write_data_catalog(
         """
         path = data_lib_fn if isabs(data_lib_fn) else join(self.root, data_lib_fn)
         cat = DataCatalog(logger=self.logger, fallback_lib=None)
-        # read hydromt_data yaml file and add to data catalog
+        # read hydromt_data configuration file and add to data catalog
         if self._read and isfile(path) and append:
             cat.from_yml(path)
         # update data catalog with new used sources

pyproject.toml

@@ -28,6 +28,8 @@ dependencies = [
     "rasterio",          # raster wrapper around gdal
     "shapely>=2.0.0",    # geometry transforms
     "scipy",             # scientific utilities
+    "tomli",             # parsing toml files
+    "tomli-w",           # writing toml files
     "xarray",            # ndim data
     "universal_pathlib", # provides path compatability between different filesystems
     "xmltodict",         # xml parser also used to read VRT

tests/test_config.py

@@ -5,7 +5,7 @@
 from hydromt import config
 
 
-@pytest.mark.parametrize("ext", ["ini", "yaml"])
+@pytest.mark.parametrize("ext", ["ini", "yaml", "toml"])
 def test_config(tmpdir, ext):
     cfdict = {
         "section1": {