Refactor to eventually remove pyyaml as a required dependency (#652)

.github/workflows/tests.yaml

@@ -122,6 +122,19 @@ jobs:
           name: junit_pydantic
           path: ./junit_py*
 
+  without-pyyaml:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: pip
+      - run: |
+          pip install .[test,all]
+          pip uninstall -y argcomplete omegaconf pyyaml reconplogger responses ruyaml types-PyYAML
+          pytest
+
   build-package:
     runs-on: ubuntu-latest
     steps:
@@ -245,7 +258,7 @@ jobs:
   pypi-publish:
     if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
-    needs: [linux, windows, macos, omegaconf, pydantic-v1, installed-package, doctest, mypy]
+    needs: [linux, windows, macos, omegaconf, pydantic-v1, without-pyyaml, installed-package, doctest, mypy]
     environment:
       name: pypi
       url: https://pypi.org/p/jsonargparse

CHANGELOG.rst

@@ -12,9 +12,15 @@ The semantic versioning only considers the public API as described in
 paths are considered internals and can change in minor and patch releases.
 
 
-v4.35.1 (2024-12-??)
+v4.36.0 (2024-12-??)
 --------------------
 
+Added
+^^^^^
+- Support without ``pyyaml``, though only an internal refactor prior to eventual
+  removal of ``pyyaml`` as a required dependency in v5.0.0 (`#652
+  <https://github.com/omni-us/jsonargparse/pull/652>`__).
+
 Fixed
 ^^^^^
 - Help for ``Protocol`` types not working correctly (`#645

jsonargparse/_actions.py

@@ -201,7 +201,7 @@ def apply_config(parser, cfg, dest, value) -> None:
                         raise ex_path
                     cfg_path = None
                     cfg_file = parser.parse_string(value, **kwargs)
-                except (TypeError,) + get_loader_exceptions() as ex_str:
+                except (TypeError, ValueError) + get_loader_exceptions() as ex_str:  # type: ignore[misc]
                     raise TypeError(f'Parser key "{dest}": {ex_str}') from ex_str
             else:
                 cfg_file = parser.parse_path(value, **kwargs)

jsonargparse/_core.py

@@ -78,6 +78,7 @@
     get_config_read_mode,
     import_fsspec,
     import_jsonnet,
+    pyyaml_available,
 )
 from ._parameter_resolvers import UnknownDefault
 from ._signatures import SignatureArguments
@@ -231,7 +232,7 @@ def __init__(
         logger: Union[bool, str, dict, logging.Logger] = False,
         version: Optional[str] = None,
         print_config: Optional[str] = "--print_config",
-        parser_mode: str = "yaml",
+        parser_mode: str = "yaml" if pyyaml_available else "json",
         dump_header: Optional[List[str]] = None,
         default_config_files: Optional[List[Union[str, os.PathLike]]] = None,
         default_env: bool = False,
@@ -250,7 +251,7 @@ def __init__(
             logger: Configures the logger, see :class:`.LoggerProperty`.
             version: Program version which will be printed by the --version argument.
             print_config: Name for print config argument, ``%s`` is replaced by config dest, set None to disable.
-            parser_mode: Mode for parsing config files: ``'yaml'``, ``'jsonnet'`` or ones added via :func:`.set_loader`.
+            parser_mode: Mode for parsing values: ``yaml``, ``json``, ``jsonnet`` or added via :func:`.set_loader`.
             dump_header: Header to include as comment when dumping a config object.
             default_config_files: Default config file locations, e.g. ``['~/.config/myapp/*.yaml']``.
             default_env: Set the default value on whether to parse environment variables.

jsonargparse/_jsonnet.py

@@ -11,6 +11,7 @@
     get_jsonschema_exceptions,
     import_jsonnet,
     import_jsonschema,
+    pyyaml_available,
 )
 from ._typehints import ActionTypeHint
 from ._util import NoneType, Path, argument_error
@@ -45,10 +46,11 @@ def __init__(
             if schema is not None:
                 jsonvalidator = import_jsonschema("ActionJsonnet")[1]
                 if isinstance(schema, str):
-                    with parser_context(load_value_mode="yaml"):
+                    mode = "yaml" if pyyaml_available else "json"
+                    with parser_context(load_value_mode=mode):
                         try:
                             schema = load_value(schema)
-                        except get_loader_exceptions() as ex:
+                        except get_loader_exceptions(mode) as ex:
                             raise ValueError(f"Problems parsing schema: {ex}") from ex
                 jsonvalidator.check_schema(schema)
                 self._validator = ActionJsonSchema._extend_jsonvalidator_with_default(jsonvalidator)(schema)
@@ -162,7 +164,7 @@ def parse(
             fname = jsonnet(absolute=False) if isinstance(jsonnet, Path) else jsonnet
             snippet = fpath.get_content()
         try:
-            with parser_context(load_value_mode="yaml"):
+            with parser_context(load_value_mode="yaml" if pyyaml_available else "json"):
                 values = load_value(_jsonnet.evaluate_snippet(fname, snippet, ext_vars=ext_vars, ext_codes=ext_codes))
         except RuntimeError as ex:
             raise argument_error(f'Problems evaluating jsonnet "{fname}": {ex}') from ex

jsonargparse/_jsonschema.py

@@ -10,6 +10,7 @@
 from ._optionals import (
     get_jsonschema_exceptions,
     import_jsonschema,
+    pyyaml_available,
 )
 from ._util import parse_value_or_config
 
@@ -35,11 +36,12 @@ def __init__(
         """
         if schema is not None:
             if isinstance(schema, str):
-                with parser_context(load_value_mode="yaml"):
+                mode = "yaml" if pyyaml_available else "json"
+                with parser_context(load_value_mode=mode):
                     try:
                         schema = load_value(schema)
-                    except get_loader_exceptions() as ex:
-                        raise ValueError(f"Problems parsing schema :: {ex}") from ex
+                    except get_loader_exceptions(mode) as ex:
+                        raise ValueError(f"Problems parsing schema: {ex}") from ex
             jsonvalidator = import_jsonschema("ActionJsonSchema")[1]
             jsonvalidator.check_schema(schema)
             self._validator = self._extend_jsonvalidator_with_default(jsonvalidator)(schema)

jsonargparse/_loaders_dumpers.py

@@ -2,12 +2,10 @@
 
 import inspect
 import re
-from typing import Any, Callable, Dict, Tuple, Type
-
-import yaml
+from typing import Any, Callable, Dict, Optional, Tuple, Type
 
 from ._common import load_value_mode, parent_parser
-from ._optionals import import_jsonnet, omegaconf_support
+from ._optionals import import_jsonnet, omegaconf_support, pyyaml_available
 from ._type_checking import ArgumentParser
 
 __all__ = [
@@ -17,44 +15,55 @@
 ]
 
 
-class DefaultLoader(getattr(yaml, "CSafeLoader", yaml.SafeLoader)):  # type: ignore[misc]
-    pass
+yaml_default_loader = None
+
 
+def get_yaml_default_loader():
+    global yaml_default_loader
+    if yaml_default_loader:
+        return yaml_default_loader
 
-# https://stackoverflow.com/a/37958106/2732151
-def remove_implicit_resolver(cls, tag_to_remove):
-    if "yaml_implicit_resolvers" not in cls.__dict__:
-        cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy()
+    import yaml
 
-    for first_letter, mappings in cls.yaml_implicit_resolvers.items():
-        cls.yaml_implicit_resolvers[first_letter] = [(tag, regexp) for tag, regexp in mappings if tag != tag_to_remove]
+    class DefaultLoader(getattr(yaml, "CSafeLoader", yaml.SafeLoader)):
+        pass
 
+    # https://stackoverflow.com/a/37958106/2732151
+    def remove_implicit_resolver(cls, tag_to_remove):
+        if "yaml_implicit_resolvers" not in cls.__dict__:
+            cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy()
 
-remove_implicit_resolver(DefaultLoader, "tag:yaml.org,2002:timestamp")
-remove_implicit_resolver(DefaultLoader, "tag:yaml.org,2002:float")
+        for first_letter, mappings in cls.yaml_implicit_resolvers.items():
+            cls.yaml_implicit_resolvers[first_letter] = [
+                (tag, regexp) for tag, regexp in mappings if tag != tag_to_remove
+            ]
 
+    remove_implicit_resolver(DefaultLoader, "tag:yaml.org,2002:timestamp")
+    remove_implicit_resolver(DefaultLoader, "tag:yaml.org,2002:float")
 
-DefaultLoader.add_implicit_resolver(
-    "tag:yaml.org,2002:float",
-    re.compile(
-        """^(?:
-     [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
-    |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
-    |\\.[0-9_]+(?:[eE][-+][0-9]+)?
-    |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
-    |[-+]?\\.(?:inf|Inf|INF)
-    |\\.(?:nan|NaN|NAN))$""",
-        re.X,
-    ),
-    list("-+0123456789."),
-)
+    DefaultLoader.add_implicit_resolver(
+        "tag:yaml.org,2002:float",
+        re.compile(
+            """^(?:
+        [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
+        |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
+        |\\.[0-9_]+(?:[eE][-+][0-9]+)?
+        |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
+        |[-+]?\\.(?:inf|Inf|INF)
+        |\\.(?:nan|NaN|NAN))$""",
+            re.X,
+        ),
+        list("-+0123456789."),
+    )
+
+    yaml_default_loader = DefaultLoader
+    return DefaultLoader
 
 
 def yaml_load(stream):
-    if stream.strip() == "-":
-        value = stream
-    else:
-        value = yaml.load(stream, Loader=DefaultLoader)
+    import yaml
+
+    value = yaml.load(stream, Loader=get_yaml_default_loader())
     if isinstance(value, dict) and value and all(v is None for v in value.values()):
         if len(value) == 1 and stream.strip() == next(iter(value.keys())) + ":":
             value = stream
@@ -65,33 +74,37 @@ def yaml_load(stream):
     return value
 
 
+def json_load(stream):
+    import json
+
+    try:
+        return json.loads(stream)
+    except json.JSONDecodeError:
+        return stream
+
+
 def jsonnet_load(stream, path="", ext_vars=None):
     from ._jsonnet import ActionJsonnet
 
     ext_vars, ext_codes = ActionJsonnet.split_ext_vars(ext_vars)
     _jsonnet = import_jsonnet("jsonnet_load")
     try:
         val = _jsonnet.evaluate_snippet(path, stream, ext_vars=ext_vars, ext_codes=ext_codes)
-    except RuntimeError as ex:
+    except RuntimeError:
         try:
-            return yaml_load(stream)
-        except pyyaml_exceptions:
+            return json_or_yaml_load(stream)
+        except json_or_yaml_loader_exceptions as ex:
             raise ValueError(str(ex)) from ex
-    return yaml_load(val)
+    return json_or_yaml_load(val)
 
 
 loaders: Dict[str, Callable] = {
     "yaml": yaml_load,
+    "json": json_load,
     "jsonnet": jsonnet_load,
 }
 
-pyyaml_exceptions = (yaml.YAMLError,)
-jsonnet_exceptions = pyyaml_exceptions + (ValueError,)
-
-loader_exceptions: Dict[str, Tuple[Type[Exception], ...]] = {
-    "yaml": pyyaml_exceptions,
-    "jsonnet": jsonnet_exceptions,
-}
+loader_exceptions: Dict[str, Tuple[Type[Exception], ...]] = {}
 
 
 def get_load_value_mode() -> str:
@@ -103,11 +116,28 @@ def get_load_value_mode() -> str:
     return mode
 
 
-def get_loader_exceptions():
-    return loader_exceptions[get_load_value_mode()]
+def get_loader_exceptions(mode: Optional[str] = None) -> Tuple[Type[Exception], ...]:
+    if mode is None:
+        mode = get_load_value_mode()
+    if mode not in loader_exceptions:
+        if mode == "yaml":
+            loader_exceptions[mode] = (__import__("yaml").YAMLError,)
+        elif mode == "json":
+            loader_exceptions[mode] = tuple()
+        elif mode == "jsonnet":
+            return get_loader_exceptions("yaml" if pyyaml_available else "json")
+    return loader_exceptions[mode]
+
+
+json_or_yaml_load = yaml_load if pyyaml_available else json_load
+json_or_yaml_loader_exceptions = get_loader_exceptions("yaml" if pyyaml_available else "json")
 
 
 def load_value(value: str, simple_types: bool = False, **kwargs):
+    if not value:
+        return None
+    elif value.strip() == "-":
+        return value
     loader = loaders[get_load_value_mode()]
     if kwargs:
         params = set(list(inspect.signature(loader).parameters)[1:])
@@ -131,6 +161,8 @@ def load_value(value: str, simple_types: bool = False, **kwargs):
 
 
 def yaml_dump(data):
+    import yaml
+
     return yaml.safe_dump(data, **dump_yaml_kwargs)
 
 
@@ -184,7 +216,11 @@ def dump_using_format(parser: "ArgumentParser", data: dict, dump_format: str) ->
     return dump
 
 
-def set_loader(mode: str, loader_fn: Callable[[str], Any], exceptions: Tuple[Type[Exception], ...] = pyyaml_exceptions):
+def set_loader(
+    mode: str,
+    loader_fn: Callable[[str], Any],
+    exceptions: Tuple[Type[Exception], ...] = tuple(),
+):
     """Sets the value loader function to be used when parsing with a certain mode.
 
     The ``loader_fn`` function must accept as input a single str type parameter
@@ -197,7 +233,7 @@ def set_loader(mode: str, loader_fn: Callable[[str], Any], exceptions: Tuple[Typ
         mode: The parser mode for which to set its loader function. Example: "yaml".
         loader_fn: The loader function to set. Example: ``yaml.safe_load``.
         exceptions: Exceptions that the loader can raise when load fails.
-            Example: (yaml.parser.ParserError, yaml.scanner.ScannerError).
+            Example: (yaml.YAMLError,).
     """
     loaders[mode] = loader_fn
     loader_exceptions[mode] = exceptions

jsonargparse/_namespace.py

@@ -354,6 +354,9 @@ def dict_to_namespace(cfg_dict: Union[Dict[str, Any], Namespace]) -> Namespace:
 
 
 # Temporal to provide backward compatibility in pytorch-lightning
-import yaml  # noqa: E402
+from importlib.util import find_spec  # noqa: E402
 
-yaml.SafeDumper.add_representer(Namespace, lambda d, x: d.represent_mapping("tag:yaml.org,2002:map", x.as_dict()))
+if find_spec("yaml"):
+    import yaml
+
+    yaml.SafeDumper.add_representer(Namespace, lambda d, x: d.represent_mapping("tag:yaml.org,2002:map", x.as_dict()))

jsonargparse/_optionals.py

@@ -14,6 +14,7 @@
 ]
 
 
+pyyaml_available = bool(find_spec("yaml"))
 typing_extensions_support = find_spec("typing_extensions") is not None
 typeshed_client_support = find_spec("typeshed_client") is not None
 jsonschema_support = find_spec("jsonschema") is not None

jsonargparse/_typehints.py

@@ -56,9 +56,9 @@
 )
 from ._loaders_dumpers import (
     get_loader_exceptions,
+    json_or_yaml_load,
+    json_or_yaml_loader_exceptions,
     load_value,
-    pyyaml_exceptions,
-    yaml_load,
 )
 from ._namespace import Namespace
 from ._optionals import (
@@ -762,8 +762,8 @@ def adapt_typehints(
     # Basic types
     elif typehint in leaf_types:
         if isinstance(val, str) and typehint is not str:
-            with suppress(*pyyaml_exceptions):
-                val = yaml_load(val)
+            with suppress(*json_or_yaml_loader_exceptions):
+                val = json_or_yaml_load(val)
         if typehint is float and isinstance(val, int) and not isinstance(val, bool):
             val = float(val)
         if not isinstance(val, typehint) or (typehint in (int, float) and isinstance(val, bool)):