Define all dependences in requirements files

README.md

@@ -88,7 +88,7 @@ is to feel natural for somebody used to coding in Python, and reasonably easy to
 who is not a pythonista.
 
 The dependencies of the operator framework are kept as minimal as possible; currently that's Python
-3.8 or greater, and `PyYAML` (both are included by default in Ubuntu's cloud images from 20.04 on).
+3.8 or greater, and the small number of Python libraries referenced in [requirements.txt](requirements.txt).
 
 For a brief intro on how to get started, check out the
 [Hello, World!](https://juju.is/docs/sdk/hello-world) section of the documentation!

ops/_private/yaml.py

@@ -14,7 +14,7 @@
 
 """Internal YAML helpers."""
 
-from typing import Any, Optional, TextIO, Union, overload
+from typing import Any, Optional, TextIO, Union
 
 import yaml
 
@@ -28,16 +28,6 @@ def safe_load(stream: Union[str, TextIO]) -> Any:
     return yaml.load(stream, Loader=_safe_loader)  # type: ignore
 
 
-@overload
-def safe_dump(data: Any, *args: Any, encoding: None = None, **kwargs: Any) -> str: ...  # noqa
-@overload
-def safe_dump(data: Any, *args: Any, encoding: str = "", **kwargs: Any) -> bytes: ...  # noqa
-
-
-def safe_dump(data: Any, stream: Optional[Union[str, TextIO]] = None, **kwargs: Any  # noqa
-              ) -> Union[str, bytes]:
-    """Same as yaml.safe_dump, but use fast C dumper if available.
-
-    If `encoding:str` is provided, return bytes. Else, return str.
-    """
-    return yaml.dump(data, stream=stream, Dumper=_safe_dumper, **kwargs)  # type: ignore
+def safe_dump(data: Any, stream: Optional[TextIO] = None) -> str:
+    """Same as yaml.safe_dump, but use fast C dumper if available."""
+    return yaml.dump(data, stream=stream, Dumper=_safe_dumper)  # type: ignore

ops/model.py

@@ -56,8 +56,6 @@
 from ops.jujuversion import JujuVersion
 
 if typing.TYPE_CHECKING:
-    from subprocess import CompletedProcess  # noqa
-
     from pebble import (  # pyright: reportMissingTypeStubs=false
         CheckInfo,
         CheckLevel,
@@ -2162,7 +2160,7 @@ def _build_fileinfo(path: StrOrPath) -> 'FileInfo':
             name=path.name,
             type=ftype,
             size=info.st_size,
-            permissions=typing.cast(int, stat.S_IMODE(info.st_mode)),  # type: ignore
+            permissions=stat.S_IMODE(info.st_mode),
             last_modified=datetime.datetime.fromtimestamp(info.st_mtime),
             user_id=info.st_uid,
             user=pwd.getpwuid(info.st_uid).pw_name,
@@ -2552,10 +2550,6 @@ def _run(self, *args: str, return_output: bool = False,
             args += ('--format=json',)
         try:
             result = run(args, **kwargs)
-
-            # pyright infers the first match when argument overloading/unpacking is used,
-            # so this needs to be coerced into the right type
-            result = typing.cast('CompletedProcess[bytes]', result)
         except CalledProcessError as e:
             raise ModelError(e.stderr)
         if return_output:

ops/storage.py

@@ -22,7 +22,7 @@
 import typing
 from datetime import timedelta
 from pathlib import Path
-from typing import Any, Callable, Generator, List, Optional, Tuple, Type, Union
+from typing import Any, Callable, Generator, List, Optional, Tuple, Union
 
 import yaml  # pyright: reportMissingModuleSource=false
 
@@ -299,11 +299,11 @@ def _save_notice_list(self, notices: '_Notices') -> None:
 
 
 # we load yaml.CSafeX if available, falling back to slower yaml.SafeX.
-_BaseDumper = getattr(yaml, 'CSafeDumper', yaml.SafeDumper)  # type: Type[yaml.SafeDumper]
-_BaseLoader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader)  # type: Type[yaml.SafeLoader]
+_BaseDumper = getattr(yaml, 'CSafeDumper', yaml.SafeDumper)
+_BaseLoader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader)
 
 
-class _SimpleLoader(_BaseLoader):
+class _SimpleLoader(_BaseLoader):  # type: ignore
     """Handle a couple basic python types.
 
     yaml.SafeLoader can handle all the basic int/float/dict/set/etc that we want. The only one
@@ -321,7 +321,7 @@ class _SimpleLoader(_BaseLoader):
     _SimpleLoader.construct_python_tuple)  # type: ignore
 
 
-class _SimpleDumper(_BaseDumper):
+class _SimpleDumper(_BaseDumper):  # type: ignore
     """Add types supported by 'marshal'.
 
     YAML can support arbitrary types, but that is generally considered unsafe (like pickle). So
@@ -330,7 +330,7 @@ class _SimpleDumper(_BaseDumper):
     represent_tuple = yaml.Dumper.represent_tuple  # type: _TupleRepresenterType
 
 
-_SimpleDumper.add_representer(tuple, _SimpleDumper.represent_tuple)
+_SimpleDumper.add_representer(tuple, _SimpleDumper.represent_tuple)  # type: ignore
 
 
 def juju_backend_available() -> bool:

requirements-dev.txt

@@ -1,3 +1,15 @@
-PyYAML
-pyright==1.1.257
-typing_extensions==4.2.0
+isort==5.11.4
+logassert==7
+autopep8==1.6.0
+flake8==4.0.1
+flake8-docstrings==1.6.0
+flake8-builtins==2.1.0
+pyproject-flake8==4.0.1
+pep8-naming==0.13.2
+pytest==7.2.1
+pyright==1.1.291
+pytest-operator==0.23.0
+coverage[toml]==7.0.5
+typing_extensions==4.2.0
+
+-r requirements.txt

requirements.txt

@@ -1,2 +1,2 @@
-PyYAML
-websocket-client
+PyYAML==6.*
+websocket-client==1.*

setup.py

@@ -14,9 +14,11 @@
 
 """Setup script for the Operator Framework."""
 
-from importlib.util import spec_from_file_location, module_from_spec
+from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
-from setuptools import setup, find_packages
+from typing import List
+
+from setuptools import find_packages, setup
 
 
 def _read_me() -> str:
@@ -26,6 +28,19 @@ def _read_me() -> str:
     return readme
 
 
+def _requirements() -> List[str]:
+    """Return the required packages to run the project."""
+    reqs = []
+    with open(Path(__file__).parent / 'requirements.txt', encoding='utf-8') as fh:
+        for line in fh.readlines():
+            # TODO(tinvaan): DRY, consider setuptools offering for requirements parsing
+            # https://setuptools.pypa.io/en/latest/pkg_resources.html#requirements-parsing
+            line = line.strip()
+            if line and not line.startswith("#"):
+                reqs.append(line)
+    return reqs
+
+
 def _get_version() -> str:
     """Get the version via ops/version.py, without loading ops/__init__.py."""
     spec = spec_from_file_location('ops.version', 'ops/version.py')
@@ -72,10 +87,7 @@ def _get_version() -> str:
             "Operating System :: POSIX :: Linux",
         ],
         python_requires='>=3.8',
-        install_requires=[
-            'PyYAML',
-            'websocket-client',
-        ],
+        install_requires=_requirements(),
         package_data={'ops': ['py.typed']},
     )
 

test/test_private.py

@@ -41,7 +41,7 @@ def test_safe_dump(self):
         self.assertEqual(s, 'baz: 123\nfoo: bar\n')
 
         f = io.StringIO()
-        s = yaml.safe_dump({'foo': 'bar', 'baz': 123}, stream=f)
+        yaml.safe_dump({'foo': 'bar', 'baz': 123}, stream=f)
         self.assertEqual(f.getvalue(), 'baz: 123\nfoo: bar\n')
 
         # Should error -- it's not safe to dump an instance of a user-defined class

tox.ini

@@ -16,7 +16,6 @@ all_path = {[vars]src_path} {[vars]tst_path}
 basepython = python3
 setenv =
   PYTHONPATH = {toxinidir}:{toxinidir}/lib:{[vars]src_path}
-  PYTHONBREAKPOINT=ipdb.set_trace
   PY_COLORS=1
 passenv =
   PYTHONPATH
@@ -34,22 +33,15 @@ commands =
 [testenv:fmt]
 description = Apply coding style standards to code
 deps =
-    isort
-    autopep8
+    -r{toxinidir}/requirements-dev.txt
 commands =
     isort {[vars]all_path}
     autopep8 --in-place {[vars]all_path}
 
 [testenv:lint]
 description = Check code against coding style standards
 deps =
-    autopep8
-    isort
-    flake8==4.0.1
-    flake8-docstrings
-    flake8-builtins
-    pyproject-flake8
-    pep8-naming
+    -r{toxinidir}/requirements-dev.txt
 commands =
     # pflake8 wrapper suppports config from pyproject.toml
     pflake8 {[vars]all_path}
@@ -59,8 +51,7 @@ commands =
 [testenv:static]
 description = Run static type checker
 deps =
-    pyright==1.1.264
-    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/requirements-dev.txt
 commands =
     pyright {posargs}
 
@@ -70,12 +61,7 @@ passenv =
     RUN_REAL_PEBBLE_TESTS
     PEBBLE
 deps =
-    pytest
-    ipdb
-    ipython!=8.1.0 # this version is broken and was causing failures
-    logassert
-    coverage[toml]
-    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/requirements-dev.txt
 commands =
     coverage run --source={[vars]src_path} \
              -m pytest --ignore={[vars]tst_path}smoke -v --tb native {posargs} 
@@ -90,10 +76,7 @@ setenv =
   PEBBLE=/tmp/pebble
   RUN_REAL_PEBBLE_TESTS=1
 deps =
-    pytest
-    pytest-operator
-    logassert
-    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/requirements-dev.txt
 commands =
     bash -c "umask 0; (pebble run --http=':4000' --create-dirs &>/dev/null & ) ; sleep 1; pytest -v --tb native -k RealPebble {posargs} ; killall -y 3m pebble"
 
@@ -103,8 +86,7 @@ whitelist_externals = juju
                       charmcraft
                       bash
 deps =
-    pytest
-    pytest-operator
+    -r{toxinidir}/requirements-dev.txt
 commands =
     # Build a source tarball for ops, and drop it into the root directory of the smoke test charm.
     bash -c 'rm -vf ./test/charms/test_smoke/*.tar.gz  # Cleanup old builds'